pg_query 6.0.0 → 6.2.1

data/ext/pg_query/src_backend_utils_activity_pgstat_database.c

@@ -130,8 +130,8 @@ __thread SessionEndType pgStatSessionEndCause = DISCONNECT_NORMAL;
 /*
  * Flush out pending stats for the entry
  *
- * If nowait is true, this function returns false if lock could not
- * immediately acquired, otherwise true is returned.
+ * If nowait is true and the lock could not be immediately acquired, returns
+ * false without flushing the entry.  Otherwise returns true.
  */
 #define PGSTAT_ACCUM_DBCOUNT(item)		\
 	(sharedent)->stats.item += (pendingent)->item

data/ext/pg_query/src_backend_utils_error_elog.c

@@ -1939,6 +1939,17 @@ write_stderr(const char *fmt,...)
 }
 
 
+
+
+/*
+ * Write errors to stderr (or by equal means when stderr is
+ * not available) - va_list version
+ */
+#ifdef WIN32
+#endif
+#ifndef WIN32
+#else
+#endif
 #ifdef WIN32
 __thread volatile int pg_signal_queue;
 

data/ext/pg_query/src_backend_utils_mb_mbutils.c

@@ -17,6 +17,8 @@
  * - pg_mbstrlen_with_len
  * - pg_mblen
  * - SetDatabaseEncoding
+ * - pg_mbcharcliplen
+ * - pg_mbstrlen
  * - GetMessageEncoding
  * - MessageEncoding
  *--------------------------------------------------------------------
@@ -386,7 +388,22 @@ pg_mblen(const char *mbstr)
 
 
 /* returns the length (counted in wchars) of a multibyte string */
+int
+pg_mbstrlen(const char *mbstr)
+{
+	int			len = 0;
 
+	/* optimization for single byte encoding */
+	if (pg_database_encoding_max_length() == 1)
+		return strlen(mbstr);
+
+	while (*mbstr)
+	{
+		mbstr += pg_mblen(mbstr);
+		len++;
+	}
+	return len;
+}
 
 /* returns the length (counted in wchars) of a multibyte string
  * (not necessarily NULL terminated)
@@ -425,7 +442,7 @@ pg_mbcliplen(const char *mbstr, int len, int limit)
 }
 
 /*
- * pg_mbcliplen with specified encoding
+ * pg_mbcliplen with specified encoding; string must be valid in encoding
  */
 int
 pg_encoding_mbcliplen(int encoding, const char *mbstr,
@@ -459,7 +476,29 @@ pg_encoding_mbcliplen(int encoding, const char *mbstr,
  * Similar to pg_mbcliplen except the limit parameter specifies the
  * character length, not the byte length.
  */
+int
+pg_mbcharcliplen(const char *mbstr, int len, int limit)
+{
+	int			clen = 0;
+	int			nch = 0;
+	int			l;
 
+	/* optimization for single byte encoding */
+	if (pg_database_encoding_max_length() == 1)
+		return cliplen(mbstr, len, limit);
+
+	while (len > 0 && *mbstr)
+	{
+		l = pg_mblen(mbstr);
+		nch++;
+		if (nch > limit)
+			break;
+		clen += l;
+		len -= l;
+		mbstr += l;
+	}
+	return clen;
+}
 
 /* mbcliplen for any single-byte encoding */
 static int
@@ -726,12 +765,12 @@ pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
  * report_invalid_encoding: complain about invalid multibyte character
  *
  * note: len is remaining length of string, not length of character;
- * len must be greater than zero, as we always examine the first byte.
+ * len must be greater than zero (or we'd neglect initializing "buf").
  */
 void
 report_invalid_encoding(int encoding, const char *mbstr, int len)
 {
-	int			l = pg_encoding_mblen(encoding, mbstr);
+	int			l = pg_encoding_mblen_or_incomplete(encoding, mbstr, len);
 	char		buf[8 * 5 + 1];
 	char	   *p = buf;
 	int			j,
@@ -758,7 +797,7 @@ report_invalid_encoding(int encoding, const char *mbstr, int len)
  * report_untranslatable_char: complain about untranslatable character
  *
  * note: len is remaining length of string, not length of character;
- * len must be greater than zero, as we always examine the first byte.
+ * len must be greater than zero (or we'd neglect initializing "buf").
  */
 
 

data/ext/pg_query/src_backend_utils_mmgr_alignedalloc.c

@@ -54,6 +54,7 @@ AlignedAllocFree(void *pointer)
 			 GetMemoryChunkContext(unaligned)->name, chunk);
 #endif
 
+	/* Recursively pfree the unaligned chunk */
 	pfree(unaligned);
 }
 
@@ -105,18 +106,32 @@ AlignedAllocRealloc(void *pointer, Size size, int flags)
 	Assert(old_size >= redirchunk->requested_size);
 #endif
 
+	/*
+	 * To keep things simple, we always allocate a new aligned chunk and copy
+	 * data into it.  Because of the above inaccuracy, this may end in copying
+	 * more data than was in the original allocation request size, but that
+	 * should be OK.
+	 */
 	ctx = GetMemoryChunkContext(unaligned);
 	newptr = MemoryContextAllocAligned(ctx, size, alignto, flags);
 
-	/*
-	 * We may memcpy beyond the end of the original allocation request size,
-	 * so we must mark the entire allocation as defined.
-	 */
-	if (likely(newptr != NULL))
+	/* Cope cleanly with OOM */
+	if (unlikely(newptr == NULL))
 	{
-		VALGRIND_MAKE_MEM_DEFINED(pointer, old_size);
-		memcpy(newptr, pointer, Min(size, old_size));
+		VALGRIND_MAKE_MEM_NOACCESS(redirchunk, sizeof(MemoryChunk));
+		return MemoryContextAllocationFailure(ctx, size, flags);
 	}
+
+	/*
+	 * We may memcpy more than the original allocation request size, which
+	 * would result in trying to copy trailing bytes that the original
+	 * MemoryContextAllocAligned call marked NOACCESS.  So we must mark the
+	 * entire old_size as defined.  That's slightly annoying, but probably not
+	 * worth improving.
+	 */
+	VALGRIND_MAKE_MEM_DEFINED(pointer, old_size);
+	memcpy(newptr, pointer, Min(size, old_size));
+
 	pfree(unaligned);
 
 	return newptr;

data/ext/pg_query/src_backend_utils_mmgr_aset.c

@@ -1632,9 +1632,9 @@ AllocSetCheck(MemoryContext context)
 		 prevblock = block, block = block->next)
 	{
 		char	   *bpoz = ((char *) block) + ALLOC_BLOCKHDRSZ;
-		long		blk_used = block->freeptr - bpoz;
-		long		blk_data = 0;
-		long		nchunks = 0;
+		Size		blk_used = block->freeptr - bpoz;
+		Size		blk_data = 0;
+		Size		nchunks = 0;
 		bool		has_external_chunk = false;
 
 		if (IsKeeperBlock(set, block))

data/ext/pg_query/src_backend_utils_mmgr_bump.c

@@ -236,7 +236,7 @@ BumpAllocLarge(MemoryContext context, Size size, int flags)
 
 	block = (BumpBlock *) malloc(blksize);
 	if (block == NULL)
-		return NULL;
+		return MemoryContextAllocationFailure(context, size, flags);
 
 	context->mem_allocated += blksize;
 

data/ext/pg_query/src_common_stringinfo.c

@@ -9,6 +9,8 @@
  * - enlargeStringInfo
  * - appendStringInfo
  * - appendStringInfoSpaces
+ * - makeStringInfo
+ * - destroyStringInfo
  *--------------------------------------------------------------------
  */
 
@@ -51,7 +53,17 @@
  *
  * Create an empty 'StringInfoData' & return a pointer to it.
  */
+StringInfo
+makeStringInfo(void)
+{
+	StringInfo	res;
+
+	res = (StringInfo) palloc(sizeof(StringInfoData));
 
+	initStringInfo(res);
+
+	return res;
+}
 
 /*
  * initStringInfo
@@ -350,4 +362,12 @@ enlargeStringInfo(StringInfo str, int needed)
  * Frees a StringInfo and its buffer (opposite of makeStringInfo()).
  * This must only be called on palloc'd StringInfos.
  */
+void
+destroyStringInfo(StringInfo str)
+{
+	/* don't allow destroys of read-only StringInfos */
+	Assert(str->maxlen != 0);
 
+	pfree(str->data);
+	pfree(str);
+}

data/ext/pg_query/src_common_wchar.c

@@ -76,6 +76,7 @@
  * - pg_johab_dsplen
  * - pg_johab_verifychar
  * - pg_johab_verifystr
+ * - pg_encoding_mblen_or_incomplete
  * - pg_encoding_mblen
  *--------------------------------------------------------------------
  */
@@ -94,10 +95,31 @@
  */
 #include "c.h"
 
+#include <limits.h>
+
 #include "mb/pg_wchar.h"
 #include "utils/ascii.h"
 
 
+/*
+ * In today's multibyte encodings other than UTF8, this two-byte sequence
+ * ensures pg_encoding_mblen() == 2 && pg_encoding_verifymbstr() == 0.
+ *
+ * For historical reasons, several verifychar implementations opt to reject
+ * this pair specifically.  Byte pair range constraints, in encoding
+ * originator documentation, always excluded this pair.  No core conversion
+ * could translate it.  However, longstanding verifychar implementations
+ * accepted any non-NUL byte.  big5_to_euc_tw and big5_to_mic even translate
+ * pairs not valid per encoding originator documentation.  To avoid tightening
+ * core or non-core conversions in a security patch, we sought this one pair.
+ *
+ * PQescapeString() historically used spaces for BYTE1; many other values
+ * could suffice for BYTE1.
+ */
+#define NONUTF8_INVALID_BYTE0 (0x8d)
+#define NONUTF8_INVALID_BYTE1 (' ')
+
+
 /*
  * Operations on multi-byte encodings are driven by a table of helper
  * functions.
@@ -1547,6 +1569,11 @@ pg_big5_verifychar(const unsigned char *s, int len)
 	if (len < l)
 		return -1;
 
+	if (l == 2 &&
+		s[0] == NONUTF8_INVALID_BYTE0 &&
+		s[1] == NONUTF8_INVALID_BYTE1)
+		return -1;
+
 	while (--l > 0)
 	{
 		if (*++s == '\0')
@@ -1596,6 +1623,11 @@ pg_gbk_verifychar(const unsigned char *s, int len)
 	if (len < l)
 		return -1;
 
+	if (l == 2 &&
+		s[0] == NONUTF8_INVALID_BYTE0 &&
+		s[1] == NONUTF8_INVALID_BYTE1)
+		return -1;
+
 	while (--l > 0)
 	{
 		if (*++s == '\0')
@@ -1645,6 +1677,11 @@ pg_uhc_verifychar(const unsigned char *s, int len)
 	if (len < l)
 		return -1;
 
+	if (l == 2 &&
+		s[0] == NONUTF8_INVALID_BYTE0 &&
+		s[1] == NONUTF8_INVALID_BYTE1)
+		return -1;
+
 	while (--l > 0)
 	{
 		if (*++s == '\0')
@@ -2089,6 +2126,12 @@ pg_utf8_islegal(const unsigned char *source, int length)
 }
 
 
+/*
+ * Fills the provided buffer with two bytes such that:
+ *   pg_encoding_mblen(dst) == 2 && pg_encoding_verifymbstr(dst) == 0
+ */
+
+
 /*
  *-------------------------------------------------------------------
  * encoding info table
@@ -2142,10 +2185,27 @@ const pg_wchar_tbl pg_wchar_table[] = {
 /*
  * Returns the byte length of a multibyte character.
  *
- * Caution: when dealing with text that is not certainly valid in the
- * specified encoding, the result may exceed the actual remaining
- * string length.  Callers that are not prepared to deal with that
- * should use pg_encoding_mblen_bounded() instead.
+ * Choose "mblen" functions based on the input string characteristics.
+ * pg_encoding_mblen() can be used when ANY of these conditions are met:
+ *
+ * - The input string is zero-terminated
+ *
+ * - The input string is known to be valid in the encoding (e.g., string
+ *   converted from database encoding)
+ *
+ * - The encoding is not GB18030 (e.g., when only database encodings are
+ *   passed to 'encoding' parameter)
+ *
+ * encoding==GB18030 requires examining up to two bytes to determine character
+ * length.  Therefore, callers satisfying none of those conditions must use
+ * pg_encoding_mblen_or_incomplete() instead, as access to mbstr[1] cannot be
+ * guaranteed to be within allocation bounds.
+ *
+ * When dealing with text that is not certainly valid in the specified
+ * encoding, the result may exceed the actual remaining string length.
+ * Callers that are not prepared to deal with that should use Min(remaining,
+ * pg_encoding_mblen_or_incomplete()).  For zero-terminated strings, that and
+ * pg_encoding_mblen_bounded() are interchangeable.
  */
 int
 pg_encoding_mblen(int encoding, const char *mbstr)
@@ -2156,8 +2216,28 @@ pg_encoding_mblen(int encoding, const char *mbstr)
 }
 
 /*
- * Returns the byte length of a multibyte character; but not more than
- * the distance to end of string.
+ * Returns the byte length of a multibyte character (possibly not
+ * zero-terminated), or INT_MAX if too few bytes remain to determine a length.
+ */
+int
+pg_encoding_mblen_or_incomplete(int encoding, const char *mbstr,
+								size_t remaining)
+{
+	/*
+	 * Define zero remaining as too few, even for single-byte encodings.
+	 * pg_gb18030_mblen() reads one or two bytes; single-byte encodings read
+	 * zero; others read one.
+	 */
+	if (remaining < 1 ||
+		(encoding == PG_GB18030 && IS_HIGHBIT_SET(*mbstr) && remaining < 2))
+		return INT_MAX;
+	return pg_encoding_mblen(encoding, mbstr);
+}
+
+/*
+ * Returns the byte length of a multibyte character; but not more than the
+ * distance to the terminating zero byte.  For input that might lack a
+ * terminating zero, use Min(remaining, pg_encoding_mblen_or_incomplete()).
  */
 
 
@@ -2188,5 +2268,11 @@ pg_encoding_max_length(int encoding)
 {
 	Assert(PG_VALID_ENCODING(encoding));
 
-	return pg_wchar_table[encoding].maxmblen;
+	/*
+	 * Check for the encoding despite the assert, due to some mingw versions
+	 * otherwise issuing bogus warnings.
+	 */
+	return PG_VALID_ENCODING(encoding) ?
+		pg_wchar_table[encoding].maxmblen :
+		pg_wchar_table[PG_SQL_ASCII].maxmblen;
 }

data/ext/pg_query/src_port_snprintf.c

@@ -3,7 +3,7 @@
  * - pg_vsnprintf
  * - dopr
  * - pg_snprintf
- * - strchrnul
+ * - pg_strchrnul
  * - dostr
  * - flushbuffer
  * - find_arguments
@@ -362,13 +362,22 @@ static void leading_pad(int zpad, int signvalue, int *padlen,
 static void trailing_pad(int padlen, PrintfTarget *target);
 
 /*
- * If strchrnul exists (it's a glibc-ism), it's a good bit faster than the
- * equivalent manual loop.  If it doesn't exist, provide a replacement.
+ * If strchrnul exists (it's a glibc-ism, but since adopted by some other
+ * platforms), it's a good bit faster than the equivalent manual loop.
+ * Use it if possible, and if it doesn't exist, use this replacement.
  *
  * Note: glibc declares this as returning "char *", but that would require
  * casting away const internally, so we don't follow that detail.
+ *
+ * Note: macOS has this too as of Sequoia 15.4, but it's hidden behind
+ * a deployment-target check that causes compile errors if the deployment
+ * target isn't high enough.  So !HAVE_DECL_STRCHRNUL may mean "yes it's
+ * declared, but it doesn't compile".  To avoid failing in that scenario,
+ * use a macro to avoid matching <string.h>'s name.
  */
-#ifndef HAVE_STRCHRNUL
+#if !HAVE_DECL_STRCHRNUL
+
+#define strchrnul pg_strchrnul
 
 static inline const char *
 strchrnul(const char *s, int c)
@@ -378,19 +387,7 @@ strchrnul(const char *s, int c)
 	return s;
 }
 
-#else
-
-/*
- * glibc's <string.h> declares strchrnul only if _GNU_SOURCE is defined.
- * While we typically use that on glibc platforms, configure will set
- * HAVE_STRCHRNUL whether it's used or not.  Fill in the missing declaration
- * so that this file will compile cleanly with or without _GNU_SOURCE.
- */
-#ifndef _GNU_SOURCE
-extern char *strchrnul(const char *s, int c);
-#endif
-
-#endif							/* HAVE_STRCHRNUL */
+#endif							/* !HAVE_DECL_STRCHRNUL */
 
 
 /*

data/lib/pg_query/deparse.rb

@@ -1,18 +1,39 @@
 module PgQuery
   class ParserResult
-    def deparse
-      PgQuery.deparse(@tree)
+    def deparse(opts: nil)
+      PgQuery.deparse(@tree, opts: opts)
     end
   end
 
+  class DeparseComment
+    attr_accessor :match_location, :newlines_before_comment, :newlines_after_comment, :str
+  end
+
+  DeparseOpts = Struct.new(:pretty_print, :comments, :indent_size, :max_line_length,
+                           :trailing_newline, :commas_start_of_line, keyword_init: true)
+
   # Reconstruct all of the parsed queries into their original form
-  def self.deparse(tree)
-    if PgQuery::ParseResult.method(:encode).arity == 1
-      PgQuery.deparse_protobuf(PgQuery::ParseResult.encode(tree)).force_encoding('UTF-8')
-    elsif PgQuery::ParseResult.method(:encode).arity == -1
-      PgQuery.deparse_protobuf(PgQuery::ParseResult.encode(tree, recursion_limit: 1_000)).force_encoding('UTF-8')
+  def self.deparse(tree, opts: nil)
+    protobuf_encoded = if PgQuery::ParseResult.method(:encode).arity == 1
+                         PgQuery::ParseResult.encode(tree)
+                       elsif PgQuery::ParseResult.method(:encode).arity == -1
+                         PgQuery::ParseResult.encode(tree, recursion_limit: 1_000)
+                       else
+                         raise ArgumentError, 'Unsupported protobuf Ruby API'
+                       end
+
+    if opts
+      PgQuery.deparse_protobuf_opts(
+        protobuf_encoded,
+        opts.pretty_print,
+        opts.comments || [],
+        opts.indent_size || 0,
+        opts.max_line_length || 0,
+        opts.trailing_newline,
+        opts.commas_start_of_line
+      ).force_encoding('UTF-8')
     else
-      raise ArgumentError, 'Unsupported protobuf Ruby API'
+      PgQuery.deparse_protobuf(protobuf_encoded).force_encoding('UTF-8')
     end
   end
 

data/lib/pg_query/fingerprint.rb

@@ -61,7 +61,7 @@ module PgQuery
 
     def ignored_node_type?(node)
       [A_Const, Alias, ParamRef, SetToDefault, IntList, OidList].include?(node.class) ||
-        node.is_a?(TypeCast) && (node.arg.node == :a_const || node.arg.node == :param_ref)
+        (node.is_a?(TypeCast) && %i[a_const param_ref].include?(node.arg.node))
     end
 
     def node_protobuf_field_name_to_json_name(node_class, field)
@@ -112,12 +112,10 @@ module PgQuery
             fingerprint_value(val.gsub(/\d{2,}/, ''), hash, postgres_node_name, postgres_field_name, true)
             next
           end
-        when 'stmt_len'
-          next if node.is_a?(RawStmt)
-        when 'stmt_location'
+        when 'stmt_len', 'stmt_location'
           next if node.is_a?(RawStmt)
         when 'kind'
-          if node.is_a?(A_Expr) && (val == :AEXPR_OP_ANY || val == :AEXPR_IN)
+          if node.is_a?(A_Expr) && %i[AEXPR_OP_ANY AEXPR_IN].include?(val)
             fingerprint_value(:AEXPR_OP, hash, postgres_node_name, postgres_field_name, true)
             next
           end

data/lib/pg_query/param_refs.rb

@@ -7,7 +7,7 @@ module PgQuery
         case node
         when PgQuery::ParamRef
           # Ignore param refs inside type casts, as these are already handled
-          next if location[-3..-1] == %i[type_cast arg param_ref]
+          next if location[-3..] == %i[type_cast arg param_ref]
 
           results << { 'location' => node.location,
                        'length' => param_ref_length(node) }

data/lib/pg_query/parse.rb

@@ -24,9 +24,7 @@ module PgQuery
   end
 
   class ParserResult
-    attr_reader :query
-    attr_reader :tree
-    attr_reader :warnings
+    attr_reader :query, :tree, :warnings
 
     def initialize(query, tree, warnings = [])
       @query = query
@@ -108,6 +106,7 @@ module PgQuery
       statements = @tree.stmts.dup.to_a.map(&:stmt)
       from_clause_items = [] # types: select, dml, ddl
       subselect_items = []
+      call_items = [] # CALL fn()
 
       loop do
         statement = statements.shift
@@ -173,10 +172,12 @@ module PgQuery
           when :copy_stmt
             from_clause_items << { item: PgQuery::Node.new(range_var: statement.copy_stmt.relation), type: :dml } if statement.copy_stmt.relation
             statements << statement.copy_stmt.query
+          when :call_stmt
+            call_items << statement.call_stmt
           # The following statement types are DDL (changing table structure)
           when :alter_table_stmt
             case statement.alter_table_stmt.objtype
-            when :OBJECT_INDEX # Index # rubocop:disable Lint/EmptyWhen
+            when :OBJECT_INDEX # Index
               # ignore `ALTER INDEX index_name`
             else
               from_clause_items << { item: PgQuery::Node.new(range_var: statement.alter_table_stmt.relation), type: :ddl }
@@ -230,11 +231,11 @@ module PgQuery
           when :grant_stmt
             objects = statement.grant_stmt.objects
             case statement.grant_stmt.objtype
-            when :OBJECT_COLUMN # Column # rubocop:disable Lint/EmptyWhen
+            when :OBJECT_COLUMN # Column
               # FIXME
             when :OBJECT_TABLE # Table
               from_clause_items += objects.map { |o| { item: o, type: :ddl } }
-            when :OBJECT_SEQUENCE # Sequence # rubocop:disable Lint/EmptyWhen
+            when :OBJECT_SEQUENCE # Sequence
               # FIXME
             end
           when :lock_stmt
@@ -278,15 +279,20 @@ module PgQuery
             end
           when :bool_expr
             subselect_items.concat(next_item.bool_expr.args.to_ary)
+          when :boolean_test
+            subselect_items << next_item.boolean_test.arg
           when :coalesce_expr
             subselect_items.concat(next_item.coalesce_expr.args.to_ary)
           when :min_max_expr
             subselect_items.concat(next_item.min_max_expr.args.to_ary)
+          when :null_test
+            subselect_items << next_item.null_test.arg
           when :res_target
             subselect_items << next_item.res_target.val
           when :sub_link
             statements << next_item.sub_link.subselect
           when :func_call
+            # See also CALL below
             subselect_items.concat(next_item.func_call.args.to_ary)
             @functions << {
               function: next_item.func_call.funcname.map { |f| f.string.sval }.join('.'),
@@ -301,6 +307,17 @@ module PgQuery
           end
         end
 
+        # CALL fn()
+        next_item = call_items.shift
+        if next_item
+          # Treat as a sub-select. Note the difference in underscore in func_call versus the above.
+          subselect_items.concat(next_item.funccall.args.to_ary)
+          @functions << {
+            function: next_item.funccall.funcname.map { |f| f.string.sval }.join('.'),
+            type: :call
+          }
+        end
+
         next_item = from_clause_items.shift
         if next_item && next_item[:item]
           case next_item[:item].node
@@ -339,7 +356,7 @@ module PgQuery
       @cte_names.uniq!
     end
 
-    def statements_and_cte_names_for_with_clause(with_clause) # FIXME
+    def statements_and_cte_names_for_with_clause(with_clause)
       statements = []
       cte_names = []
 

data/lib/pg_query/parse_error.rb

@@ -1,6 +1,7 @@
 module PgQuery
   class ParseError < ArgumentError
     attr_reader :location
+
     def initialize(message, source_file, source_line, location)
       super("#{message} (#{source_file}:#{source_line})")
       @location = location