pg_query 2.1.0 → 2.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ede23ff0f5d9af4599a7732fe624b3e2fba355ebb54bb6b88df3ef3932d7692a
-  data.tar.gz: d11105ea7323b8908dde6b328908190858d150f12a13a109d943ae3bf8af475b
+  metadata.gz: fe9be3677078ec8a4990a53c9e0aa74c21f2a66672fe3a0990b23c00a6da34af
+  data.tar.gz: 31e3bbcdfac0cd27a1553ac0f0d33f7cbd7e36ce8c5080f29a314da67adcc76a
 SHA512:
-  metadata.gz: 37b3301df52b4743098ded9bbb69b05f4a360f7e1335386833ac619f4e5423f7b3c42a6506b3519c59ed3d0bb62e6e59ef04c8515fc9a3eb5daa95c53ab228e7
-  data.tar.gz: cd94026574593d6e78ac41e88c3378a6c46ea6d0869f8ddd9a200c367a969ff67695a780b0609518eafcff7a8dfaff015ce6408662eff7616cb866b46b52593a
+  metadata.gz: 7a8e0ef4b389446f952988903aa6b5772c6caabaea80d97e4f40affdf0830227285e60af0848a2786fcdab98232e96a5a75d3b5f73b24378b9ef67e9d24d9bf8
+  data.tar.gz: ffd90f7bdf9a155cb7a014eaf9c8bfbf06d0905e70bc5dadc739015c4d11695ead10ae1960e1bcee44e3d81d3c8a5b7d47b6705a32ef93daa8d909b328ee9602

data/CHANGELOG.md

@@ -4,6 +4,61 @@
 
 * ...
 
+## 2.1.4     2022-09-19
+
+* Truncate: Simplify VALUES(...) lists
+* Truncate: Correctly handle UPDATE and ON CONFLICT target lists
+* Support complex queries with deeply nested ASTs ([#238](https://github.com/pganalyze/pg_query/pull/238))
+* Find table references inside type casts
+* Find function calls referenced in expression indexes ([#249](https://github.com/pganalyze/pg_query/pull/249))
+* Drop `Init_pg_query` from exported symbol map ([#256](https://github.com/pganalyze/pg_query/pull/256))
+
+## 2.1.3     2022-01-28
+
+* Track tables in EXCEPT and INTERSECT queries ([#239](https://github.com/pganalyze/pg_query/pull/239))
+* Get filter_columns working with UNION/EXCEPT/INTERSECT ([#240](https://github.com/pganalyze/pg_query/pull/240))
+* Update google-protobuf to address CVE scanner complaints
+  - Note that none of the CVEs apply to pg_query, but this avoids unnecessary errors when
+    the google-protobuf dependency is pulled in
+
+
+## 2.1.2     2021-11-12
+
+* Find tables in using clause of delete statement ([#234](https://github.com/pganalyze/pg_query/pull/234))
+* Find tables in case statements ([#235](https://github.com/pganalyze/pg_query/pull/235))
+* Correctly find nested tables in a subselect in a join condition ([#233](https://github.com/pganalyze/pg_query/pull/233))
+* Mark Postgres methods as visibility hidden, to avoid bloating dynamic symbol table ([#232](https://github.com/pganalyze/pg_query/pull/232))
+  - This is required on ELF platforms (i.e. Linux, etc) to avoid including all global
+    symbols in the shared library's symbol table, bloating the size, and causing
+    potential conflicts with other C libraries using the same symbol names.
+
+
+## 2.1.1     2021-10-13
+
+* Update to libpg_query 13-2.1.0 ([#230](https://github.com/pganalyze/pg_query/pull/230))
+  - Normalize: add funcname error object
+  - Normalize: Match GROUP BY against target list and re-use param refs
+  - PL/pgSQL: Setup namespace items for parameters, support RECORD types
+    - This significantly improves parsing for PL/pgSQL functions, to the extent
+      that most functions should now parse successfully
+  - Normalize: Don't modify constants in TypeName typmods/arrayBounds fields
+    - This matches how pg_stat_statement behaves, and avoids causing parsing
+      errors on the normalized statement
+  - Don't fail builds on systems that have strchrnul support (FreeBSD)
+* Fix build on FreeBSD ([#222](https://github.com/pganalyze/pg_query/pull/222))
+* Add workaround for Ruby garbage collection bug ([#227](https://github.com/pganalyze/pg_query/pull/227))
+  - The Ruby interpreter has a bug in `String#concat` where the appended
+    array may be garbage collected prematurely because the compiler
+    optimized out a Ruby stack variable. We now call `to_ary` on the
+    Protobuf object to ensure the array lands on the Ruby stack so the
+    garbage collector sees it.
+  - The real fix in the interpreter is described in
+    https://bugs.ruby-lang.org/issues/18140#note-2, but most current Ruby
+    interpreters won't have this fix for some time.
+* Table/function extraction: Support subselects and LATERAL better ([#229](https://github.com/pganalyze/pg_query/pull/229))
+  - This reworks the parsing logic so we don't ignore certain kinds of
+    subselects.
+
 
 ## 2.1.0     2021-07-04
 

data/Rakefile

@@ -5,8 +5,8 @@ require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
 require 'open-uri'
 
-LIB_PG_QUERY_TAG = '13-2.0.6'.freeze
-LIB_PG_QUERY_SHA256SUM = '61f384ac949bd7404efe6bcc37a8a6fca79030e59c02659f108ee1db45e93414'.freeze
+LIB_PG_QUERY_TAG = '13-2.1.0'.freeze
+LIB_PG_QUERY_SHA256SUM = 'a01329ae5bac19b10b8ddf8012bd663a20f85f180d6d7b900c1a1ca8444d19a5'.freeze
 
 Rake::ExtensionTask.new 'pg_query' do |ext|
   ext.lib_dir = 'lib/pg_query'

data/ext/pg_query/extconf.rb

@@ -7,11 +7,17 @@ require 'pathname'
 
 $objs = Dir.glob(File.join(__dir__, '*.c')).map { |f| Pathname.new(f).sub_ext('.o').to_s }
 
-$CFLAGS << " -O3 -Wall -fno-strict-aliasing -fwrapv -fstack-protector -Wno-unused-function -Wno-unused-variable -g"
+$CFLAGS << " -fvisibility=hidden -O3 -Wall -fno-strict-aliasing -fwrapv -fstack-protector -Wno-unused-function -Wno-unused-variable -Wno-clobbered -Wno-sign-compare -Wno-discarded-qualifiers -g"
 
 $INCFLAGS = "-I#{File.join(__dir__, 'include')} " + $INCFLAGS
 
-SYMFILE = File.join(__dir__, 'pg_query_ruby.sym')
+SYMFILE =
+  if RUBY_PLATFORM =~ /freebsd/
+    File.join(__dir__, 'pg_query_ruby_freebsd.sym')
+  else
+    File.join(__dir__, 'pg_query_ruby.sym')
+  end
+
 if RUBY_PLATFORM =~ /darwin/
   $DLDFLAGS << " -Wl,-exported_symbols_list #{SYMFILE}" unless defined?(::Rubinius)
 else

data/ext/pg_query/include/pg_config.h

@@ -987,3 +987,6 @@
 #undef HAVE_EXECINFO_H
 #undef HAVE_BACKTRACE_SYMBOLS
 #undef HAVE__GET_CPUID
+#ifdef __FreeBSD__
+#define HAVE_STRCHRNUL
+#endif

data/ext/pg_query/pg_query_fingerprint.c

@@ -291,6 +291,21 @@ _fingerprintNode(FingerprintContext *ctx, const void *obj, const void *parent, c
 	}
 }
 
+uint64_t pg_query_fingerprint_node(const void *node)
+{
+	FingerprintContext ctx;
+	uint64 result;
+
+	_fingerprintInitContext(&ctx, NULL, false);
+	_fingerprintNode(&ctx, node, NULL, NULL, 0);
+
+	result = XXH3_64bits_digest(ctx.xxh_state);
+
+	_fingerprintFreeContext(&ctx);
+
+	return result;
+}
+
 PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens)
 {
 	MemoryContext ctx = NULL;

data/ext/pg_query/pg_query_fingerprint.h

@@ -3,6 +3,8 @@
 
 #include <stdbool.h>
 
-PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens);
+extern PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens);
+
+extern uint64_t pg_query_fingerprint_node(const void * node);
 
 #endif

data/ext/pg_query/pg_query_normalize.c

@@ -1,5 +1,6 @@
 #include "pg_query.h"
 #include "pg_query_internal.h"
+#include "pg_query_fingerprint.h"
 
 #include "parser/parser.h"
 #include "parser/scanner.h"
@@ -14,6 +15,7 @@ typedef struct pgssLocationLen
 {
 	int			location;		/* start offset in query text */
 	int			length;			/* length in bytes, or -1 to ignore */
+	int			param_id;		/* Param id to use - if negative prefix, need to abs(..) and add highest_extern_param_id */
 } pgssLocationLen;
 
 /*
@@ -30,14 +32,32 @@ typedef struct pgssConstLocations
 	/* Current number of valid entries in clocations array */
 	int			clocations_count;
 
+	/* highest Param id we have assigned, not yet taking into account external param refs */
+	int			highest_normalize_param_id;
+
 	/* highest Param id we've seen, in order to start normalization correctly */
 	int			highest_extern_param_id;
 
 	/* query text */
 	const char * query;
 	int			query_len;
+
+	/* optional recording of assigned or discovered param refs, only active if param_refs is not NULL */
+	int *param_refs;
+	int param_refs_buf_size;
+	int param_refs_count;
 } pgssConstLocations;
 
+/*
+ * Intermediate working state struct to remember param refs for individual target list elements
+ */
+typedef struct FpAndParamRefs
+{
+	uint64_t fp;
+	int* param_refs;
+	int param_refs_count;
+} FpAndParamRefs;
+
 /*
  * comp_location: comparator for qsorting pgssLocationLen structs by location
  */
@@ -230,7 +250,8 @@ generate_normalized_query(pgssConstLocations *jstate, int query_loc, int* query_
 	for (i = 0; i < jstate->clocations_count; i++)
 	{
 		int			off,		/* Offset from start for cur tok */
-					tok_len;	/* Length (in bytes) of that tok */
+					tok_len,	/* Length (in bytes) of that tok */
+					param_id;	/* Param ID to be assigned */
 
 		off = jstate->clocations[i].location;
 		/* Adjust recorded location if we're dealing with partial string */
@@ -250,8 +271,10 @@ generate_normalized_query(pgssConstLocations *jstate, int query_loc, int* query_
 		n_quer_loc += len_to_wrt;
 
 		/* And insert a param symbol in place of the constant token */
-		n_quer_loc += sprintf(norm_query + n_quer_loc, "$%d",
-							  i + 1 + jstate->highest_extern_param_id);
+		param_id = (jstate->clocations[i].param_id < 0) ?
+					jstate->highest_extern_param_id + abs(jstate->clocations[i].param_id) :
+					jstate->clocations[i].param_id;
+		n_quer_loc += sprintf(norm_query + n_quer_loc, "$%d", param_id);
 
 		quer_loc = off + tok_len;
 		last_off = off;
@@ -292,6 +315,18 @@ static void RecordConstLocation(pgssConstLocations *jstate, int location)
 		jstate->clocations[jstate->clocations_count].location = location;
 		/* initialize lengths to -1 to simplify fill_in_constant_lengths */
 		jstate->clocations[jstate->clocations_count].length = -1;
+		/* by default we assume that we need a new param ref */
+		jstate->clocations[jstate->clocations_count].param_id = - jstate->highest_normalize_param_id;
+		jstate->highest_normalize_param_id++;
+		/* record param ref number if requested */
+		if (jstate->param_refs != NULL) {
+			jstate->param_refs[jstate->param_refs_count] = jstate->clocations[jstate->clocations_count].param_id;
+			jstate->param_refs_count++;
+			if (jstate->param_refs_count >= jstate->param_refs_buf_size) {
+				jstate->param_refs_buf_size *= 2;
+				jstate->param_refs = (int *) repalloc(jstate->param_refs, jstate->param_refs_buf_size * sizeof(int));
+			}
+		}
 		jstate->clocations_count++;
 	}
 }
@@ -313,6 +348,15 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 				/* Track the highest ParamRef number */
 				if (((ParamRef *) node)->number > jstate->highest_extern_param_id)
 					jstate->highest_extern_param_id = castNode(ParamRef, node)->number;
+
+				if (jstate->param_refs != NULL) {
+					jstate->param_refs[jstate->param_refs_count] = ((ParamRef *) node)->number;
+					jstate->param_refs_count++;
+					if (jstate->param_refs_count >= jstate->param_refs_buf_size) {
+						jstate->param_refs_buf_size *= 2;
+						jstate->param_refs = (int *) repalloc(jstate->param_refs, jstate->param_refs_buf_size * sizeof(int));
+					}
+				}
 			}
 			break;
 		case T_DefElem:
@@ -343,39 +387,95 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 			return const_record_walker((Node *) ((AlterRoleStmt *) node)->options, jstate);
 		case T_DeclareCursorStmt:
 			return const_record_walker((Node *) ((DeclareCursorStmt *) node)->query, jstate);
+		case T_TypeName:
+			/* Don't normalize constants in typmods or arrayBounds */
+			return false;
 		case T_SelectStmt:
 			{
 				SelectStmt *stmt = (SelectStmt *) node;
 				ListCell *lc;
+				List *fp_and_param_refs_list = NIL;
 
 				if (const_record_walker((Node *) stmt->distinctClause, jstate))
 					return true;
 				if (const_record_walker((Node *) stmt->intoClause, jstate))
 					return true;
-				if (const_record_walker((Node *) stmt->targetList, jstate))
-					return true;
+				foreach(lc, stmt->targetList)
+				{
+					ResTarget *res_target = lfirst_node(ResTarget, lc);
+					FpAndParamRefs *fp_and_param_refs = palloc0(sizeof(FpAndParamRefs));
+
+					/* Save all param refs we encounter or assign */
+					jstate->param_refs = palloc0(1 * sizeof(int));
+					jstate->param_refs_buf_size = 1;
+					jstate->param_refs_count = 0;
+
+					/* Walk the element */
+					if (const_record_walker((Node *) res_target, jstate))
+						return true;
+
+					/* Remember fingerprint and param refs for later */
+					fp_and_param_refs->fp = pg_query_fingerprint_node(res_target->val);
+					fp_and_param_refs->param_refs = jstate->param_refs;
+					fp_and_param_refs->param_refs_count = jstate->param_refs_count;
+					fp_and_param_refs_list = lappend(fp_and_param_refs_list, fp_and_param_refs);
+
+					/* Reset for next element, or stop recording if this is the last element */
+					jstate->param_refs = NULL;
+					jstate->param_refs_buf_size = 0;
+					jstate->param_refs_count = 0;
+				}
 				if (const_record_walker((Node *) stmt->fromClause, jstate))
 					return true;
 				if (const_record_walker((Node *) stmt->whereClause, jstate))
 					return true;
 
-				// Instead of walking all of groupClause (like raw_expression_tree_walker does),
-				// only walk certain items.
+				/*
+				 * Instead of walking all of groupClause (like raw_expression_tree_walker does),
+				 * only walk certain items.
+				 */
 				foreach(lc, stmt->groupClause)
 				{
-					// Do not walk A_Const values that are simple integers, this avoids
-					// turning "GROUP BY 1" into "GROUP BY $n", which obscures an important
-					// semantic meaning. This matches how pg_stat_statements handles the
-					// GROUP BY clause (i.e. it doesn't touch these constants)
+					/*
+					 * Do not walk A_Const values that are simple integers, this avoids
+					 * turning "GROUP BY 1" into "GROUP BY $n", which obscures an important
+					 * semantic meaning. This matches how pg_stat_statements handles the
+					 * GROUP BY clause (i.e. it doesn't touch these constants)
+					 */
 					if (IsA(lfirst(lc), A_Const) && IsA(&castNode(A_Const, lfirst(lc))->val, Integer))
 						continue;
 
+					/*
+					 * Match up GROUP BY clauses against the target list, to assign the same
+					 * param refs as used in the target list - this ensures the query is valid,
+					 * instead of throwing a bogus "columns ... must appear in the GROUP BY
+					 * clause or be used in an aggregate function" error
+					 */
+					uint64_t fp = pg_query_fingerprint_node(lfirst(lc));
+					FpAndParamRefs *fppr = NULL;
+					ListCell *lc2;
+					foreach(lc2, fp_and_param_refs_list) {
+						if (fp == ((FpAndParamRefs *) lfirst(lc2))->fp) {
+							fppr = (FpAndParamRefs *) lfirst(lc2);
+							foreach_delete_current(fp_and_param_refs_list, lc2);
+							break;
+						}
+					}
+
+					int prev_cloc_count = jstate->clocations_count;
 					if (const_record_walker((Node *) lfirst(lc), jstate))
 						return true;
+
+					if (fppr != NULL && fppr->param_refs_count == jstate->clocations_count - prev_cloc_count) {
+						for (int i = prev_cloc_count; i < jstate->clocations_count; i++) {
+							jstate->clocations[i].param_id = fppr->param_refs[i - prev_cloc_count];
+						}
+						jstate->highest_normalize_param_id -= fppr->param_refs_count;
+					}
 				}
 				foreach(lc, stmt->sortClause)
 				{
-					// Similarly, don't turn "ORDER BY 1" into "ORDER BY $n"
+					/* Similarly, don't turn "ORDER BY 1" into "ORDER BY $n" */
 					if (IsA(lfirst(lc), SortBy) && IsA(castNode(SortBy, lfirst(lc))->node, A_Const) &&
 					    IsA(&castNode(A_Const, castNode(SortBy, lfirst(lc))->node)->val, Integer))
 						continue;
@@ -445,9 +545,13 @@ PgQueryNormalizeResult pg_query_normalize(const char* input)
 		jstate.clocations = (pgssLocationLen *)
 			palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen));
 		jstate.clocations_count = 0;
+		jstate.highest_normalize_param_id = 1;
 		jstate.highest_extern_param_id = 0;
 		jstate.query = input;
 		jstate.query_len = query_len;
+		jstate.param_refs = NULL;
+		jstate.param_refs_buf_size = 0;
+		jstate.param_refs_count = 0;
 
 		/* Walk tree and record const locations */
 		const_record_walker((Node *) tree, &jstate);
@@ -466,6 +570,8 @@ PgQueryNormalizeResult pg_query_normalize(const char* input)
 		error = malloc(sizeof(PgQueryError));
 		error->message   = strdup(error_data->message);
 		error->filename  = strdup(error_data->filename);
+		error->funcname  = strdup(error_data->funcname);
+		error->context   = NULL;
 		error->lineno    = error_data->lineno;
 		error->cursorpos = error_data->cursorpos;
 
@@ -484,6 +590,7 @@ void pg_query_free_normalize_result(PgQueryNormalizeResult result)
   if (result.error) {
     free(result.error->message);
     free(result.error->filename);
+    free(result.error->funcname);
     free(result.error);
   }
 

data/ext/pg_query/pg_query_parse_plpgsql.c

@@ -108,7 +108,7 @@ static PLpgSQL_function *compile_create_function_stmt(CreateFunctionStmt* stmt)
 		}
 	}
 
-	assert(proc_source);
+	assert(proc_source != NULL);
 
 	if (stmt->returnType != NULL) {
 		foreach(lc3, stmt->returnType->names)
@@ -179,6 +179,26 @@ static PLpgSQL_function *compile_create_function_stmt(CreateFunctionStmt* stmt)
 	plpgsql_DumpExecTree = false;
 	plpgsql_start_datums();
 
+	/* Setup parameter names */
+	foreach(lc, stmt->parameters)
+	{
+		FunctionParameter *param = lfirst_node(FunctionParameter, lc);
+		if (param->name != NULL)
+		{
+			char buf[32];
+			PLpgSQL_type *argdtype;
+			PLpgSQL_variable *argvariable;
+			PLpgSQL_nsitem_type argitemtype;
+			snprintf(buf, sizeof(buf), "$%d", foreach_current_index(lc) + 1);
+			argdtype = plpgsql_build_datatype(UNKNOWNOID, -1, InvalidOid, NULL);
+			argvariable = plpgsql_build_variable(param->name ? param->name : buf, 0, argdtype, false);
+			argitemtype = argvariable->dtype == PLPGSQL_DTYPE_VAR ? PLPGSQL_NSTYPE_VAR : PLPGSQL_NSTYPE_REC;
+			plpgsql_ns_additem(argitemtype, argvariable->dno, buf);
+			if (param->name != NULL)
+				plpgsql_ns_additem(argitemtype, argvariable->dno, param->name);
+		}
+	}
+
 	/* Set up as though in a function returning VOID */
 	function->fn_rettype = VOIDOID;
 	function->fn_retset = is_setof;

data/ext/pg_query/pg_query_ruby.c

@@ -14,7 +14,7 @@ VALUE pg_query_ruby_fingerprint(VALUE self, VALUE input);
 VALUE pg_query_ruby_scan(VALUE self, VALUE input);
 VALUE pg_query_ruby_hash_xxh3_64(VALUE self, VALUE input, VALUE seed);
 
-void Init_pg_query(void)
+__attribute__((visibility ("default"))) void Init_pg_query(void)
 {
 	VALUE cPgQuery;
 

data/ext/pg_query/pg_query_ruby_freebsd.sym

@@ -0,0 +1,2 @@
+_Init_pg_query
+Init_pg_query

data/ext/pg_query/src_pl_plpgsql_src_pl_gram.c

@@ -6147,7 +6147,7 @@ plpgsql_sql_error_callback(void *arg)
  * This is handled the same as in check_sql_expr(), and we likewise
  * expect that the given string is a copy from the source text.
  */
-static PLpgSQL_type * parse_datatype(const char *string, int location) { PLpgSQL_type *typ; typ = (PLpgSQL_type *) palloc0(sizeof(PLpgSQL_type)); typ->typname = pstrdup(string); typ->ttype = PLPGSQL_TTYPE_SCALAR; return typ; }
+static PLpgSQL_type * parse_datatype(const char *string, int location) { PLpgSQL_type *typ; typ = (PLpgSQL_type *) palloc0(sizeof(PLpgSQL_type)); typ->typname = pstrdup(string); typ->ttype = strcmp(string, "RECORD") == 0 ? PLPGSQL_TTYPE_REC : PLPGSQL_TTYPE_SCALAR; return typ; }
 
 
 /*

data/lib/pg_query/filter_columns.rb

@@ -42,14 +42,16 @@ module PgQuery
                   statements << item.common_table_expr.ctequery if item.node == :common_table_expr
                 end
               end
-            when :SETOP_UNION
-              statements << statement.select_stmt.larg if statement.select_stmt.larg
-              statements << statement.select_stmt.rarg if statement.select_stmt.rarg
+            when :SETOP_UNION, :SETOP_EXCEPT, :SETOP_INTERSECT
+              statements << PgQuery::Node.new(select_stmt: statement.select_stmt.larg) if statement.select_stmt.larg
+              statements << PgQuery::Node.new(select_stmt: statement.select_stmt.rarg) if statement.select_stmt.rarg
             end
           when :update_stmt
             condition_items << statement.update_stmt.where_clause if statement.update_stmt.where_clause
           when :delete_stmt
             condition_items << statement.delete_stmt.where_clause if statement.delete_stmt.where_clause
+          when :index_stmt
+            condition_items << statement.index_stmt.where_clause if statement.index_stmt.where_clause
           end
         end
 

data/lib/pg_query/parse.rb

@@ -3,7 +3,13 @@ module PgQuery
     result, stderr = parse_protobuf(query)
 
     begin
-      result = PgQuery::ParseResult.decode(result)
+      result = if PgQuery::ParseResult.method(:decode).arity == 1
+                 PgQuery::ParseResult.decode(result)
+               elsif PgQuery::ParseResult.method(:decode).arity == -1
+                 PgQuery::ParseResult.decode(result, recursion_limit: 1_000)
+               else
+                 raise ArgumentError, 'Unsupported protobuf Ruby API'
+               end
     rescue Google::Protobuf::ParseError => e
       raise PgQuery::ParseError.new(format('Failed to parse tree: %s', e.message), __FILE__, __LINE__, -1)
     end
@@ -89,6 +95,10 @@ module PgQuery
 
     protected
 
+    # Parses the query and finds table and function references
+    #
+    # Note we use ".to_ary" on arrays from the Protobuf library before
+    # passing them to concat, because of https://bugs.ruby-lang.org/issues/18140
     def load_objects! # rubocop:disable Metrics/CyclomaticComplexity
       @tables = [] # types: select, dml, ddl
       @cte_names = []
@@ -108,22 +118,18 @@ module PgQuery
           # The following statement types do not modify tables and are added to from_clause_items
           # (and subsequently @tables)
           when :select_stmt
-            subselect_items.concat(statement.select_stmt.target_list)
+            subselect_items.concat(statement.select_stmt.target_list.to_ary)
             subselect_items << statement.select_stmt.where_clause if statement.select_stmt.where_clause
             subselect_items.concat(statement.select_stmt.sort_clause.collect { |h| h.sort_by.node })
-            subselect_items.concat(statement.select_stmt.group_clause)
+            subselect_items.concat(statement.select_stmt.group_clause.to_ary)
             subselect_items << statement.select_stmt.having_clause if statement.select_stmt.having_clause
 
             case statement.select_stmt.op
             when :SETOP_NONE
               (statement.select_stmt.from_clause || []).each do |item|
-                if item.node == :range_subselect
-                  statements << item.range_subselect.subquery
-                else
-                  from_clause_items << { item: item, type: :select }
-                end
+                from_clause_items << { item: item, type: :select }
               end
-            when :SETOP_UNION
+            when :SETOP_UNION, :SETOP_EXCEPT, :SETOP_INTERSECT
               statements << PgQuery::Node.new(select_stmt: statement.select_stmt.larg) if statement.select_stmt.larg
               statements << PgQuery::Node.new(select_stmt: statement.select_stmt.rarg) if statement.select_stmt.rarg
             end
@@ -143,12 +149,18 @@ module PgQuery
               value.from_clause.each do |item|
                 from_clause_items << { item: item, type: :select }
               end
-              subselect_items.concat(statement.update_stmt.target_list)
+              subselect_items.concat(statement.update_stmt.target_list.to_ary)
             end
 
             subselect_items << statement.update_stmt.where_clause if statement.node == :update_stmt && statement.update_stmt.where_clause
             subselect_items << statement.delete_stmt.where_clause if statement.node == :delete_stmt && statement.delete_stmt.where_clause
 
+            if statement.node == :delete_stmt
+              statement.delete_stmt.using_clause.each do |using_clause|
+                from_clause_items << { item: using_clause, type: :select }
+              end
+            end
+
             if value.with_clause
               cte_statements, cte_names = statements_and_cte_names_for_with_clause(value.with_clause)
               @cte_names.concat(cte_names)
@@ -174,6 +186,11 @@ module PgQuery
             statements << statement.view_stmt.query
           when :index_stmt
             from_clause_items << { item: PgQuery::Node.new(range_var: statement.index_stmt.relation), type: :ddl }
+            statement.index_stmt.index_params.each do |p|
+              next if p.index_elem.expr.nil?
+              subselect_items << p.index_elem.expr
+            end
+            subselect_items << statement.index_stmt.where_clause if statement.index_stmt.where_clause
           when :create_trig_stmt
             from_clause_items << { item: PgQuery::Node.new(range_var: statement.create_trig_stmt.relation), type: :ddl }
           when :rule_stmt
@@ -236,6 +253,8 @@ module PgQuery
         next_item = subselect_items.shift
         if next_item
           case next_item.node
+          when :list
+            subselect_items += next_item.list.items.to_ary
           when :a_expr
             %w[lexpr rexpr].each do |side|
               elem = next_item.a_expr.public_send(side)
@@ -247,56 +266,61 @@ module PgQuery
               end
             end
           when :bool_expr
-            subselect_items.concat(next_item.bool_expr.args)
+            subselect_items.concat(next_item.bool_expr.args.to_ary)
           when :coalesce_expr
-            subselect_items.concat(next_item.coalesce_expr.args)
+            subselect_items.concat(next_item.coalesce_expr.args.to_ary)
           when :min_max_expr
-            subselect_items.concat(next_item.min_max_expr.args)
+            subselect_items.concat(next_item.min_max_expr.args.to_ary)
           when :res_target
             subselect_items << next_item.res_target.val
           when :sub_link
             statements << next_item.sub_link.subselect
           when :func_call
+            subselect_items.concat(next_item.func_call.args.to_ary)
             @functions << {
-              function: next_item.func_call.funcname[0].string.str,
+              function: next_item.func_call.funcname.map { |f| f.string.str }.join('.'),
               type: :call
             }
+          when :case_expr
+            subselect_items.concat(next_item.case_expr.args.map { |arg| arg.case_when.expr })
+            subselect_items.concat(next_item.case_expr.args.map { |arg| arg.case_when.result })
+            subselect_items << next_item.case_expr.defresult
+          when :type_cast
+            subselect_items << next_item.type_cast.arg
           end
         end
 
-        break if subselect_items.empty? && statements.empty?
-      end
-
-      loop do
         next_item = from_clause_items.shift
-        break unless next_item && next_item[:item]
-
-        case next_item[:item].node
-        when :join_expr
-          from_clause_items << { item: next_item[:item].join_expr.larg, type: next_item[:type] }
-          from_clause_items << { item: next_item[:item].join_expr.rarg, type: next_item[:type] }
-        when :row_expr
-          from_clause_items += next_item[:item].row_expr.args.map { |a| { item: a, type: next_item[:type] } }
-        when :range_var
-          rangevar = next_item[:item].range_var
-          next if rangevar.schemaname.empty? && @cte_names.include?(rangevar.relname)
-
-          table = [rangevar.schemaname, rangevar.relname].reject { |s| s.nil? || s.empty? }.join('.')
-          @tables << {
-            name: table,
-            type: next_item[:type],
-            location: rangevar.location,
-            schemaname: (rangevar.schemaname unless rangevar.schemaname.empty?),
-            relname: rangevar.relname,
-            inh: rangevar.inh
-          }
-          @aliases[rangevar.alias.aliasname] = table if rangevar.alias
-        when :range_subselect
-          from_clause_items << { item: next_item[:item].range_subselect.subquery, type: next_item[:type] }
-        when :select_stmt
-          from_clause = next_item[:item].select_stmt.from_clause
-          from_clause_items += from_clause.map { |r| { item: r, type: next_item[:type] } } if from_clause
+        if next_item && next_item[:item]
+          case next_item[:item].node
+          when :join_expr
+            from_clause_items << { item: next_item[:item].join_expr.larg, type: next_item[:type] }
+            from_clause_items << { item: next_item[:item].join_expr.rarg, type: next_item[:type] }
+            subselect_items << next_item[:item].join_expr.quals
+          when :row_expr
+            from_clause_items += next_item[:item].row_expr.args.map { |a| { item: a, type: next_item[:type] } }
+          when :range_var
+            rangevar = next_item[:item].range_var
+            next if rangevar.schemaname.empty? && @cte_names.include?(rangevar.relname)
+
+            table = [rangevar.schemaname, rangevar.relname].reject { |s| s.nil? || s.empty? }.join('.')
+            @tables << {
+              name: table,
+              type: next_item[:type],
+              location: rangevar.location,
+              schemaname: (rangevar.schemaname unless rangevar.schemaname.empty?),
+              relname: rangevar.relname,
+              inh: rangevar.inh
+            }
+            @aliases[rangevar.alias.aliasname] = table if rangevar.alias
+          when :range_subselect
+            statements << next_item[:item].range_subselect.subquery
+          when :range_function
+            subselect_items += next_item[:item].range_function.functions
+          end
         end
+
+        break if subselect_items.empty? && statements.empty? && from_clause_items.empty?
       end
 
       @tables.uniq!

data/lib/pg_query/truncate.rb

@@ -32,6 +32,12 @@ module PgQuery
             )
           when :where_clause
             node.where_clause = dummy_column_ref
+          when :values_lists
+            node.values_lists.replace(
+              [
+                PgQuery::Node.new(list: PgQuery::List.new(items: [dummy_column_ref]))
+              ]
+            )
           when :ctequery
             node.ctequery = PgQuery::Node.new(select_stmt: PgQuery::SelectStmt.new(where_clause: dummy_column_ref, op: :SETOP_NONE))
           when :cols
@@ -58,7 +64,11 @@ module PgQuery
         case k
         when :target_list
           next unless node.is_a?(PgQuery::SelectStmt) || node.is_a?(PgQuery::UpdateStmt) || node.is_a?(PgQuery::OnConflictClause)
-          length = PgQuery.deparse_stmt(PgQuery::SelectStmt.new(k => v.to_a, op: :SETOP_NONE)).size - 7 # 'SELECT '.size
+          length = if node.is_a?(PgQuery::SelectStmt)
+                     select_target_list_len(v)
+                   else # UpdateStmt / OnConflictClause
+                     update_target_list_len(v)
+                   end
           truncations << PossibleTruncation.new(location, :target_list, length, true)
         when :where_clause
           next unless node.is_a?(PgQuery::SelectStmt) || node.is_a?(PgQuery::UpdateStmt) || node.is_a?(PgQuery::DeleteStmt) ||
@@ -67,23 +77,59 @@ module PgQuery
 
           length = PgQuery.deparse_expr(v).size
           truncations << PossibleTruncation.new(location, :where_clause, length, false)
+        when :values_lists
+          length = select_values_lists_len(v)
+          truncations << PossibleTruncation.new(location, :values_lists, length, false)
         when :ctequery
           next unless node.is_a?(PgQuery::CommonTableExpr)
           length = PgQuery.deparse_stmt(v[v.node.to_s]).size
           truncations << PossibleTruncation.new(location, :ctequery, length, false)
         when :cols
           next unless node.is_a?(PgQuery::InsertStmt)
-          length = PgQuery.deparse_stmt(
-            PgQuery::InsertStmt.new(
-              relation: PgQuery::RangeVar.new(relname: 'x', inh: true),
-              cols: v.to_a
-            )
-          ).size - 31 # "INSERT INTO x () DEFAULT VALUES".size
+          length = cols_len(v)
           truncations << PossibleTruncation.new(location, :cols, length, true)
         end
       end
 
       truncations
     end
+
+    def select_target_list_len(target_list)
+      deparsed_len = PgQuery.deparse_stmt(
+        PgQuery::SelectStmt.new(
+          target_list: target_list.to_a, op: :SETOP_NONE
+        )
+      ).size
+      deparsed_len - 7 # 'SELECT '.size
+    end
+
+    def select_values_lists_len(values_lists)
+      deparsed_len = PgQuery.deparse_stmt(
+        PgQuery::SelectStmt.new(
+          values_lists: values_lists.to_a, op: :SETOP_NONE
+        )
+      ).size
+      deparsed_len - 7 # 'SELECT '.size
+    end
+
+    def update_target_list_len(target_list)
+      deparsed_len = PgQuery.deparse_stmt(
+        PgQuery::UpdateStmt.new(
+          target_list: target_list.to_a,
+          relation: PgQuery::RangeVar.new(relname: 'x', inh: true)
+        )
+      ).size
+      deparsed_len - 13 # 'UPDATE x SET '.size
+    end
+
+    def cols_len(cols)
+      deparsed_len = PgQuery.deparse_stmt(
+        PgQuery::InsertStmt.new(
+          relation: PgQuery::RangeVar.new(relname: 'x', inh: true),
+          cols: cols.to_a
+        )
+      ).size
+      deparsed_len - 31 # "INSERT INTO x () DEFAULT VALUES".size
+    end
   end
 end

data/lib/pg_query/version.rb

@@ -1,3 +1,3 @@
 module PgQuery
-  VERSION = '2.1.0'.freeze
+  VERSION = '2.1.4'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_query
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.4
 platform: ruby
 authors:
 - Lukas Fittl
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-04 00:00:00.000000000 Z
+date: 2022-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.17.1
+        version: 3.19.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.17.1
+        version: 3.19.2
 description: Parses SQL queries using a copy of the PostgreSQL server query parser
 email: lukas@fittl.com
 executables: []
@@ -484,6 +484,7 @@ files:
 - ext/pg_query/pg_query_readfuncs_protobuf.c
 - ext/pg_query/pg_query_ruby.c
 - ext/pg_query/pg_query_ruby.sym
+- ext/pg_query/pg_query_ruby_freebsd.sym
 - ext/pg_query/pg_query_scan.c
 - ext/pg_query/pg_query_split.c
 - ext/pg_query/protobuf-c.c
@@ -561,7 +562,7 @@ files:
 - lib/pg_query/treewalker.rb
 - lib/pg_query/truncate.rb
 - lib/pg_query/version.rb
-homepage: http://github.com/pganalyze/pg_query
+homepage: https://github.com/pganalyze/pg_query
 licenses:
 - BSD-3-Clause
 metadata: {}