pg_query 4.2.1 → 6.1.0

data/ext/pg_query/pg_query_fingerprint.c

@@ -1,5 +1,3 @@
-// Ensure we have asprintf's definition on glibc-based platforms to avoid compiler warnings
-#define _GNU_SOURCE
 #include <stdio.h>
 
 #include "pg_query.h"
@@ -326,7 +324,7 @@ uint64_t pg_query_fingerprint_node(const void *node)
 	return result;
 }
 
-PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens)
+PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, int parser_options, bool printTokens)
 {
 	MemoryContext ctx = NULL;
 	PgQueryInternalParsetreeAndError parsetree_and_error;
@@ -334,7 +332,7 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 
 	ctx = pg_query_enter_memory_context();
 
-	parsetree_and_error = pg_query_raw_parse(input);
+	parsetree_and_error = pg_query_raw_parse(input, parser_options);
 
 	// These are all malloc-ed and will survive exiting the memory context, the caller is responsible to free them now
 	result.stderr_buffer = parsetree_and_error.stderr_buffer;
@@ -369,12 +367,13 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 		_fingerprintFreeContext(&ctx);
 
 		XXH64_canonicalFromHash(&chash, result.fingerprint);
-		int err = asprintf(&result.fingerprint_str, "%02x%02x%02x%02x%02x%02x%02x%02x",
+		result.fingerprint_str = malloc(17 * sizeof(char));
+		int n = snprintf(result.fingerprint_str, 17, "%02x%02x%02x%02x%02x%02x%02x%02x",
 						   chash.digest[0], chash.digest[1], chash.digest[2], chash.digest[3],
 						   chash.digest[4], chash.digest[5], chash.digest[6], chash.digest[7]);
-		if (err == -1) {
+		if (n < 0 || n >= 17) {
 			PgQueryError* error = malloc(sizeof(PgQueryError));
-			error->message = strdup("Failed to output fingerprint string due to asprintf failure");
+			error->message = strdup("Failed to output fingerprint string due to snprintf failure");
 			result.error = error;
 		}
 	}
@@ -386,7 +385,12 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 
 PgQueryFingerprintResult pg_query_fingerprint(const char* input)
 {
-	return pg_query_fingerprint_with_opts(input, false);
+	return pg_query_fingerprint_with_opts(input, PG_QUERY_PARSE_DEFAULT, false);
+}
+
+PgQueryFingerprintResult pg_query_fingerprint_opts(const char* input, int parser_options)
+{
+	return pg_query_fingerprint_with_opts(input, parser_options, false);
 }
 
 void pg_query_free_fingerprint_result(PgQueryFingerprintResult result)

data/ext/pg_query/pg_query_fingerprint.h

@@ -3,7 +3,7 @@
 
 #include <stdbool.h>
 
-extern PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens);
+extern PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, int parser_options, bool printTokens);
 
 extern uint64_t pg_query_fingerprint_node(const void * node);
 

data/ext/pg_query/pg_query_internal.h

@@ -14,7 +14,7 @@ typedef struct {
   PgQueryError* error;
 } PgQueryInternalParsetreeAndError;
 
-PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input);
+PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input, int parser_options);
 
 void pg_query_free_error(PgQueryError *error);
 

data/ext/pg_query/pg_query_json_plpgsql.c

@@ -626,6 +626,7 @@ dump_expr(StringInfo out, PLpgSQL_expr *node)
 	WRITE_NODE_TYPE("PLpgSQL_expr");
 
 	WRITE_STRING_FIELD(query, query, query);
+	WRITE_ENUM_FIELD(parseMode, parseMode, parseMode);
 }
 
 static void

data/ext/pg_query/pg_query_normalize.c

@@ -48,6 +48,9 @@ typedef struct pgssConstLocations
 	int *param_refs;
 	int param_refs_buf_size;
 	int param_refs_count;
+
+	/* Should only utility statements be normalized? Set by pg_query_normalize_utility */
+	bool normalize_utility_only;
 } pgssConstLocations;
 
 /*
@@ -398,8 +401,10 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 		case T_RawStmt:
 			return const_record_walker((Node *) ((RawStmt *) node)->stmt, jstate);
 		case T_VariableSetStmt:
+			if (jstate->normalize_utility_only) return false;
 			return const_record_walker((Node *) ((VariableSetStmt *) node)->args, jstate);
 		case T_CopyStmt:
+			if (jstate->normalize_utility_only) return false;
 			return const_record_walker((Node *) ((CopyStmt *) node)->query, jstate);
 		case T_ExplainStmt:
 			return const_record_walker((Node *) ((ExplainStmt *) node)->query, jstate);
@@ -408,16 +413,19 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 		case T_AlterRoleStmt:
 			return const_record_walker((Node *) ((AlterRoleStmt *) node)->options, jstate);
 		case T_DeclareCursorStmt:
+			if (jstate->normalize_utility_only) return false;
 			return const_record_walker((Node *) ((DeclareCursorStmt *) node)->query, jstate);
 		case T_CreateFunctionStmt:
+			if (jstate->normalize_utility_only) return false;
 			return const_record_walker((Node *) ((CreateFunctionStmt *) node)->options, jstate);
 		case T_DoStmt:
+			if (jstate->normalize_utility_only) return false;
 			return const_record_walker((Node *) ((DoStmt *) node)->args, jstate);
 		case T_CreateSubscriptionStmt:
 			record_matching_string(jstate, ((CreateSubscriptionStmt *) node)->conninfo);
 			break;
 		case T_AlterSubscriptionStmt:
-			record_matching_string(jstate, ((CreateSubscriptionStmt *) node)->conninfo);
+			record_matching_string(jstate, ((AlterSubscriptionStmt *) node)->conninfo);
 			break;
 		case T_CreateUserMappingStmt:
 			return const_record_walker((Node *) ((CreateUserMappingStmt *) node)->options, jstate);
@@ -428,6 +436,7 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 			return false;
 		case T_SelectStmt:
 			{
+				if (jstate->normalize_utility_only) return false;
 				SelectStmt *stmt = (SelectStmt *) node;
 				ListCell *lc;
 				List *fp_and_param_refs_list = NIL;
@@ -540,6 +549,26 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 
 				return false;
 			}
+		case T_MergeStmt:
+			{
+				if (jstate->normalize_utility_only) return false;
+				return raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
+			}
+		case T_InsertStmt:
+			{
+				if (jstate->normalize_utility_only) return false;
+				return raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
+			}
+		case T_UpdateStmt:
+			{
+				if (jstate->normalize_utility_only) return false;
+				return raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
+			}
+		case T_DeleteStmt:
+			{
+				if (jstate->normalize_utility_only) return false;
+				return raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
+			}
 		default:
 			{
 				PG_TRY();
@@ -558,7 +587,7 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 	return false;
 }
 
-PgQueryNormalizeResult pg_query_normalize(const char* input)
+PgQueryNormalizeResult pg_query_normalize_ext(const char* input, bool normalize_utility_only)
 {
 	MemoryContext ctx = NULL;
 	PgQueryNormalizeResult result = {0};
@@ -588,6 +617,7 @@ PgQueryNormalizeResult pg_query_normalize(const char* input)
 		jstate.param_refs = NULL;
 		jstate.param_refs_buf_size = 0;
 		jstate.param_refs_count = 0;
+		jstate.normalize_utility_only = normalize_utility_only;
 
 		/* Walk tree and record const locations */
 		const_record_walker((Node *) tree, &jstate);
@@ -621,6 +651,17 @@ PgQueryNormalizeResult pg_query_normalize(const char* input)
 	return result;
 }
 
+PgQueryNormalizeResult pg_query_normalize(const char* input)
+{
+	return pg_query_normalize_ext(input, false);
+}
+
+
+PgQueryNormalizeResult pg_query_normalize_utility(const char* input)
+{
+	return pg_query_normalize_ext(input, true);
+}
+
 void pg_query_free_normalize_result(PgQueryNormalizeResult result)
 {
   if (result.error) {

data/ext/pg_query/pg_query_outfuncs_json.c

@@ -36,6 +36,12 @@
 		appendStringInfo(out, "\"" CppAsString(outname_json) "\":%u,", node->fldname); \
 	}
 
+/* Write an unsigned integer field */
+#define WRITE_UINT64_FIELD(outname, outname_json, fldname) \
+	if (node->fldname != 0) { \
+		appendStringInfo(out, "\"" CppAsString(outname_json) "\":" UINT64_FORMAT ",", node->fldname); \
+	}
+
 /* Write a long-integer field */
 #define WRITE_LONG_FIELD(outname, outname_json, fldname) \
 	if (node->fldname != 0) { \
@@ -197,7 +203,9 @@ _outOidList(StringInfo out, const List *node)
 static void
 _outInteger(StringInfo out, const Integer *node)
 {
-	if (node->ival > 0)
+	/* Don't output anything if the value is the default (0), to match
+	 * protobuf's behavior. */
+	if (node->ival != 0)
 		appendStringInfo(out, "\"ival\":%d", node->ival);
 }
 

data/ext/pg_query/pg_query_outfuncs_protobuf.c

@@ -23,6 +23,7 @@
 
 #define WRITE_INT_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
 #define WRITE_UINT_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
+#define WRITE_UINT64_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
 #define WRITE_LONG_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
 #define WRITE_FLOAT_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
 #define WRITE_BOOL_FIELD(outname, outname_json, fldname) out->outname = node->fldname;
@@ -57,11 +58,11 @@
 #define WRITE_BITMAPSET_FIELD(outname, outname_json, fldname) \
 	if (!bms_is_empty(node->fldname)) \
 	{ \
-		int x = 0; \
+		int x = -1; \
 		int i = 0; \
 		out->n_##outname = bms_num_members(node->fldname); \
 		out->outname = palloc(sizeof(PgQuery__Node*) * out->n_##outname); \
-		while ((x = bms_first_member(node->fldname)) >= 0) \
+		while ((x = bms_next_member(node->fldname, x)) >= 0) \
 			out->outname[i++] = x; \
     }
 

data/ext/pg_query/pg_query_parse.c

@@ -9,7 +9,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 
-PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input)
+PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input, int parser_options)
 {
 	PgQueryInternalParsetreeAndError result = {0};
 	MemoryContext parse_context = CurrentMemoryContext;
@@ -42,7 +42,39 @@ PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input)
 
 	PG_TRY();
 	{
-		result.tree = raw_parser(input, RAW_PARSE_DEFAULT);
+		RawParseMode rawParseMode = RAW_PARSE_DEFAULT;
+		switch (parser_options & PG_QUERY_PARSE_MODE_BITMASK)
+		{
+			case PG_QUERY_PARSE_TYPE_NAME:
+				rawParseMode = RAW_PARSE_TYPE_NAME;
+				break;
+			case PG_QUERY_PARSE_PLPGSQL_EXPR:
+				rawParseMode = RAW_PARSE_PLPGSQL_EXPR;
+				break;
+			case PG_QUERY_PARSE_PLPGSQL_ASSIGN1:
+				rawParseMode = RAW_PARSE_PLPGSQL_ASSIGN1;
+				break;
+			case PG_QUERY_PARSE_PLPGSQL_ASSIGN2:
+				rawParseMode = RAW_PARSE_PLPGSQL_ASSIGN2;
+				break;
+			case PG_QUERY_PARSE_PLPGSQL_ASSIGN3:
+				rawParseMode = RAW_PARSE_PLPGSQL_ASSIGN3;
+				break;
+		}
+
+		if ((parser_options & PG_QUERY_DISABLE_BACKSLASH_QUOTE) == PG_QUERY_DISABLE_BACKSLASH_QUOTE) {
+			backslash_quote = BACKSLASH_QUOTE_OFF;
+		} else {
+			backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING;
+		}
+		standard_conforming_strings = !((parser_options & PG_QUERY_DISABLE_STANDARD_CONFORMING_STRINGS) == PG_QUERY_DISABLE_STANDARD_CONFORMING_STRINGS);
+		escape_string_warning = !((parser_options & PG_QUERY_DISABLE_ESCAPE_STRING_WARNING) == PG_QUERY_DISABLE_ESCAPE_STRING_WARNING);
+
+		result.tree = raw_parser(input, rawParseMode);
+
+		backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING;
+		standard_conforming_strings = true;
+		escape_string_warning = true;
 
 #ifndef DEBUG
 		// Save stderr for result
@@ -84,6 +116,11 @@ PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input)
 }
 
 PgQueryParseResult pg_query_parse(const char* input)
+{
+	return pg_query_parse_opts(input, PG_QUERY_PARSE_DEFAULT);
+}
+
+PgQueryParseResult pg_query_parse_opts(const char* input, int parser_options)
 {
 	MemoryContext ctx = NULL;
 	PgQueryInternalParsetreeAndError parsetree_and_error;
@@ -92,7 +129,7 @@ PgQueryParseResult pg_query_parse(const char* input)
 
 	ctx = pg_query_enter_memory_context();
 
-	parsetree_and_error = pg_query_raw_parse(input);
+	parsetree_and_error = pg_query_raw_parse(input, parser_options);
 
 	// These are all malloc-ed and will survive exiting the memory context, the caller is responsible to free them now
 	result.stderr_buffer = parsetree_and_error.stderr_buffer;
@@ -108,14 +145,19 @@ PgQueryParseResult pg_query_parse(const char* input)
 }
 
 PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input)
+{
+	return pg_query_parse_protobuf_opts(input, PG_QUERY_PARSE_DEFAULT);
+}
+
+PgQueryProtobufParseResult pg_query_parse_protobuf_opts(const char* input, int parser_options)
 {
 	MemoryContext ctx = NULL;
 	PgQueryInternalParsetreeAndError parsetree_and_error;
-	PgQueryProtobufParseResult result = {};
+	PgQueryProtobufParseResult result = {0};
 
 	ctx = pg_query_enter_memory_context();
 
-	parsetree_and_error = pg_query_raw_parse(input);
+	parsetree_and_error = pg_query_raw_parse(input, parser_options);
 
 	// These are all malloc-ed and will survive exiting the memory context, the caller is responsible to free them now
 	result.stderr_buffer = parsetree_and_error.stderr_buffer;

data/ext/pg_query/pg_query_parse_plpgsql.c

@@ -1,4 +1,3 @@
-#define _GNU_SOURCE // Necessary to get asprintf (which is a GNU extension)
 #include <stdio.h>
 
 #include "pg_query.h"
@@ -220,20 +219,19 @@ static PLpgSQL_function *compile_create_function_stmt(CreateFunctionStmt* stmt)
 	foreach(lc, stmt->parameters)
 	{
 		FunctionParameter *param = lfirst_node(FunctionParameter, lc);
+		char buf[32];
+		PLpgSQL_type *argdtype;
+		PLpgSQL_variable *argvariable;
+		PLpgSQL_nsitem_type argitemtype;
+		snprintf(buf, sizeof(buf), "$%d", foreach_current_index(lc) + 1);
+		argdtype = plpgsql_build_datatype(UNKNOWNOID, -1, InvalidOid, param->argType);
+		argvariable = plpgsql_build_variable(param->name ? param->name : buf, 0, argdtype, false);
+		if (param->mode == FUNC_PARAM_OUT || param->mode == FUNC_PARAM_INOUT || param->mode == FUNC_PARAM_TABLE)
+			function->out_param_varno = argvariable->dno;
+		argitemtype = argvariable->dtype == PLPGSQL_DTYPE_VAR ? PLPGSQL_NSTYPE_VAR : PLPGSQL_NSTYPE_REC;
+		plpgsql_ns_additem(argitemtype, argvariable->dno, buf);
 		if (param->name != NULL)
-		{
-			char buf[32];
-			PLpgSQL_type *argdtype;
-			PLpgSQL_variable *argvariable;
-			PLpgSQL_nsitem_type argitemtype;
-			snprintf(buf, sizeof(buf), "$%d", foreach_current_index(lc) + 1);
-			argdtype = plpgsql_build_datatype(UNKNOWNOID, -1, InvalidOid, NULL);
-			argvariable = plpgsql_build_variable(param->name ? param->name : buf, 0, argdtype, false);
-			argitemtype = argvariable->dtype == PLPGSQL_DTYPE_VAR ? PLPGSQL_NSTYPE_VAR : PLPGSQL_NSTYPE_REC;
-			plpgsql_ns_additem(argitemtype, argvariable->dno, buf);
-			if (param->name != NULL)
-				plpgsql_ns_additem(argitemtype, argvariable->dno, param->name);
-		}
+			plpgsql_ns_additem(argitemtype, argvariable->dno, param->name);
 	}
 
 	/* Set up as though in a function returning VOID */
@@ -449,7 +447,7 @@ PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input)
 
 	ctx = pg_query_enter_memory_context();
 
-	parse_result = pg_query_raw_parse(input);
+	parse_result = pg_query_raw_parse(input, PG_QUERY_PARSE_DEFAULT);
 	result.error = parse_result.error;
 	if (result.error != NULL) {
 		pg_query_exit_memory_context(ctx);
@@ -486,14 +484,17 @@ PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input)
 		if (func_and_error.func != NULL) {
 			char *func_json;
 			char *new_out;
+			size_t new_out_len;
 
 			func_json = plpgsqlToJSON(func_and_error.func);
 			plpgsql_free_function_memory(func_and_error.func);
 
-			int err = asprintf(&new_out, "%s%s,\n", result.plpgsql_funcs, func_json);
-			if (err == -1) {
+			new_out_len = strlen(result.plpgsql_funcs) + strlen(func_json) + 3;
+			new_out = malloc(new_out_len);
+			int n = snprintf(new_out, new_out_len, "%s%s,\n", result.plpgsql_funcs, func_json);
+			if (n < 0 || n >= new_out_len) {
 				PgQueryError* error = malloc(sizeof(PgQueryError));
-				error->message = strdup("Failed to output PL/pgSQL functions due to asprintf failure");
+				error->message = strdup("Failed to output PL/pgSQL functions due to snprintf failure");
 				result.error = error;
 			} else {
 				free(result.plpgsql_funcs);

data/ext/pg_query/pg_query_readfuncs_protobuf.c

@@ -14,6 +14,7 @@
 
 #define READ_INT_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
 #define READ_UINT_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
+#define READ_UINT64_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
 #define READ_LONG_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
 #define READ_FLOAT_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
 #define READ_BOOL_FIELD(outname, outname_json, fldname) node->fldname = msg->outname;
@@ -49,9 +50,9 @@
 		node->fldname = _readNode(msg->outname); \
 	}
 
-#define READ_EXPR_PTR_FIELD(outname, outname_json, fldname) \
+#define READ_ABSTRACT_PTR_FIELD(outname, outname_json, fldname, fldtype) \
 	if (msg->outname != NULL) { \
-		node->fldname = (Expr *) _readNode(msg->outname); \
+		node->fldname = (fldtype) _readNode(msg->outname); \
 	}
 
 #define READ_VALUE_FIELD(outname, outname_json, fldname) \

data/ext/pg_query/pg_query_ruby.c

@@ -221,5 +221,10 @@ VALUE pg_query_ruby_hash_xxh3_64(VALUE self, VALUE input, VALUE seed)
 	Check_Type(input, T_STRING);
 	Check_Type(seed, T_FIXNUM);
 
+#ifdef HAVE_LONG_LONG
+	return ULL2NUM(XXH3_64bits_withSeed(StringValuePtr(input), RSTRING_LEN(input), NUM2ULONG(seed)));
+#else
 	return ULONG2NUM(XXH3_64bits_withSeed(StringValuePtr(input), RSTRING_LEN(input), NUM2ULONG(seed)));
+#endif
+	
 }

data/ext/pg_query/pg_query_scan.c

@@ -1,7 +1,7 @@
 #include "pg_query.h"
 #include "pg_query_internal.h"
 
-#include "parser/gramparse.h"
+#include "gramparse.h"
 #include "lib/stringinfo.h"
 
 #include "protobuf/pg_query.pb-c.h"
@@ -93,7 +93,7 @@ PgQueryScanResult pg_query_scan(const char* input)
       output_tokens[i] = malloc(sizeof(PgQuery__ScanToken));
       pg_query__scan_token__init(output_tokens[i]);
       output_tokens[i]->start = yylloc;
-      if (tok == SCONST || tok == BCONST || tok == XCONST || tok == IDENT || tok == C_COMMENT) {
+      if (tok == SCONST || tok == USCONST || tok == BCONST || tok == XCONST || tok == IDENT || tok == UIDENT || tok == C_COMMENT) {
         output_tokens[i]->end = yyextra.yyllocend;
       } else {
         output_tokens[i]->end = yylloc + ((struct yyguts_t*) yyscanner)->yyleng_r;

data/ext/pg_query/pg_query_split.c

@@ -1,7 +1,7 @@
 #include "pg_query.h"
 #include "pg_query_internal.h"
 
-#include "parser/gramparse.h"
+#include "gramparse.h"
 #include "lib/stringinfo.h"
 
 #include <unistd.h>
@@ -166,11 +166,11 @@ PgQuerySplitResult pg_query_split_with_parser(const char* input)
 {
 	MemoryContext ctx = NULL;
 	PgQueryInternalParsetreeAndError parsetree_and_error;
-	PgQuerySplitResult result = {};
+	PgQuerySplitResult result = {0};
 
 	ctx = pg_query_enter_memory_context();
 
-	parsetree_and_error = pg_query_raw_parse(input);
+	parsetree_and_error = pg_query_raw_parse(input, PG_QUERY_PARSE_DEFAULT);
 
 	// These are all malloc-ed and will survive exiting the memory context, the caller is responsible to free them now
 	result.stderr_buffer = parsetree_and_error.stderr_buffer;