pg_query 2.0.3 → 5.1.0

data/ext/pg_query/pg_query_fingerprint.c

@@ -1,5 +1,3 @@
-// Ensure we have asprintf's definition on glibc-based platforms to avoid compiler warnings
-#define _GNU_SOURCE
 #include <stdio.h>
 
 #include "pg_query.h"
@@ -17,6 +15,8 @@
 #include "nodes/parsenodes.h"
 #include "nodes/value.h"
 
+#include "common/hashfn.h"
+
 #include <unistd.h>
 #include <fcntl.h>
 
@@ -26,15 +26,41 @@ typedef struct FingerprintContext
 {
 	XXH3_state_t *xxh_state;
 
+	struct listsort_cache_hash *listsort_cache;
+
 	bool write_tokens;
 	dlist_head tokens;
 } FingerprintContext;
 
-typedef struct FingerprintListContext
+typedef struct FingerprintListsortItem
 {
 	XXH64_hash_t hash;
 	size_t list_pos;
-} FingerprintListContext;
+} FingerprintListsortItem;
+
+typedef struct FingerprintListsortItemCacheEntry
+{
+	/* List node this cache entry is for */
+	uintptr_t node;
+
+	/* Hashes of all list items -- this is expensive to calculate */
+	FingerprintListsortItem **listsort_items;
+	size_t listsort_items_size;
+
+	/* hash entry status */
+	char status;
+} FingerprintListsortItemCacheEntry;
+
+#define SH_PREFIX listsort_cache
+#define SH_ELEMENT_TYPE FingerprintListsortItemCacheEntry
+#define SH_KEY_TYPE uintptr_t
+#define SH_KEY node
+#define SH_HASH_KEY(tb, key) hash_bytes((const unsigned char *) &key, sizeof(uintptr_t))
+#define SH_EQUAL(tb, a, b) a == b
+#define SH_SCOPE static inline
+#define SH_DEFINE
+#define SH_DECLARE
+#include "lib/simplehash.h"
 
 typedef struct FingerprintToken
 {
@@ -43,7 +69,7 @@ typedef struct FingerprintToken
 } FingerprintToken;
 
 static void _fingerprintNode(FingerprintContext *ctx, const void *obj, const void *parent, char *parent_field_name, unsigned int depth);
-static void _fingerprintInitContext(FingerprintContext *ctx, bool write_tokens);
+static void _fingerprintInitContext(FingerprintContext *ctx, FingerprintContext *parent, bool write_tokens);
 static void _fingerprintFreeContext(FingerprintContext *ctx);
 
 #define PG_QUERY_FINGERPRINT_VERSION 3
@@ -65,41 +91,55 @@ _fingerprintString(FingerprintContext *ctx, const char *str)
 }
 
 static void
-_fingerprintInteger(FingerprintContext *ctx, const Value *node)
+_fingerprintInteger(FingerprintContext *ctx, const union ValUnion *value)
 {
-	if (node->val.ival != 0) {
+	if (value->ival.ival != 0) {
 		_fingerprintString(ctx, "Integer");
 		_fingerprintString(ctx, "ival");
 		char buffer[50];
-		sprintf(buffer, "%d", node->val.ival);
+		sprintf(buffer, "%d", value->ival.ival);
 		_fingerprintString(ctx, buffer);
 	}
 }
 
 static void
-_fingerprintFloat(FingerprintContext *ctx, const Value *node)
+_fingerprintFloat(FingerprintContext *ctx, const union ValUnion *value)
 {
-	if (node->val.str != NULL) {
+	if (value->fval.fval != NULL) {
+		// NB: We output `str` here intentionally, to match the output format from libpg_query 14
+		// and below. This results in stable fingerprints, despite the field name being changed in
+		// PG15 to `fval`.
 		_fingerprintString(ctx, "Float");
 		_fingerprintString(ctx, "str");
-		_fingerprintString(ctx, node->val.str);
+		_fingerprintString(ctx, value->fval.fval);
 	}
 }
 
 static void
-_fingerprintBitString(FingerprintContext *ctx, const Value *node)
+_fingerprintBoolean(FingerprintContext *ctx, const union ValUnion *value)
 {
-	if (node->val.str != NULL) {
+	_fingerprintString(ctx, "Boolean");
+	_fingerprintString(ctx, "boolval");
+	_fingerprintString(ctx, value->boolval.boolval ? "true" : "false");
+}
+
+static void
+_fingerprintBitString(FingerprintContext *ctx, const union ValUnion *value)
+{
+	if (value->bsval.bsval != NULL) {
+		// NB: We output `str` here intentionally, to match the output format from libpg_query 14
+		// and below. This results in stable fingerprints, despite the field name being changed in
+		// PG15 to `bsval`.
 		_fingerprintString(ctx, "BitString");
 		_fingerprintString(ctx, "str");
-		_fingerprintString(ctx, node->val.str);
+		_fingerprintString(ctx, value->bsval.bsval);
 	}
 }
 
-static int compareFingerprintListContext(const void *a, const void *b)
+static int compareFingerprintListsortItem(const void *a, const void *b)
 {
-	FingerprintListContext *ca = *(FingerprintListContext**) a;
-	FingerprintListContext *cb = *(FingerprintListContext**) b;
+	FingerprintListsortItem *ca = *(FingerprintListsortItem**) a;
+	FingerprintListsortItem *cb = *(FingerprintListsortItem**) b;
 	if (ca->hash > cb->hash)
 		return 1;
 	else if (ca->hash < cb->hash)
@@ -111,38 +151,69 @@ static void
 _fingerprintList(FingerprintContext *ctx, const List *node, const void *parent, char *field_name, unsigned int depth)
 {
 	if (field_name != NULL && (strcmp(field_name, "fromClause") == 0 || strcmp(field_name, "targetList") == 0 ||
-			strcmp(field_name, "cols") == 0 || strcmp(field_name, "rexpr") == 0 || strcmp(field_name, "valuesLists") == 0 ||
-			strcmp(field_name, "args") == 0)) {
+		strcmp(field_name, "cols") == 0 || strcmp(field_name, "rexpr") == 0 || strcmp(field_name, "valuesLists") == 0 ||
+		strcmp(field_name, "args") == 0))
+	{
+		/*
+		 * Check for cached values for the hashes of subnodes
+		 *
+		 * Note this cache is important so we avoid exponential runtime behavior,
+		 * which would be the case if we fingerprinted each node twice, which
+		 * then would also again have to fingerprint each of its subnodes twice,
+		 * etc., leading to deep nodes to be fingerprinted many many times over.
+		 *
+		 * We have seen real-world problems with this logic here without
+		 * a cache in place.
+		 */
+		FingerprintListsortItem** listsort_items = NULL;
+		size_t listsort_items_size = 0;
+		FingerprintListsortItemCacheEntry *entry = listsort_cache_lookup(ctx->listsort_cache, (uintptr_t) node);
+		if (entry != NULL)
+		{
+			listsort_items = entry->listsort_items;
+			listsort_items_size = entry->listsort_items_size;
+		}
+		else
+		{
+			listsort_items = palloc0(node->length * sizeof(FingerprintListsortItem*));
+			listsort_items_size = 0;
+			ListCell *lc;
+			bool found;
 
-		FingerprintListContext** listCtxArr = palloc0(node->length * sizeof(FingerprintListContext*));
-		size_t listCtxCount = 0;
-		const ListCell *lc;
+			foreach(lc, node)
+			{
+				FingerprintContext fctx;
+				FingerprintListsortItem* lctx = palloc0(sizeof(FingerprintListsortItem));
 
-		foreach(lc, node)
-		{
-			FingerprintContext subCtx;
-			FingerprintListContext* listCtx = palloc0(sizeof(FingerprintListContext));
+				_fingerprintInitContext(&fctx, ctx, false);
+				_fingerprintNode(&fctx, lfirst(lc), parent, field_name, depth + 1);
+				lctx->hash = XXH3_64bits_digest(fctx.xxh_state);
+				lctx->list_pos = listsort_items_size;
+				_fingerprintFreeContext(&fctx);
 
-			_fingerprintInitContext(&subCtx, false);
-			_fingerprintNode(&subCtx, lfirst(lc), parent, field_name, depth + 1);
-			listCtx->hash = XXH3_64bits_digest(subCtx.xxh_state);
-			listCtx->list_pos = listCtxCount;
-			_fingerprintFreeContext(&subCtx);
+				listsort_items[listsort_items_size] = lctx;
+				listsort_items_size += 1;
+			}
 
-			listCtxArr[listCtxCount] = listCtx;
-			listCtxCount += 1;
-		}
+			pg_qsort(listsort_items, listsort_items_size, sizeof(FingerprintListsortItem*), compareFingerprintListsortItem);
+
+			FingerprintListsortItemCacheEntry *entry = listsort_cache_insert(ctx->listsort_cache, (uintptr_t) node, &found);
+			Assert(!found);
 
-		pg_qsort(listCtxArr, listCtxCount, sizeof(FingerprintListContext*), compareFingerprintListContext);
+			entry->listsort_items = listsort_items;
+			entry->listsort_items_size = listsort_items_size;
+		}
 
-		for (size_t i = 0; i < listCtxCount; i++)
+		for (size_t i = 0; i < listsort_items_size; i++)
 		{
-			if (i > 0 && listCtxArr[i - 1]->hash == listCtxArr[i]->hash)
+			if (i > 0 && listsort_items[i - 1]->hash == listsort_items[i]->hash)
 				continue; // Ignore duplicates
 
-			_fingerprintNode(ctx, lfirst(list_nth_cell(node, listCtxArr[i]->list_pos)), parent, field_name, depth + 1);
+			_fingerprintNode(ctx, lfirst(list_nth_cell(node, listsort_items[i]->list_pos)), parent, field_name, depth + 1);
 		}
-	} else {
+	}
+	else
+	{
 		const ListCell *lc;
 
 		foreach(lc, node)
@@ -155,15 +226,28 @@ _fingerprintList(FingerprintContext *ctx, const List *node, const void *parent,
 }
 
 static void
-_fingerprintInitContext(FingerprintContext *ctx, bool write_tokens) {
+_fingerprintInitContext(FingerprintContext *ctx, FingerprintContext *parent, bool write_tokens)
+{
 	ctx->xxh_state = XXH3_createState();
 	if (ctx->xxh_state == NULL) abort();
 	if (XXH3_64bits_reset_withSeed(ctx->xxh_state, PG_QUERY_FINGERPRINT_VERSION) == XXH_ERROR) abort();
 
-	if (write_tokens) {
+	if (parent != NULL)
+	{
+		ctx->listsort_cache = parent->listsort_cache;
+	}
+	else
+	{
+		ctx->listsort_cache = listsort_cache_create(CurrentMemoryContext, 128, NULL);
+	}
+
+	if (write_tokens)
+	{
 		ctx->write_tokens = true;
 		dlist_init(&ctx->tokens);
-	} else {
+	}
+	else
+	{
 		ctx->write_tokens = false;
 	}
 }
@@ -200,10 +284,16 @@ _fingerprintNode(FingerprintContext *ctx, const void *obj, const void *parent, c
 		case T_Float:
 			_fingerprintFloat(ctx, obj);
 			break;
+		case T_Boolean:
+			_fingerprintBoolean(ctx, obj);
+			break;
 		case T_String:
+			// NB: We output `str` here intentionally, to match the output format from libpg_query
+			// 14 and below. This results in stable fingerprints, despite the field name being
+			// changed in PG15 to `sval`.
 			_fingerprintString(ctx, "String");
 			_fingerprintString(ctx, "str");
-			_fingerprintString(ctx, ((Value*) obj)->val.str);
+			_fingerprintString(ctx, ((union ValUnion*) obj)->sval.sval);
 			break;
 		case T_BitString:
 			_fingerprintBitString(ctx, obj);
@@ -219,7 +309,22 @@ _fingerprintNode(FingerprintContext *ctx, const void *obj, const void *parent, c
 	}
 }
 
-PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens)
+uint64_t pg_query_fingerprint_node(const void *node)
+{
+	FingerprintContext ctx;
+	uint64 result;
+
+	_fingerprintInitContext(&ctx, NULL, false);
+	_fingerprintNode(&ctx, node, NULL, NULL, 0);
+
+	result = XXH3_64bits_digest(ctx.xxh_state);
+
+	_fingerprintFreeContext(&ctx);
+
+	return result;
+}
+
+PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, int parser_options, bool printTokens)
 {
 	MemoryContext ctx = NULL;
 	PgQueryInternalParsetreeAndError parsetree_and_error;
@@ -227,7 +332,7 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 
 	ctx = pg_query_enter_memory_context();
 
-	parsetree_and_error = pg_query_raw_parse(input);
+	parsetree_and_error = pg_query_raw_parse(input, parser_options);
 
 	// These are all malloc-ed and will survive exiting the memory context, the caller is responsible to free them now
 	result.stderr_buffer = parsetree_and_error.stderr_buffer;
@@ -237,7 +342,7 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 		FingerprintContext ctx;
 		XXH64_canonical_t chash;
 
-		_fingerprintInitContext(&ctx, printTokens);
+		_fingerprintInitContext(&ctx, NULL, printTokens);
 
 		if (parsetree_and_error.tree != NULL) {
 			_fingerprintNode(&ctx, parsetree_and_error.tree, NULL, NULL, 0);
@@ -262,12 +367,13 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 		_fingerprintFreeContext(&ctx);
 
 		XXH64_canonicalFromHash(&chash, result.fingerprint);
-		int err = asprintf(&result.fingerprint_str, "%02x%02x%02x%02x%02x%02x%02x%02x",
+		result.fingerprint_str = malloc(17 * sizeof(char));
+		int n = snprintf(result.fingerprint_str, 17, "%02x%02x%02x%02x%02x%02x%02x%02x",
 						   chash.digest[0], chash.digest[1], chash.digest[2], chash.digest[3],
 						   chash.digest[4], chash.digest[5], chash.digest[6], chash.digest[7]);
-		if (err == -1) {
+		if (n < 0 || n >= 17) {
 			PgQueryError* error = malloc(sizeof(PgQueryError));
-			error->message = strdup("Failed to output fingerprint string due to asprintf failure");
+			error->message = strdup("Failed to output fingerprint string due to snprintf failure");
 			result.error = error;
 		}
 	}
@@ -279,7 +385,12 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 
 PgQueryFingerprintResult pg_query_fingerprint(const char* input)
 {
-	return pg_query_fingerprint_with_opts(input, false);
+	return pg_query_fingerprint_with_opts(input, PG_QUERY_PARSE_DEFAULT, false);
+}
+
+PgQueryFingerprintResult pg_query_fingerprint_opts(const char* input, int parser_options)
+{
+	return pg_query_fingerprint_with_opts(input, parser_options, false);
 }
 
 void pg_query_free_fingerprint_result(PgQueryFingerprintResult result)
@@ -287,6 +398,7 @@ void pg_query_free_fingerprint_result(PgQueryFingerprintResult result)
 	if (result.error) {
 		free(result.error->message);
 		free(result.error->filename);
+		free(result.error->funcname);
 		free(result.error);
 	}
 

data/ext/pg_query/pg_query_fingerprint.h

@@ -3,6 +3,8 @@
 
 #include <stdbool.h>
 
-PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool printTokens);
+extern PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, int parser_options, bool printTokens);
+
+extern uint64_t pg_query_fingerprint_node(const void * node);
 
 #endif

data/ext/pg_query/pg_query_internal.h

@@ -14,7 +14,7 @@ typedef struct {
   PgQueryError* error;
 } PgQueryInternalParsetreeAndError;
 
-PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input);
+PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input, int parser_options);
 
 void pg_query_free_error(PgQueryError *error);
 

data/ext/pg_query/pg_query_json_plpgsql.c

@@ -96,7 +96,6 @@ static void dump_row(StringInfo out, PLpgSQL_row *stmt);
 static void dump_var(StringInfo out, PLpgSQL_var *stmt);
 static void dump_variable(StringInfo out, PLpgSQL_variable *stmt);
 static void dump_record_field(StringInfo out, PLpgSQL_recfield *node);
-static void dump_array_elem(StringInfo out, PLpgSQL_arrayelem *node);
 static void dump_stmt(StringInfo out, PLpgSQL_stmt *stmt);
 static void dump_block(StringInfo out, PLpgSQL_stmt_block *block);
 static void dump_exception_block(StringInfo out, PLpgSQL_exception_block *node);
@@ -117,6 +116,7 @@ static void dump_return_next(StringInfo out, PLpgSQL_stmt_return_next *stmt);
 static void dump_return_query(StringInfo out, PLpgSQL_stmt_return_query *stmt);
 static void dump_raise(StringInfo out, PLpgSQL_stmt_raise *stmt);
 static void dump_raise_option(StringInfo out, PLpgSQL_raise_option *node);
+static void dump_assert(StringInfo out, PLpgSQL_stmt_assert *stmt);
 static void dump_execsql(StringInfo out, PLpgSQL_stmt_execsql *stmt);
 static void dump_dynexecute(StringInfo out, PLpgSQL_stmt_dynexecute *stmt);
 static void dump_dynfors(StringInfo out, PLpgSQL_stmt_dynfors *stmt);
@@ -126,6 +126,9 @@ static void dump_open(StringInfo out, PLpgSQL_stmt_open *stmt);
 static void dump_fetch(StringInfo out, PLpgSQL_stmt_fetch *stmt);
 static void dump_close(StringInfo out, PLpgSQL_stmt_close *stmt);
 static void dump_perform(StringInfo out, PLpgSQL_stmt_perform *stmt);
+static void dump_call(StringInfo out, PLpgSQL_stmt_call *stmt);
+static void dump_commit(StringInfo out, PLpgSQL_stmt_commit *stmt);
+static void dump_rollback(StringInfo out, PLpgSQL_stmt_rollback *stmt);
 static void dump_expr(StringInfo out, PLpgSQL_expr *expr);
 static void dump_function(StringInfo out, PLpgSQL_function *func);
 static void dump_exception(StringInfo out, PLpgSQL_exception *node);
@@ -183,6 +186,9 @@ dump_stmt(StringInfo out, PLpgSQL_stmt *node)
 		case PLPGSQL_STMT_RAISE:
 			dump_raise(out, (PLpgSQL_stmt_raise *) node);
 			break;
+		case PLPGSQL_STMT_ASSERT:
+			dump_assert(out, (PLpgSQL_stmt_assert *) node);
+			break;
 		case PLPGSQL_STMT_EXECSQL:
 			dump_execsql(out, (PLpgSQL_stmt_execsql *) node);
 			break;
@@ -207,6 +213,15 @@ dump_stmt(StringInfo out, PLpgSQL_stmt *node)
 		case PLPGSQL_STMT_PERFORM:
 			dump_perform(out, (PLpgSQL_stmt_perform *) node);
 			break;
+		case PLPGSQL_STMT_CALL:
+			dump_call(out, (PLpgSQL_stmt_call *) node);
+			break;
+		case PLPGSQL_STMT_COMMIT:
+			dump_commit(out, (PLpgSQL_stmt_commit *) node);
+			break;
+		case PLPGSQL_STMT_ROLLBACK:
+			dump_rollback(out, (PLpgSQL_stmt_rollback *) node);
+			break;
 		default:
 			elog(ERROR, "unrecognized cmd_type: %d", node->cmd_type);
 			break;
@@ -444,6 +459,35 @@ dump_perform(StringInfo out, PLpgSQL_stmt_perform *node)
 	WRITE_EXPR_FIELD(expr);
 }
 
+static void
+dump_call(StringInfo out, PLpgSQL_stmt_call *node)
+{
+	WRITE_NODE_TYPE("PLpgSQL_stmt_call");
+
+	WRITE_INT_FIELD(lineno, lineno, lineno);
+	WRITE_EXPR_FIELD(expr);
+	WRITE_BOOL_FIELD(is_call, is_call, is_call);
+	WRITE_VARIABLE_FIELD(target);
+}
+
+static void
+dump_commit(StringInfo out, PLpgSQL_stmt_commit *node)
+{
+	WRITE_NODE_TYPE("PLpgSQL_stmt_commit");
+
+	WRITE_INT_FIELD(lineno, lineno, lineno);
+	WRITE_BOOL_FIELD(chain, chain, chain);
+}
+
+static void
+dump_rollback(StringInfo out, PLpgSQL_stmt_rollback *node)
+{
+	WRITE_NODE_TYPE("PLpgSQL_stmt_rollback");
+
+	WRITE_INT_FIELD(lineno, lineno, lineno);
+	WRITE_BOOL_FIELD(chain, chain, chain);
+}
+
 static void
 dump_exit(StringInfo out, PLpgSQL_stmt_exit *node)
 {
@@ -508,6 +552,16 @@ dump_raise_option(StringInfo out, PLpgSQL_raise_option *node)
 	WRITE_EXPR_FIELD(expr);
 }
 
+static void
+dump_assert(StringInfo out, PLpgSQL_stmt_assert *node)
+{
+	WRITE_NODE_TYPE("PLpgSQL_stmt_assert");
+
+	WRITE_INT_FIELD(lineno, lineno, lineno);
+	WRITE_EXPR_FIELD(cond);
+	WRITE_EXPR_FIELD(message);
+}
+
 static void
 dump_execsql(StringInfo out, PLpgSQL_stmt_execsql *node)
 {
@@ -572,6 +626,7 @@ dump_expr(StringInfo out, PLpgSQL_expr *node)
 	WRITE_NODE_TYPE("PLpgSQL_expr");
 
 	WRITE_STRING_FIELD(query, query, query);
+	WRITE_ENUM_FIELD(parseMode, parseMode, parseMode);
 }
 
 static void
@@ -605,9 +660,6 @@ dump_function(StringInfo out, PLpgSQL_function *node)
 			case PLPGSQL_DTYPE_RECFIELD:
 				dump_record_field(out, (PLpgSQL_recfield *) d);
 				break;
-			case PLPGSQL_DTYPE_ARRAYELEM:
-				dump_array_elem(out, (PLpgSQL_arrayelem *) d);
-				break;
 			default:
 				elog(WARNING, "could not dump unrecognized dtype: %d",
 					 (int) d->dtype);
@@ -712,14 +764,6 @@ dump_record_field(StringInfo out, PLpgSQL_recfield *node) {
 	WRITE_INT_FIELD(recparentno, recparentno, recparentno);
 }
 
-static void
-dump_array_elem(StringInfo out, PLpgSQL_arrayelem *node) {
-	WRITE_NODE_TYPE("PLpgSQL_arrayelem");
-
-	WRITE_EXPR_FIELD(subscript);
-	WRITE_INT_FIELD(arrayparentno, arrayparentno, arrayparentno);
-}
-
 char *
 plpgsqlToJSON(PLpgSQL_function *func)
 {