pg_query 2.0.3 → 2.1.0

data/ext/pg_query/pg_query_deparse.c

@@ -2247,6 +2247,12 @@ static void deparseRangeVar(StringInfo str, RangeVar *range_var, DeparseNodeCont
 	if (!range_var->inh && context != DEPARSE_NODE_CONTEXT_CREATE_TYPE && context != DEPARSE_NODE_CONTEXT_ALTER_TYPE)
 		appendStringInfoString(str, "ONLY ");
 
+	if (range_var->catalogname != NULL)
+	{
+		appendStringInfoString(str, quote_identifier(range_var->catalogname));
+		appendStringInfoChar(str, '.');
+	}
+
 	if (range_var->schemaname != NULL)
 	{
 		appendStringInfoString(str, quote_identifier(range_var->schemaname));

data/ext/pg_query/pg_query_fingerprint.c

@@ -17,6 +17,8 @@
 #include "nodes/parsenodes.h"
 #include "nodes/value.h"
 
+#include "common/hashfn.h"
+
 #include <unistd.h>
 #include <fcntl.h>
 
@@ -26,15 +28,41 @@ typedef struct FingerprintContext
 {
 	XXH3_state_t *xxh_state;
 
+	struct listsort_cache_hash *listsort_cache;
+
 	bool write_tokens;
 	dlist_head tokens;
 } FingerprintContext;
 
-typedef struct FingerprintListContext
+typedef struct FingerprintListsortItem
 {
 	XXH64_hash_t hash;
 	size_t list_pos;
-} FingerprintListContext;
+} FingerprintListsortItem;
+
+typedef struct FingerprintListsortItemCacheEntry
+{
+	/* List node this cache entry is for */
+	uintptr_t node;
+
+	/* Hashes of all list items -- this is expensive to calculate */
+	FingerprintListsortItem **listsort_items;
+	size_t listsort_items_size;
+
+	/* hash entry status */
+	char status;
+} FingerprintListsortItemCacheEntry;
+
+#define SH_PREFIX listsort_cache
+#define SH_ELEMENT_TYPE FingerprintListsortItemCacheEntry
+#define SH_KEY_TYPE uintptr_t
+#define SH_KEY node
+#define SH_HASH_KEY(tb, key) hash_bytes((const unsigned char *) &key, sizeof(uintptr_t))
+#define SH_EQUAL(tb, a, b) a == b
+#define SH_SCOPE static inline
+#define SH_DEFINE
+#define SH_DECLARE
+#include "lib/simplehash.h"
 
 typedef struct FingerprintToken
 {
@@ -43,7 +71,7 @@ typedef struct FingerprintToken
 } FingerprintToken;
 
 static void _fingerprintNode(FingerprintContext *ctx, const void *obj, const void *parent, char *parent_field_name, unsigned int depth);
-static void _fingerprintInitContext(FingerprintContext *ctx, bool write_tokens);
+static void _fingerprintInitContext(FingerprintContext *ctx, FingerprintContext *parent, bool write_tokens);
 static void _fingerprintFreeContext(FingerprintContext *ctx);
 
 #define PG_QUERY_FINGERPRINT_VERSION 3
@@ -96,10 +124,10 @@ _fingerprintBitString(FingerprintContext *ctx, const Value *node)
 	}
 }
 
-static int compareFingerprintListContext(const void *a, const void *b)
+static int compareFingerprintListsortItem(const void *a, const void *b)
 {
-	FingerprintListContext *ca = *(FingerprintListContext**) a;
-	FingerprintListContext *cb = *(FingerprintListContext**) b;
+	FingerprintListsortItem *ca = *(FingerprintListsortItem**) a;
+	FingerprintListsortItem *cb = *(FingerprintListsortItem**) b;
 	if (ca->hash > cb->hash)
 		return 1;
 	else if (ca->hash < cb->hash)
@@ -111,38 +139,69 @@ static void
 _fingerprintList(FingerprintContext *ctx, const List *node, const void *parent, char *field_name, unsigned int depth)
 {
 	if (field_name != NULL && (strcmp(field_name, "fromClause") == 0 || strcmp(field_name, "targetList") == 0 ||
-			strcmp(field_name, "cols") == 0 || strcmp(field_name, "rexpr") == 0 || strcmp(field_name, "valuesLists") == 0 ||
-			strcmp(field_name, "args") == 0)) {
+		strcmp(field_name, "cols") == 0 || strcmp(field_name, "rexpr") == 0 || strcmp(field_name, "valuesLists") == 0 ||
+		strcmp(field_name, "args") == 0))
+	{
+		/*
+		 * Check for cached values for the hashes of subnodes
+		 *
+		 * Note this cache is important so we avoid exponential runtime behavior,
+		 * which would be the case if we fingerprinted each node twice, which
+		 * then would also again have to fingerprint each of its subnodes twice,
+		 * etc., leading to deep nodes to be fingerprinted many many times over.
+		 *
+		 * We have seen real-world problems with this logic here without
+		 * a cache in place.
+		 */
+		FingerprintListsortItem** listsort_items = NULL;
+		size_t listsort_items_size = 0;
+		FingerprintListsortItemCacheEntry *entry = listsort_cache_lookup(ctx->listsort_cache, (uintptr_t) node);
+		if (entry != NULL)
+		{
+			listsort_items = entry->listsort_items;
+			listsort_items_size = entry->listsort_items_size;
+		}
+		else
+		{
+			listsort_items = palloc0(node->length * sizeof(FingerprintListsortItem*));
+			listsort_items_size = 0;
+			ListCell *lc;
+			bool found;
 
-		FingerprintListContext** listCtxArr = palloc0(node->length * sizeof(FingerprintListContext*));
-		size_t listCtxCount = 0;
-		const ListCell *lc;
+			foreach(lc, node)
+			{
+				FingerprintContext fctx;
+				FingerprintListsortItem* lctx = palloc0(sizeof(FingerprintListsortItem));
 
-		foreach(lc, node)
-		{
-			FingerprintContext subCtx;
-			FingerprintListContext* listCtx = palloc0(sizeof(FingerprintListContext));
+				_fingerprintInitContext(&fctx, ctx, false);
+				_fingerprintNode(&fctx, lfirst(lc), parent, field_name, depth + 1);
+				lctx->hash = XXH3_64bits_digest(fctx.xxh_state);
+				lctx->list_pos = listsort_items_size;
+				_fingerprintFreeContext(&fctx);
 
-			_fingerprintInitContext(&subCtx, false);
-			_fingerprintNode(&subCtx, lfirst(lc), parent, field_name, depth + 1);
-			listCtx->hash = XXH3_64bits_digest(subCtx.xxh_state);
-			listCtx->list_pos = listCtxCount;
-			_fingerprintFreeContext(&subCtx);
+				listsort_items[listsort_items_size] = lctx;
+				listsort_items_size += 1;
+			}
 
-			listCtxArr[listCtxCount] = listCtx;
-			listCtxCount += 1;
-		}
+			pg_qsort(listsort_items, listsort_items_size, sizeof(FingerprintListsortItem*), compareFingerprintListsortItem);
 
-		pg_qsort(listCtxArr, listCtxCount, sizeof(FingerprintListContext*), compareFingerprintListContext);
+			FingerprintListsortItemCacheEntry *entry = listsort_cache_insert(ctx->listsort_cache, (uintptr_t) node, &found);
+			Assert(!found);
 
-		for (size_t i = 0; i < listCtxCount; i++)
+			entry->listsort_items = listsort_items;
+			entry->listsort_items_size = listsort_items_size;
+		}
+
+		for (size_t i = 0; i < listsort_items_size; i++)
 		{
-			if (i > 0 && listCtxArr[i - 1]->hash == listCtxArr[i]->hash)
+			if (i > 0 && listsort_items[i - 1]->hash == listsort_items[i]->hash)
 				continue; // Ignore duplicates
 
-			_fingerprintNode(ctx, lfirst(list_nth_cell(node, listCtxArr[i]->list_pos)), parent, field_name, depth + 1);
+			_fingerprintNode(ctx, lfirst(list_nth_cell(node, listsort_items[i]->list_pos)), parent, field_name, depth + 1);
 		}
-	} else {
+	}
+	else
+	{
 		const ListCell *lc;
 
 		foreach(lc, node)
@@ -155,15 +214,28 @@ _fingerprintList(FingerprintContext *ctx, const List *node, const void *parent,
 }
 
 static void
-_fingerprintInitContext(FingerprintContext *ctx, bool write_tokens) {
+_fingerprintInitContext(FingerprintContext *ctx, FingerprintContext *parent, bool write_tokens)
+{
 	ctx->xxh_state = XXH3_createState();
 	if (ctx->xxh_state == NULL) abort();
 	if (XXH3_64bits_reset_withSeed(ctx->xxh_state, PG_QUERY_FINGERPRINT_VERSION) == XXH_ERROR) abort();
 
-	if (write_tokens) {
+	if (parent != NULL)
+	{
+		ctx->listsort_cache = parent->listsort_cache;
+	}
+	else
+	{
+		ctx->listsort_cache = listsort_cache_create(CurrentMemoryContext, 128, NULL);
+	}
+
+	if (write_tokens)
+	{
 		ctx->write_tokens = true;
 		dlist_init(&ctx->tokens);
-	} else {
+	}
+	else
+	{
 		ctx->write_tokens = false;
 	}
 }
@@ -237,7 +309,7 @@ PgQueryFingerprintResult pg_query_fingerprint_with_opts(const char* input, bool
 		FingerprintContext ctx;
 		XXH64_canonical_t chash;
 
-		_fingerprintInitContext(&ctx, printTokens);
+		_fingerprintInitContext(&ctx, NULL, printTokens);
 
 		if (parsetree_and_error.tree != NULL) {
 			_fingerprintNode(&ctx, parsetree_and_error.tree, NULL, NULL, 0);

data/ext/pg_query/pg_query_normalize.c

@@ -303,71 +303,123 @@ static bool const_record_walker(Node *node, pgssConstLocations *jstate)
 
 	if (node == NULL) return false;
 
-	if (IsA(node, A_Const))
+	switch (nodeTag(node))
 	{
-		RecordConstLocation(jstate, castNode(A_Const, node)->location);
-	}
-	else if (IsA(node, ParamRef))
-	{
-		/* Track the highest ParamRef number */
-		if (((ParamRef *) node)->number > jstate->highest_extern_param_id)
-			jstate->highest_extern_param_id = castNode(ParamRef, node)->number;
-	}
-	else if (IsA(node, DefElem))
-	{
-		DefElem * defElem = (DefElem *) node;
-		if (defElem->arg != NULL && IsA(defElem->arg, String)) {
-			for (int i = defElem->location; i < jstate->query_len; i++) {
-				if (jstate->query[i] == '\'') {
-					RecordConstLocation(jstate, i);
-					break;
-			  }
+		case T_A_Const:
+			RecordConstLocation(jstate, castNode(A_Const, node)->location);
+			break;
+		case T_ParamRef:
+			{
+				/* Track the highest ParamRef number */
+				if (((ParamRef *) node)->number > jstate->highest_extern_param_id)
+					jstate->highest_extern_param_id = castNode(ParamRef, node)->number;
 			}
-		}
-		return const_record_walker((Node *) ((DefElem *) node)->arg, jstate);
-	}
-	else if (IsA(node, RawStmt))
-	{
-		return const_record_walker((Node *) ((RawStmt *) node)->stmt, jstate);
-	}
-	else if (IsA(node, VariableSetStmt))
-	{
-		return const_record_walker((Node *) ((VariableSetStmt *) node)->args, jstate);
-	}
-	else if (IsA(node, CopyStmt))
-	{
-		return const_record_walker((Node *) ((CopyStmt *) node)->query, jstate);
-	}
-	else if (IsA(node, ExplainStmt))
-	{
-		return const_record_walker((Node *) ((ExplainStmt *) node)->query, jstate);
-	}
-	else if (IsA(node, CreateRoleStmt))
-	{
-		return const_record_walker((Node *) ((CreateRoleStmt *) node)->options, jstate);
-	}
-	else if (IsA(node, AlterRoleStmt))
-	{
-		return const_record_walker((Node *) ((AlterRoleStmt *) node)->options, jstate);
-	}
-	else if (IsA(node, DeclareCursorStmt))
-	{
-		return const_record_walker((Node *) ((DeclareCursorStmt *) node)->query, jstate);
-	}
+			break;
+		case T_DefElem:
+			{
+				DefElem * defElem = (DefElem *) node;
+				if (defElem->arg != NULL && IsA(defElem->arg, String)) {
+					for (int i = defElem->location; i < jstate->query_len; i++) {
+						if (jstate->query[i] == '\'') {
+							RecordConstLocation(jstate, i);
+							break;
+						}
+					}
+				}
+				return const_record_walker((Node *) ((DefElem *) node)->arg, jstate);
+			}
+			break;
+		case T_RawStmt:
+			return const_record_walker((Node *) ((RawStmt *) node)->stmt, jstate);
+		case T_VariableSetStmt:
+			return const_record_walker((Node *) ((VariableSetStmt *) node)->args, jstate);
+		case T_CopyStmt:
+			return const_record_walker((Node *) ((CopyStmt *) node)->query, jstate);
+		case T_ExplainStmt:
+			return const_record_walker((Node *) ((ExplainStmt *) node)->query, jstate);
+		case T_CreateRoleStmt:
+			return const_record_walker((Node *) ((CreateRoleStmt *) node)->options, jstate);
+		case T_AlterRoleStmt:
+			return const_record_walker((Node *) ((AlterRoleStmt *) node)->options, jstate);
+		case T_DeclareCursorStmt:
+			return const_record_walker((Node *) ((DeclareCursorStmt *) node)->query, jstate);
+		case T_SelectStmt:
+			{
+				SelectStmt *stmt = (SelectStmt *) node;
+				ListCell *lc;
+
+				if (const_record_walker((Node *) stmt->distinctClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->intoClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->targetList, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->fromClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->whereClause, jstate))
+					return true;
+
+				// Instead of walking all of groupClause (like raw_expression_tree_walker does),
+				// only walk certain items.
+				foreach(lc, stmt->groupClause)
+				{
+					// Do not walk A_Const values that are simple integers, this avoids
+					// turning "GROUP BY 1" into "GROUP BY $n", which obscures an important
+					// semantic meaning. This matches how pg_stat_statements handles the
+					// GROUP BY clause (i.e. it doesn't touch these constants)
+					if (IsA(lfirst(lc), A_Const) && IsA(&castNode(A_Const, lfirst(lc))->val, Integer))
+						continue;
+
+					if (const_record_walker((Node *) lfirst(lc), jstate))
+						return true;
+				}
+				foreach(lc, stmt->sortClause)
+				{
+					// Similarly, don't turn "ORDER BY 1" into "ORDER BY $n"
+					if (IsA(lfirst(lc), SortBy) && IsA(castNode(SortBy, lfirst(lc))->node, A_Const) &&
+					    IsA(&castNode(A_Const, castNode(SortBy, lfirst(lc))->node)->val, Integer))
+						continue;
 
-	PG_TRY();
-	{
-		result = raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
-	}
-	PG_CATCH();
-	{
-		MemoryContextSwitchTo(normalize_context);
-		result = false;
-		FlushErrorState();
+					if (const_record_walker((Node *) lfirst(lc), jstate))
+						return true;
+				}
+				if (const_record_walker((Node *) stmt->havingClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->windowClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->valuesLists, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->limitOffset, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->limitCount, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->lockingClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->withClause, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->larg, jstate))
+					return true;
+				if (const_record_walker((Node *) stmt->rarg, jstate))
+					return true;
+
+				return false;
+			}
+		default:
+			{
+				PG_TRY();
+				{
+					return raw_expression_tree_walker(node, const_record_walker, (void*) jstate);
+				}
+				PG_CATCH();
+				{
+					MemoryContextSwitchTo(normalize_context);
+					FlushErrorState();
+				}
+				PG_END_TRY();
+			}
 	}
-	PG_END_TRY();
 
-	return result;
+	return false;
 }
 
 PgQueryNormalizeResult pg_query_normalize(const char* input)

data/ext/pg_query/protobuf-c.c

@@ -84,7 +84,9 @@
 # define PROTOBUF_C_UNPACK_ERROR(...)
 #endif
 
+#if !defined(_WIN32) || !defined(PROTOBUF_C_USE_SHARED_LIB)
 const char protobuf_c_empty_string[] = "";
+#endif
 
 /**
  * Internal `ProtobufCMessage` manipulation macro.
@@ -148,12 +150,14 @@ protobuf_c_version_number(void)
 static void *
 system_alloc(void *allocator_data, size_t size)
 {
+	(void)allocator_data;
 	return malloc(size);
 }
 
 static void
 system_free(void *allocator_data, void *data)
 {
+	(void)allocator_data;
 	free(data);
 }
 
@@ -1618,22 +1622,19 @@ required_field_pack_to_buffer(const ProtobufCFieldDescriptor *field,
 		break;
 	}
 	case PROTOBUF_C_TYPE_MESSAGE: {
-		uint8_t simple_buffer_scratch[256];
-		size_t sublen;
 		const ProtobufCMessage *msg = *(ProtobufCMessage * const *) member;
-		ProtobufCBufferSimple simple_buffer =
-			PROTOBUF_C_BUFFER_SIMPLE_INIT(simple_buffer_scratch);
-
+		
 		scratch[0] |= PROTOBUF_C_WIRE_TYPE_LENGTH_PREFIXED;
-		if (msg == NULL)
-			sublen = 0;
-		else
-			sublen = protobuf_c_message_pack_to_buffer(msg, &simple_buffer.base);
-		rv += uint32_pack(sublen, scratch + rv);
-		buffer->append(buffer, rv, scratch);
-		buffer->append(buffer, sublen, simple_buffer.data);
-		rv += sublen;
-		PROTOBUF_C_BUFFER_SIMPLE_CLEAR(&simple_buffer);
+		if (msg == NULL) {
+			rv += uint32_pack(0, scratch + rv);
+			buffer->append(buffer, rv, scratch);
+		} else {
+			size_t sublen = protobuf_c_message_get_packed_size(msg);
+			rv += uint32_pack(sublen, scratch + rv);
+			buffer->append(buffer, rv, scratch);
+			protobuf_c_message_pack_to_buffer(msg, buffer);
+			rv += sublen;
+		}
 		break;
 	}
 	default:
@@ -2063,7 +2064,7 @@ static size_t
 parse_tag_and_wiretype(size_t len,
 		       const uint8_t *data,
 		       uint32_t *tag_out,
-		       ProtobufCWireType *wiretype_out)
+		       uint8_t *wiretype_out)
 {
 	unsigned max_rv = len > 5 ? 5 : len;
 	uint32_t tag = (data[0] & 0x7f) >> 3;
@@ -2095,9 +2096,9 @@ parse_tag_and_wiretype(size_t len,
 
 /* sizeof(ScannedMember) must be <= (1UL<<BOUND_SIZEOF_SCANNED_MEMBER_LOG2) */
 #define BOUND_SIZEOF_SCANNED_MEMBER_LOG2 5
-typedef struct _ScannedMember ScannedMember;
+typedef struct ScannedMember ScannedMember;
 /** Field as it's being read. */
-struct _ScannedMember {
+struct ScannedMember {
 	uint32_t tag;              /**< Field tag. */
 	uint8_t wire_type;         /**< Field type. */
 	uint8_t length_prefix_len; /**< Prefix length. */
@@ -2132,11 +2133,13 @@ scan_length_prefixed_data(size_t len, const uint8_t *data,
 		// Protobuf messages should always be less than 2 GiB in size.
 		// We also want to return early here so that hdr_len + val does
 		// not overflow on 32-bit systems.
-		PROTOBUF_C_UNPACK_ERROR("length prefix of %lu is too large", val);
+		PROTOBUF_C_UNPACK_ERROR("length prefix of %lu is too large",
+			(unsigned long int)val);
 		return 0;
 	}
 	if (hdr_len + val > len) {
-		PROTOBUF_C_UNPACK_ERROR("data too short after length-prefix of %lu", val);
+		PROTOBUF_C_UNPACK_ERROR("data too short after length-prefix of %lu",
+			(unsigned long int)val);
 		return 0;
 	}
 	return hdr_len + val;
@@ -2498,7 +2501,7 @@ parse_required_member(ScannedMember *scanned_member,
 {
 	unsigned len = scanned_member->len;
 	const uint8_t *data = scanned_member->data;
-	ProtobufCWireType wire_type = scanned_member->wire_type;
+	uint8_t wire_type = scanned_member->wire_type;
 
 	switch (scanned_member->field->type) {
 	case PROTOBUF_C_TYPE_ENUM:
@@ -2579,7 +2582,7 @@ parse_required_member(ScannedMember *scanned_member,
 		{
 			do_free(allocator, bd->data);
 		}
-		if (len - pref_len > 0) {
+		if (len > pref_len) {
 			bd->data = do_alloc(allocator, len - pref_len);
 			if (bd->data == NULL)
 				return FALSE;
@@ -2747,7 +2750,9 @@ parse_packed_repeated_member(ScannedMember *scanned_member,
 	const uint8_t *at = scanned_member->data + scanned_member->length_prefix_len;
 	size_t rem = scanned_member->len - scanned_member->length_prefix_len;
 	size_t count = 0;
+#if defined(WORDS_BIGENDIAN)
 	unsigned i;
+#endif
 
 	switch (field->type) {
 	case PROTOBUF_C_TYPE_SFIXED32:
@@ -2840,13 +2845,15 @@ parse_packed_repeated_member(ScannedMember *scanned_member,
 		}
 		break;
 	case PROTOBUF_C_TYPE_BOOL:
-		count = rem;
-		for (i = 0; i < count; i++) {
-			if (at[i] > 1) {
+		while (rem > 0) {
+			unsigned s = scan_varint(rem, at);
+			if (s == 0) {
 				PROTOBUF_C_UNPACK_ERROR("bad packed-repeated boolean value");
 				return FALSE;
 			}
-			((protobuf_c_boolean *) array)[i] = at[i];
+			((protobuf_c_boolean *) array)[count++] = parse_boolean(s, at);
+			at += s;
+			rem -= s;
 		}
 		break;
 	default:
@@ -3080,7 +3087,7 @@ protobuf_c_message_unpack(const ProtobufCMessageDescriptor *desc,
 
 	while (rem > 0) {
 		uint32_t tag;
-		ProtobufCWireType wire_type;
+		uint8_t wire_type;
 		size_t used = parse_tag_and_wiretype(rem, at, &tag, &wire_type);
 		const ProtobufCFieldDescriptor *field;
 		ScannedMember tmp;
@@ -3657,4 +3664,4 @@ protobuf_c_service_descriptor_get_method_by_name(const ProtobufCServiceDescripto
 	if (strcmp(desc->methods[desc->method_indices_by_name[start]].name, name) == 0)
 		return desc->methods + desc->method_indices_by_name[start];
 	return NULL;
-}
+}