pg_query 2.1.4 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe9be3677078ec8a4990a53c9e0aa74c21f2a66672fe3a0990b23c00a6da34af
-  data.tar.gz: 31e3bbcdfac0cd27a1553ac0f0d33f7cbd7e36ce8c5080f29a314da67adcc76a
+  metadata.gz: 9625ed6fd4d8f67673ee922f191996d2c157addf6d97aa092af682a21bf96ce7
+  data.tar.gz: 9a20dedc0420e5a7f2162c642c2ddee60cd98fd6bbfeff1b1affc181f6dcbc27
 SHA512:
-  metadata.gz: 7a8e0ef4b389446f952988903aa6b5772c6caabaea80d97e4f40affdf0830227285e60af0848a2786fcdab98232e96a5a75d3b5f73b24378b9ef67e9d24d9bf8
-  data.tar.gz: ffd90f7bdf9a155cb7a014eaf9c8bfbf06d0905e70bc5dadc739015c4d11695ead10ae1960e1bcee44e3d81d3c8a5b7d47b6705a32ef93daa8d909b328ee9602
+  metadata.gz: 7f4633dd45e1530cebc92731315a238f22e4111ba23e10ea6991b6466f444ce2b974ffd858aaf5c93c5b6597e004e0b2334ac1832c24851956e34dd629fe58ff
+  data.tar.gz: 36d225dc15a58db4336cdbdd571a385308403b93d5507ec25b0deb5179be87b8a00fb029b1e5b77bdf62f79861fedd39ba350a50897bcfec4d61e266b2cdf639

data/CHANGELOG.md

@@ -4,6 +4,34 @@
 
 * ...
 
+## 2.2.1     2022-01-20
+
+* Detect tables used in the query of a PREPARE statement ([#273](https://github.com/pganalyze/pg_query/pull/273))
+* Expose recursive walk functionality via walk! ([#268](https://github.com/pganalyze/pg_query/pull/268))
+* Retain schema in name when parsing out functions ([#272](https://github.com/pganalyze/pg_query/pull/272))
+
+## 2.2.0     2022-11-02
+
+* Update to libpg_query 13-2.2.0 ([#264](https://github.com/pganalyze/pg_query/pull/264))
+  - Fingerprinting version 3.1
+    - Fixes issue with "SELECT DISTINCT" having the same fingerprint as "SELECT"
+      (fingerprints for "SELECT DISTINCT" will change with this revision)
+    - Group additional DDL statements together that otherwise generate a lot of
+      unique fingerprints (ListenStmt, UnlistenStmt, NotifyStmt, CreateFunctionStmt,
+      FunctionParameter and DoStmt)
+  - Deparser improvements
+    - Prefix errors with "deparse", and remove some asserts
+    - Fix potential segfault when passing invalid protobuf (RawStmt without Stmt)
+  - Update to Postgres 13.8 patch release
+  - Backport Xcode 14.1 build fix from upcoming 13.9 release
+  - Normalize additional DDL statements
+  - Add support for analyzing PL/pgSQL code inside DO blocks
+  - Fix memory leak in pg_query_fingerprint error handling
+  - PL/pgSQL parser: Add support for Assert, SET, COMMIT, ROLLBACK and CALL
+  - Add support for parsing more operators that include a `?` character
+* Support deparsing deeply nested queries ([#259](https://github.com/pganalyze/pg_query/pull/259))
+
+
 ## 2.1.4     2022-09-19
 
 * Truncate: Simplify VALUES(...) lists
@@ -13,6 +41,7 @@
 * Find function calls referenced in expression indexes ([#249](https://github.com/pganalyze/pg_query/pull/249))
 * Drop `Init_pg_query` from exported symbol map ([#256](https://github.com/pganalyze/pg_query/pull/256))
 
+
 ## 2.1.3     2022-01-28
 
 * Track tables in EXCEPT and INTERSECT queries ([#239](https://github.com/pganalyze/pg_query/pull/239))

data/README.md

@@ -72,6 +72,8 @@ PgQuery.parse("SELECT 1")
 
 ### Modifying a parsed query and turning it into SQL again
 
+This is a simple example for `deparse`, for more complex modification, use `walk!`.
+
 ```ruby
 parsed_query = PgQuery.parse("SELECT * FROM users")
 
@@ -145,6 +147,36 @@ PgQuery.scan('SELECT 1 --comment')
  []]
 ```
 
+### Walking the parse tree
+
+For generalized use, PgQuery provides `walk!` as a means to recursively work with the parsed query.
+
+This can be used to create a bespoke pretty printer:
+
+```ruby
+parsed_query = PgQuery.parse "SELECT * FROM tbl"
+parsed_query.walk! { |node, k, v, location| puts k }
+```
+
+More usefully, this can be used to rewrite a query. For example:
+
+```ruby
+parsed_query.walk! do |node, k, v, location| puts k
+  next unless k.eql?(:range_var) || k.eql?(:relation)
+  next if v.relname.nil?
+  v.relname = "X_" + v.relname
+end
+
+parsed_query.deparse
+```
+
+There are some caveats, and limitations, in this example.
+
+First, some of the tree nodes are frozen. You can replace them, but you cannot modify in place.
+
+Second, table rewriting is a bit more nuanced than this example. While this will rewrite the table names, it will
+not correctly handle all CTEs, or rewrite columns with explicit table names.
+
 ## Differences from Upstream PostgreSQL
 
 This gem is based on [libpg_query](https://github.com/pganalyze/libpg_query),

data/Rakefile

@@ -5,8 +5,8 @@ require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
 require 'open-uri'
 
-LIB_PG_QUERY_TAG = '13-2.1.0'.freeze
-LIB_PG_QUERY_SHA256SUM = 'a01329ae5bac19b10b8ddf8012bd663a20f85f180d6d7b900c1a1ca8444d19a5'.freeze
+LIB_PG_QUERY_TAG = '13-2.2.0'.freeze
+LIB_PG_QUERY_SHA256SUM = '07916be1a2b780dee6feed936aaa04ccee2a3afde8570a6920c3a839c87539c6'.freeze
 
 Rake::ExtensionTask.new 'pg_query' do |ext|
   ext.lib_dir = 'lib/pg_query'

data/ext/pg_query/include/access/twophase.h

@@ -34,6 +34,8 @@ extern void TwoPhaseShmemInit(void);
 extern void AtAbort_Twophase(void);
 extern void PostPrepare_Twophase(void);
 
+extern TransactionId TwoPhaseGetXidByVirtualXID(VirtualTransactionId vxid,
+												bool *have_more);
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
 

data/ext/pg_query/include/access/xact.h

@@ -103,6 +103,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_NEEDIMMEDIATECOMMIT - records whether the top level statement
+ * is one that requires immediate commit, such as CREATE DATABASE.
+ */
+#define XACT_FLAGS_NEEDIMMEDIATECOMMIT			(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */

data/ext/pg_query/include/access/xlog_internal.h

@@ -79,8 +79,10 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
 #define XLP_LONG_HEADER				0x0002
 /* This flag indicates backup blocks starting in this page are optional */
 #define XLP_BKP_REMOVABLE			0x0004
+/* Replaces a missing contrecord; see CreateOverwriteContrecordRecord */
+#define XLP_FIRST_IS_OVERWRITE_CONTRECORD 0x0008
 /* All defined flag bits in xlp_info (used for validity checking of header) */
-#define XLP_ALL_FLAGS				0x0007
+#define XLP_ALL_FLAGS				0x000F
 
 #define XLogPageHeaderSize(hdr)		\
 	(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
@@ -252,6 +254,13 @@ typedef struct xl_restore_point
 	char		rp_name[MAXFNAMELEN];
 } xl_restore_point;
 
+/* Overwrite of prior contrecord */
+typedef struct xl_overwrite_contrecord
+{
+	XLogRecPtr	overwritten_lsn;
+	TimestampTz overwrite_time;
+} xl_overwrite_contrecord;
+
 /* End of recovery mark, when we don't do an END_OF_RECOVERY checkpoint */
 typedef struct xl_end_of_recovery
 {

data/ext/pg_query/include/access/xlogreader.h

@@ -250,6 +250,16 @@ struct XLogReaderState
 
 	/* Buffer to hold error message */
 	char	   *errormsg_buf;
+
+	/*
+	 * Set at the end of recovery: the start point of a partial record at the
+	 * end of WAL (InvalidXLogRecPtr if there wasn't one), and the start
+	 * location of its first contrecord that went missing.
+	 */
+	XLogRecPtr	abortedRecPtr;
+	XLogRecPtr	missingContrecPtr;
+	/* Set when XLP_FIRST_IS_OVERWRITE_CONTRECORD is found */
+	XLogRecPtr	overwrittenRecPtr;
 };
 
 /* Get a new XLogReader */

data/ext/pg_query/include/catalog/dependency.h

@@ -201,6 +201,8 @@ extern void recordMultipleDependencies(const ObjectAddress *depender,
 extern void recordDependencyOnCurrentExtension(const ObjectAddress *object,
 											   bool isReplace);
 
+extern void checkMembershipInCurrentExtension(const ObjectAddress *object);
+
 extern long deleteDependencyRecordsFor(Oid classId, Oid objectId,
 									   bool skipExtensionDeps);
 

data/ext/pg_query/include/catalog/pg_class.h

@@ -178,7 +178,7 @@ typedef FormData_pg_class *Form_pg_class;
 /*
  * an explicitly chosen candidate key's columns are used as replica identity.
  * Note this will still be set if the index has been dropped; in that case it
- * has the same meaning as 'd'.
+ * has the same meaning as 'n'.
  */
 #define		  REPLICA_IDENTITY_INDEX	'i'
 

data/ext/pg_query/include/catalog/pg_class_d.h

@@ -82,7 +82,7 @@
 /*
  * an explicitly chosen candidate key's columns are used as replica identity.
  * Note this will still be set if the index has been dropped; in that case it
- * has the same meaning as 'd'.
+ * has the same meaning as 'n'.
  */
 #define		  REPLICA_IDENTITY_INDEX	'i'
 

data/ext/pg_query/include/catalog/pg_control.h

@@ -76,6 +76,8 @@ typedef struct CheckPoint
 #define XLOG_END_OF_RECOVERY			0x90
 #define XLOG_FPI_FOR_HINT				0xA0
 #define XLOG_FPI						0xB0
+/* 0xC0 is used in Postgres 9.5-11 */
+#define XLOG_OVERWRITE_CONTRECORD		0xD0
 
 
 /*

data/ext/pg_query/include/catalog/pg_operator.h

@@ -95,7 +95,9 @@ extern ObjectAddress OperatorCreate(const char *operatorName,
 									bool canMerge,
 									bool canHash);
 
-extern ObjectAddress makeOperatorDependencies(HeapTuple tuple, bool isUpdate);
+extern ObjectAddress makeOperatorDependencies(HeapTuple tuple,
+											  bool makeExtensionDep,
+											  bool isUpdate);
 
 extern void OperatorUpd(Oid baseId, Oid commId, Oid negId, bool isDelete);
 

data/ext/pg_query/include/catalog/pg_publication.h

@@ -107,6 +107,9 @@ extern List *GetAllTablesPublicationRelations(bool pubviaroot);
 extern bool is_publishable_relation(Relation rel);
 extern ObjectAddress publication_add_relation(Oid pubid, Relation targetrel,
 											  bool if_not_exists);
+extern List *GetPubPartitionOptionRelations(List *result,
+											PublicationPartOpt pub_partopt,
+											Oid relid);
 
 extern Oid	get_publication_oid(const char *pubname, bool missing_ok);
 extern char *get_publication_name(Oid pubid, bool missing_ok);

data/ext/pg_query/include/catalog/pg_type.h

@@ -359,6 +359,7 @@ extern void GenerateTypeDependencies(HeapTuple typeTuple,
 														 * rowtypes */
 									 bool isImplicitArray,
 									 bool isDependentType,
+									 bool makeExtensionDep,
 									 bool rebuild);
 
 extern void RenameTypeInternal(Oid typeOid, const char *newTypeName,

data/ext/pg_query/include/commands/async.h

@@ -49,6 +49,6 @@ extern void ProcessCompletedNotifies(void);
 extern void HandleNotifyInterrupt(void);
 
 /* process interrupts */
-extern void ProcessNotifyInterrupt(void);
+extern void ProcessNotifyInterrupt(bool flush);
 
 #endif							/* ASYNC_H */

data/ext/pg_query/include/commands/tablespace.h

@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern bool allow_in_place_tablespaces;
+
 /* XLOG stuff */
 #define XLOG_TBLSPC_CREATE		0x00
 #define XLOG_TBLSPC_DROP		0x10

data/ext/pg_query/include/commands/trigger.h

@@ -162,12 +162,20 @@ extern ObjectAddress CreateTrigger(CreateTrigStmt *stmt, const char *queryString
 								   Oid relOid, Oid refRelOid, Oid constraintOid, Oid indexOid,
 								   Oid funcoid, Oid parentTriggerOid, Node *whenClause,
 								   bool isInternal, bool in_partition);
+extern ObjectAddress CreateTriggerFiringOn(CreateTrigStmt *stmt, const char *queryString,
+										   Oid relOid, Oid refRelOid, Oid constraintOid,
+										   Oid indexOid, Oid funcoid, Oid parentTriggerOid,
+										   Node *whenClause, bool isInternal, bool in_partition,
+										   char trigger_fires_when);
 
 extern void RemoveTriggerById(Oid trigOid);
 extern Oid	get_trigger_oid(Oid relid, const char *name, bool missing_ok);
 
 extern ObjectAddress renametrig(RenameStmt *stmt);
 
+extern void EnableDisableTriggerNew(Relation rel, const char *tgname,
+									char fires_when, bool skip_system, bool recurse,
+									LOCKMODE lockmode);
 extern void EnableDisableTrigger(Relation rel, const char *tgname,
 								 char fires_when, bool skip_system, LOCKMODE lockmode);
 

data/ext/pg_query/include/lib/simplehash.h

@@ -55,6 +55,11 @@
  *	  presence is relevant to determine whether a lookup needs to continue
  *	  looking or is done - buckets following a deleted element are shifted
  *	  backwards, unless they're empty or already at their optimal position.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/lib/simplehash.h
  */
 
 #include "port/pg_bitutils.h"
@@ -156,7 +161,7 @@ SH_SCOPE	SH_TYPE *SH_CREATE(MemoryContext ctx, uint32 nelements,
 #endif
 SH_SCOPE void SH_DESTROY(SH_TYPE * tb);
 SH_SCOPE void SH_RESET(SH_TYPE * tb);
-SH_SCOPE void SH_GROW(SH_TYPE * tb, uint32 newsize);
+SH_SCOPE void SH_GROW(SH_TYPE * tb, uint64 newsize);
 SH_SCOPE	SH_ELEMENT_TYPE *SH_INSERT(SH_TYPE * tb, SH_KEY_TYPE key, bool *found);
 SH_SCOPE	SH_ELEMENT_TYPE *SH_INSERT_HASH(SH_TYPE * tb, SH_KEY_TYPE key,
 											uint32 hash, bool *found);
@@ -218,7 +223,8 @@ SH_SCOPE void SH_STAT(SH_TYPE * tb);
 #define SIMPLEHASH_H
 
 #ifdef FRONTEND
-#define sh_error(...) pg_log_error(__VA_ARGS__)
+#define sh_error(...) \
+	do { pg_log_fatal(__VA_ARGS__); exit(1); } while(0)
 #define sh_log(...) pg_log_info(__VA_ARGS__)
 #else
 #define sh_error(...) elog(ERROR, __VA_ARGS__)
@@ -232,7 +238,7 @@ SH_SCOPE void SH_STAT(SH_TYPE * tb);
  * the hashtable.
  */
 static inline void
-SH_COMPUTE_PARAMETERS(SH_TYPE * tb, uint32 newsize)
+SH_COMPUTE_PARAMETERS(SH_TYPE * tb, uint64 newsize)
 {
 	uint64		size;
 
@@ -247,16 +253,12 @@ SH_COMPUTE_PARAMETERS(SH_TYPE * tb, uint32 newsize)
 	 * Verify that allocation of ->data is possible on this platform, without
 	 * overflowing Size.
 	 */
-	if ((((uint64) sizeof(SH_ELEMENT_TYPE)) * size) >= SIZE_MAX / 2)
+	if (unlikely((((uint64) sizeof(SH_ELEMENT_TYPE)) * size) >= SIZE_MAX / 2))
 		sh_error("hash table too large");
 
 	/* now set size */
 	tb->size = size;
-
-	if (tb->size == SH_MAX_SIZE)
-		tb->sizemask = 0;
-	else
-		tb->sizemask = tb->size - 1;
+	tb->sizemask = (uint32) (size - 1);
 
 	/*
 	 * Compute the next threshold at which we need to grow the hash table
@@ -406,7 +408,7 @@ SH_RESET(SH_TYPE * tb)
  * performance-wise, when known at some point.
  */
 SH_SCOPE void
-SH_GROW(SH_TYPE * tb, uint32 newsize)
+SH_GROW(SH_TYPE * tb, uint64 newsize)
 {
 	uint64		oldsize = tb->size;
 	SH_ELEMENT_TYPE *olddata = tb->data;
@@ -536,10 +538,8 @@ restart:
 	 */
 	if (unlikely(tb->members >= tb->grow_threshold))
 	{
-		if (tb->size == SH_MAX_SIZE)
-		{
+		if (unlikely(tb->size == SH_MAX_SIZE))
 			sh_error("hash table size exceeded");
-		}
 
 		/*
 		 * When optimizing, it can be very useful to print these out.

data/ext/pg_query/include/libpq/libpq.h

@@ -72,6 +72,7 @@ extern int	pq_getmessage(StringInfo s, int maxlen);
 extern int	pq_getbyte(void);
 extern int	pq_peekbyte(void);
 extern int	pq_getbyte_if_available(unsigned char *c);
+extern bool pq_buffer_has_data(void);
 extern int	pq_putbytes(const char *s, size_t len);
 
 /*

data/ext/pg_query/include/mb/pg_wchar.h

@@ -553,6 +553,7 @@ extern int	pg_valid_server_encoding_id(int encoding);
  * earlier in this file are also available from libpgcommon.
  */
 extern int	pg_encoding_mblen(int encoding, const char *mbstr);
+extern int	pg_encoding_mblen_bounded(int encoding, const char *mbstr);
 extern int	pg_encoding_dsplen(int encoding, const char *mbstr);
 extern int	pg_encoding_verifymb(int encoding, const char *mbstr, int len);
 extern int	pg_encoding_max_length(int encoding);

data/ext/pg_query/include/miscadmin.h

@@ -57,6 +57,15 @@
  * allowing die interrupts: HOLD_CANCEL_INTERRUPTS() and
  * RESUME_CANCEL_INTERRUPTS().
  *
+ * Note that ProcessInterrupts() has also acquired a number of tasks that
+ * do not necessarily cause a query-cancel-or-die response.  Hence, it's
+ * possible that it will just clear InterruptPending and return.
+ *
+ * INTERRUPTS_PENDING_CONDITION() can be checked to see whether an
+ * interrupt needs to be serviced, without trying to do so immediately.
+ * Some callers are also interested in INTERRUPTS_CAN_BE_PROCESSED(),
+ * which tells whether ProcessInterrupts is sure to clear the interrupt.
+ *
  * Special mechanisms are used to let an interrupt be accepted when we are
  * waiting for a lock or when we are waiting for command input (but, of
  * course, only if the interrupt holdoff counter is zero).  See the
@@ -94,24 +103,27 @@ extern PGDLLIMPORT __thread volatile uint32 CritSectionCount;
 /* in tcop/postgres.c */
 extern void ProcessInterrupts(void);
 
+/* Test whether an interrupt is pending */
 #ifndef WIN32
+#define INTERRUPTS_PENDING_CONDITION() \
+	(unlikely(InterruptPending))
+#else
+#define INTERRUPTS_PENDING_CONDITION() \
+	(unlikely(UNBLOCKED_SIGNAL_QUEUE()) ? pgwin32_dispatch_queued_signals() : 0, \
+	 unlikely(InterruptPending))
+#endif
 
+/* Service interrupt, if one is pending and it's safe to service it now */
 #define CHECK_FOR_INTERRUPTS() \
 do { \
-	if (unlikely(InterruptPending)) \
-		ProcessInterrupts(); \
-} while(0)
-#else							/* WIN32 */
-
-#define CHECK_FOR_INTERRUPTS() \
-do { \
-	if (unlikely(UNBLOCKED_SIGNAL_QUEUE())) \
-		pgwin32_dispatch_queued_signals(); \
-	if (unlikely(InterruptPending)) \
+	if (INTERRUPTS_PENDING_CONDITION()) \
 		ProcessInterrupts(); \
 } while(0)
-#endif							/* WIN32 */
 
+/* Is ProcessInterrupts() guaranteed to clear InterruptPending? */
+#define INTERRUPTS_CAN_BE_PROCESSED() \
+	(InterruptHoldoffCount == 0 && CritSectionCount == 0 && \
+	 QueryCancelHoldoffCount == 0)
 
 #define HOLD_INTERRUPTS()  (InterruptHoldoffCount++)
 
@@ -471,6 +483,7 @@ extern bool BackupInProgress(void);
 extern void CancelBackup(void);
 
 /* in executor/nodeHash.c */
+extern size_t get_hash_memory_limit(void);
 extern int	get_hash_mem(void);
 
 #endif							/* MISCADMIN_H */

data/ext/pg_query/include/nodes/execnodes.h

@@ -1454,7 +1454,7 @@ typedef struct IndexScanState
 /* ----------------
  *	 IndexOnlyScanState information
  *
- *		indexqual		   execution state for indexqual expressions
+ *		recheckqual		   execution state for recheckqual expressions
  *		ScanKeys		   Skey structures for index quals
  *		NumScanKeys		   number of ScanKeys
  *		OrderByKeys		   Skey structures for index ordering operators
@@ -1473,7 +1473,7 @@ typedef struct IndexScanState
 typedef struct IndexOnlyScanState
 {
 	ScanState	ss;				/* its first field is NodeTag */
-	ExprState  *indexqual;
+	ExprState  *recheckqual;
 	struct ScanKeyData *ioss_ScanKeys;
 	int			ioss_NumScanKeys;
 	struct ScanKeyData *ioss_OrderByKeys;

data/ext/pg_query/include/nodes/parsenodes.h

@@ -915,10 +915,10 @@ typedef struct PartitionCmd
  *	  inFromCl marks those range variables that are listed in the FROM clause.
  *	  It's false for RTEs that are added to a query behind the scenes, such
  *	  as the NEW and OLD variables for a rule, or the subqueries of a UNION.
- *	  This flag is not used anymore during parsing, since the parser now uses
- *	  a separate "namespace" data structure to control visibility, but it is
- *	  needed by ruleutils.c to determine whether RTEs should be shown in
- *	  decompiled queries.
+ *	  This flag is not used during parsing (except in transformLockingClause,
+ *	  q.v.); the parser now uses a separate "namespace" data structure to
+ *	  control visibility.  But it is needed by ruleutils.c to determine
+ *	  whether RTEs should be shown in decompiled queries.
  *
  *	  requiredPerms and checkAsUser specify run-time access permissions
  *	  checks to be performed at query startup.  The user must have *all*
@@ -1876,6 +1876,7 @@ typedef struct AlterTableCmd	/* one subcommand of an ALTER TABLE */
 								 * constraint, or parent table */
 	DropBehavior behavior;		/* RESTRICT or CASCADE for DROP cases */
 	bool		missing_ok;		/* skip error if missing? */
+	bool		recurse;		/* exec-time recursion */
 } AlterTableCmd;
 
 

data/ext/pg_query/include/nodes/pathnodes.h

@@ -258,7 +258,8 @@ struct PlannerInfo
 
 	List	   *init_plans;		/* init SubPlans for query */
 
-	List	   *cte_plan_ids;	/* per-CTE-item list of subplan IDs */
+	List	   *cte_plan_ids;	/* per-CTE-item list of subplan IDs (or -1 if
+								 * no subplan was made for that CTE) */
 
 	List	   *multiexpr_params;	/* List of Lists of Params for MULTIEXPR
 									 * subquery outputs */

data/ext/pg_query/include/nodes/pg_list.h

@@ -560,6 +560,7 @@ extern List *list_delete_int(List *list, int datum);
 extern List *list_delete_oid(List *list, Oid datum);
 extern List *list_delete_first(List *list);
 extern List *list_delete_last(List *list);
+extern List *list_delete_first_n(List *list, int n);
 extern List *list_delete_nth_cell(List *list, int n);
 extern List *list_delete_cell(List *list, ListCell *cell);
 

data/ext/pg_query/include/nodes/plannodes.h

@@ -418,14 +418,28 @@ typedef struct IndexScan
  * index-only scan, in which the data comes from the index not the heap.
  * Because of this, *all* Vars in the plan node's targetlist, qual, and
  * index expressions reference index columns and have varno = INDEX_VAR.
- * Hence we do not need separate indexqualorig and indexorderbyorig lists,
- * since their contents would be equivalent to indexqual and indexorderby.
+ *
+ * We could almost use indexqual directly against the index's output tuple
+ * when rechecking lossy index operators, but that won't work for quals on
+ * index columns that are not retrievable.  Hence, recheckqual is needed
+ * for rechecks: it expresses the same condition as indexqual, but using
+ * only index columns that are retrievable.  (We will not generate an
+ * index-only scan if this is not possible.  An example is that if an
+ * index has table column "x" in a retrievable index column "ind1", plus
+ * an expression f(x) in a non-retrievable column "ind2", an indexable
+ * query on f(x) will use "ind2" in indexqual and f(ind1) in recheckqual.
+ * Without the "ind1" column, an index-only scan would be disallowed.)
+ *
+ * We don't currently need a recheckable equivalent of indexorderby,
+ * because we don't support lossy operators in index ORDER BY.
  *
  * To help EXPLAIN interpret the index Vars for display, we provide
  * indextlist, which represents the contents of the index as a targetlist
  * with one TLE per index column.  Vars appearing in this list reference
  * the base table, and this is the only field in the plan node that may
- * contain such Vars.
+ * contain such Vars.  Also, for the convenience of setrefs.c, TLEs in
+ * indextlist are marked as resjunk if they correspond to columns that
+ * the index AM cannot reconstruct.
  * ----------------
  */
 typedef struct IndexOnlyScan
@@ -436,6 +450,7 @@ typedef struct IndexOnlyScan
 	List	   *indexorderby;	/* list of index ORDER BY exprs */
 	List	   *indextlist;		/* TargetEntry list describing index's cols */
 	ScanDirection indexorderdir;	/* forward or backward or don't care */
+	List	   *recheckqual;	/* index quals in recheckable form */
 } IndexOnlyScan;
 
 /* ----------------

data/ext/pg_query/include/optimizer/optimizer.h

@@ -24,11 +24,6 @@
 
 #include "nodes/parsenodes.h"
 
-/* Test if an expression node represents a SRF call.  Beware multiple eval! */
-#define IS_SRF_CALL(node) \
-	((IsA(node, FuncExpr) && ((FuncExpr *) (node))->funcretset) || \
-	 (IsA(node, OpExpr) && ((OpExpr *) (node))->opretset))
-
 /*
  * We don't want to include nodes/pathnodes.h here, because non-planner
  * code should generally treat PlannerInfo as an opaque typedef.

data/ext/pg_query/include/parser/gram.h

@@ -363,7 +363,7 @@
      REASSIGN = 579,
      RECHECK = 580,
      RECURSIVE = 581,
-     REF = 582,
+     REF_P = 582,
      REFERENCES = 583,
      REFERENCING = 584,
      REFRESH = 585,
@@ -840,7 +840,7 @@
 #define REASSIGN 579
 #define RECHECK 580
 #define RECURSIVE 581
-#define REF 582
+#define REF_P 582
 #define REFERENCES 583
 #define REFERENCING 584
 #define REFRESH 585

data/ext/pg_query/include/parser/kwlist.h

@@ -330,7 +330,7 @@ PG_KEYWORD("real", REAL, COL_NAME_KEYWORD)
 PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD)
 PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD)
 PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD)
-PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD)
+PG_KEYWORD("ref", REF_P, UNRESERVED_KEYWORD)
 PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD)
 PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD)
 PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD)

data/ext/pg_query/include/parser/parse_coerce.h

@@ -70,6 +70,7 @@ extern Oid	select_common_type(ParseState *pstate, List *exprs,
 extern Node *coerce_to_common_type(ParseState *pstate, Node *node,
 								   Oid targetTypeId,
 								   const char *context);
+extern bool verify_common_type(Oid common_type, List *exprs);
 
 extern bool check_generic_type_consistency(const Oid *actual_arg_types,
 										   const Oid *declared_arg_types,