pg_query 1.1.0 → 2.0.1

data/Rakefile

@@ -2,6 +2,10 @@ require 'bundler/gem_tasks'
 require 'rake/extensiontask'
 require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
+require 'open-uri'
+
+LIB_PG_QUERY_TAG = '13-2.0.0'.freeze
+LIB_PG_QUERY_SHA256SUM = 'e5e6514b1ae0e1ab19a2441de3088988bf1b89778164f9c725c3d85c6903133e'.freeze
 
 Rake::ExtensionTask.new 'pg_query' do |ext|
   ext.lib_dir = 'lib/pg_query'
@@ -12,7 +16,7 @@ RuboCop::RakeTask.new
 
 task spec: :compile
 
-task default: %i[lint spec]
+task default: %i[spec lint]
 task test: :spec
 task lint: :rubocop
 
@@ -21,3 +25,80 @@ task :clean do
   FileUtils.rm_f Dir.glob(File.join(__dir__, 'ext/pg_query/*.o'))
   FileUtils.rm_f File.join(__dir__, 'lib/pg_query/pg_query.bundle')
 end
+
+task :update_source do
+  workdir = File.join(__dir__, 'tmp')
+  libdir = File.join(workdir, 'libpg_query-' + LIB_PG_QUERY_TAG)
+  filename = File.join(workdir, 'libpg_query-' + LIB_PG_QUERY_TAG + '.tar.gz')
+  testfilesdir = File.join(__dir__, 'spec/files')
+  extdir = File.join(__dir__, 'ext/pg_query')
+  extbakdir = File.join(workdir, 'extbak')
+
+  unless File.exist?(filename)
+    system("mkdir -p #{workdir}")
+    File.open(filename, 'wb') do |target_file|
+      URI.open('https://codeload.github.com/pganalyze/libpg_query/tar.gz/' + LIB_PG_QUERY_TAG, 'rb') do |read_file|
+        target_file.write(read_file.read)
+      end
+    end
+
+    checksum = Digest::SHA256.hexdigest(File.read(filename))
+
+    if checksum != LIB_PG_QUERY_SHA256SUM
+      raise "SHA256 of #{filename} does not match: got #{checksum}, expected #{LIB_PG_QUERY_SHA256SUM}"
+    end
+  end
+
+  unless Dir.exist?(libdir)
+    system("tar -xzf #{filename} -C #{workdir}") || raise('ERROR')
+  end
+
+  # Backup important files from ext dir
+  system("rm -fr #{extbakdir}")
+  system("mkdir -p #{extbakdir}")
+  system("cp -a #{extdir}/pg_query_ruby.{c,sym} #{extdir}/extconf.rb #{extbakdir}")
+
+  FileUtils.rm_rf extdir
+
+  # Reduce everything down to one directory
+  system("mkdir -p #{extdir}")
+  system("cp -a #{libdir}/src/* #{extdir}/")
+  system("mv #{extdir}/postgres/* #{extdir}/")
+  system("rmdir #{extdir}/postgres")
+  system("cp -a #{libdir}/pg_query.h #{extdir}/include")
+  # Make sure every .c file in the top-level directory is its own translation unit
+  system("mv #{extdir}/*{_conds,_defs,_helper}.c #{extdir}/include")
+  # Protobuf definitions
+  system("protoc --proto_path=#{libdir}/protobuf --ruby_out=#{File.join(__dir__, 'lib/pg_query')} #{libdir}/protobuf/pg_query.proto")
+  system("mkdir -p #{extdir}/include/protobuf")
+  system("cp -a #{libdir}/protobuf/*.h #{extdir}/include/protobuf")
+  system("cp -a #{libdir}/protobuf/*.c #{extdir}/")
+  # Protobuf library code
+  system("mkdir -p #{extdir}/include/protobuf-c")
+  system("cp -a #{libdir}/vendor/protobuf-c/*.h #{extdir}/include")
+  system("cp -a #{libdir}/vendor/protobuf-c/*.h #{extdir}/include/protobuf-c")
+  system("cp -a #{libdir}/vendor/protobuf-c/*.c #{extdir}/")
+  # xxhash library code
+  system("mkdir -p #{extdir}/include/xxhash")
+  system("cp -a #{libdir}/vendor/xxhash/*.h #{extdir}/include")
+  system("cp -a #{libdir}/vendor/xxhash/*.h #{extdir}/include/xxhash")
+  system("cp -a #{libdir}/vendor/xxhash/*.c #{extdir}/")
+  # Other support files
+  system("cp -a #{libdir}/testdata/* #{testfilesdir}")
+  # Copy back the custom ext files
+  system("cp -a #{extbakdir}/pg_query_ruby.{c,sym} #{extbakdir}/extconf.rb #{extdir}")
+
+  # Generate JSON field name helper (workaround until https://github.com/protocolbuffers/protobuf/pull/8356 is merged)
+  str = "module PgQuery\n  INTERNAL_PROTO_FIELD_NAME_TO_JSON_NAME = {\n"
+  cur_type = nil
+  File.read(File.join(libdir, 'protobuf/pg_query.proto')).each_line do |line|
+    if line[/^message (\w+)/]
+      cur_type = $1
+      next
+    end
+    next unless line[/(repeated )?\w+ (\w+) = \d+( \[json_name="(\w+)"\])?;/]
+    str += format("    [%s, :%s] => '%s',\n", cur_type, $2, $4 || $2)
+  end
+  str += "  }\nend\n"
+  File.write(File.join(__dir__, 'lib/pg_query/json_field_names.rb'), str)
+end

data/ext/pg_query/extconf.rb

@@ -1,40 +1,12 @@
 # rubocop:disable Style/GlobalVars
 
+require 'digest'
 require 'mkmf'
 require 'open-uri'
 
-LIB_PG_QUERY_TAG = '10-1.0.1'.freeze
+$objs = Dir.glob(File.join(__dir__, '*.c')).map { |f| f.gsub('.c', '.o') }
 
-workdir = Dir.pwd
-libdir = File.join(workdir, 'libpg_query-' + LIB_PG_QUERY_TAG)
-gemdir = File.join(__dir__, '../..')
-libfile = libdir + '/libpg_query.a'
-
-unless File.exist?("#{workdir}/libpg_query.tar.gz")
-  File.open("#{workdir}/libpg_query.tar.gz", 'wb') do |target_file|
-    open('https://codeload.github.com/lfittl/libpg_query/tar.gz/' + LIB_PG_QUERY_TAG, 'rb') do |read_file|
-      target_file.write(read_file.read)
-    end
-  end
-end
-
-unless Dir.exist?(libdir)
-  system("tar -xf #{workdir}/libpg_query.tar.gz") || raise('ERROR')
-end
-
-unless Dir.exist?(libfile)
-  # Build libpg_query (and parts of PostgreSQL)
-  system("cd #{libdir}; #{ENV['MAKE'] || (RUBY_PLATFORM =~ /bsd/ ? 'gmake' : 'make')} build")
-end
-
-# Copy test files (this intentionally overwrites existing files!)
-system("cp #{libdir}/testdata/* #{gemdir}/spec/files/")
-
-$objs = ['pg_query_ruby.o']
-
-$LOCAL_LIBS << '-lpg_query'
-$LIBPATH << libdir
-$CFLAGS << " -I #{libdir} -O3 -Wall -fno-strict-aliasing -fwrapv -g"
+$CFLAGS << " -I#{File.join(__dir__, 'include')} -O3 -Wall -fno-strict-aliasing -fwrapv -fstack-protector -Wno-unused-function -Wno-unused-variable -g"
 
 SYMFILE = File.join(__dir__, 'pg_query_ruby.sym')
 if RUBY_PLATFORM =~ /darwin/

data/ext/pg_query/include/access/amapi.h

@@ -0,0 +1,246 @@
+/*-------------------------------------------------------------------------
+ *
+ * amapi.h
+ *	  API for Postgres index access methods.
+ *
+ * Copyright (c) 2015-2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/amapi.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef AMAPI_H
+#define AMAPI_H
+
+#include "access/genam.h"
+
+/*
+ * We don't wish to include planner header files here, since most of an index
+ * AM's implementation isn't concerned with those data structures.  To allow
+ * declaring amcostestimate_function here, use forward struct references.
+ */
+struct PlannerInfo;
+struct IndexPath;
+
+/* Likewise, this file shouldn't depend on execnodes.h. */
+struct IndexInfo;
+
+
+/*
+ * Properties for amproperty API.  This list covers properties known to the
+ * core code, but an index AM can define its own properties, by matching the
+ * string property name.
+ */
+typedef enum IndexAMProperty
+{
+	AMPROP_UNKNOWN = 0,			/* anything not known to core code */
+	AMPROP_ASC,					/* column properties */
+	AMPROP_DESC,
+	AMPROP_NULLS_FIRST,
+	AMPROP_NULLS_LAST,
+	AMPROP_ORDERABLE,
+	AMPROP_DISTANCE_ORDERABLE,
+	AMPROP_RETURNABLE,
+	AMPROP_SEARCH_ARRAY,
+	AMPROP_SEARCH_NULLS,
+	AMPROP_CLUSTERABLE,			/* index properties */
+	AMPROP_INDEX_SCAN,
+	AMPROP_BITMAP_SCAN,
+	AMPROP_BACKWARD_SCAN,
+	AMPROP_CAN_ORDER,			/* AM properties */
+	AMPROP_CAN_UNIQUE,
+	AMPROP_CAN_MULTI_COL,
+	AMPROP_CAN_EXCLUDE,
+	AMPROP_CAN_INCLUDE
+} IndexAMProperty;
+
+
+/*
+ * Callback function signatures --- see indexam.sgml for more info.
+ */
+
+/* build new index */
+typedef IndexBuildResult *(*ambuild_function) (Relation heapRelation,
+											   Relation indexRelation,
+											   struct IndexInfo *indexInfo);
+
+/* build empty index */
+typedef void (*ambuildempty_function) (Relation indexRelation);
+
+/* insert this tuple */
+typedef bool (*aminsert_function) (Relation indexRelation,
+								   Datum *values,
+								   bool *isnull,
+								   ItemPointer heap_tid,
+								   Relation heapRelation,
+								   IndexUniqueCheck checkUnique,
+								   struct IndexInfo *indexInfo);
+
+/* bulk delete */
+typedef IndexBulkDeleteResult *(*ambulkdelete_function) (IndexVacuumInfo *info,
+														 IndexBulkDeleteResult *stats,
+														 IndexBulkDeleteCallback callback,
+														 void *callback_state);
+
+/* post-VACUUM cleanup */
+typedef IndexBulkDeleteResult *(*amvacuumcleanup_function) (IndexVacuumInfo *info,
+															IndexBulkDeleteResult *stats);
+
+/* can indexscan return IndexTuples? */
+typedef bool (*amcanreturn_function) (Relation indexRelation, int attno);
+
+/* estimate cost of an indexscan */
+typedef void (*amcostestimate_function) (struct PlannerInfo *root,
+										 struct IndexPath *path,
+										 double loop_count,
+										 Cost *indexStartupCost,
+										 Cost *indexTotalCost,
+										 Selectivity *indexSelectivity,
+										 double *indexCorrelation,
+										 double *indexPages);
+
+/* parse index reloptions */
+typedef bytea *(*amoptions_function) (Datum reloptions,
+									  bool validate);
+
+/* report AM, index, or index column property */
+typedef bool (*amproperty_function) (Oid index_oid, int attno,
+									 IndexAMProperty prop, const char *propname,
+									 bool *res, bool *isnull);
+
+/* name of phase as used in progress reporting */
+typedef char *(*ambuildphasename_function) (int64 phasenum);
+
+/* validate definition of an opclass for this AM */
+typedef bool (*amvalidate_function) (Oid opclassoid);
+
+/* prepare for index scan */
+typedef IndexScanDesc (*ambeginscan_function) (Relation indexRelation,
+											   int nkeys,
+											   int norderbys);
+
+/* (re)start index scan */
+typedef void (*amrescan_function) (IndexScanDesc scan,
+								   ScanKey keys,
+								   int nkeys,
+								   ScanKey orderbys,
+								   int norderbys);
+
+/* next valid tuple */
+typedef bool (*amgettuple_function) (IndexScanDesc scan,
+									 ScanDirection direction);
+
+/* fetch all valid tuples */
+typedef int64 (*amgetbitmap_function) (IndexScanDesc scan,
+									   TIDBitmap *tbm);
+
+/* end index scan */
+typedef void (*amendscan_function) (IndexScanDesc scan);
+
+/* mark current scan position */
+typedef void (*ammarkpos_function) (IndexScanDesc scan);
+
+/* restore marked scan position */
+typedef void (*amrestrpos_function) (IndexScanDesc scan);
+
+/*
+ * Callback function signatures - for parallel index scans.
+ */
+
+/* estimate size of parallel scan descriptor */
+typedef Size (*amestimateparallelscan_function) (void);
+
+/* prepare for parallel index scan */
+typedef void (*aminitparallelscan_function) (void *target);
+
+/* (re)start parallel index scan */
+typedef void (*amparallelrescan_function) (IndexScanDesc scan);
+
+/*
+ * API struct for an index AM.  Note this must be stored in a single palloc'd
+ * chunk of memory.
+ */
+typedef struct IndexAmRoutine
+{
+	NodeTag		type;
+
+	/*
+	 * Total number of strategies (operators) by which we can traverse/search
+	 * this AM.  Zero if AM does not have a fixed set of strategy assignments.
+	 */
+	uint16		amstrategies;
+	/* total number of support functions that this AM uses */
+	uint16		amsupport;
+	/* opclass options support function number or 0 */
+	uint16		amoptsprocnum;
+	/* does AM support ORDER BY indexed column's value? */
+	bool		amcanorder;
+	/* does AM support ORDER BY result of an operator on indexed column? */
+	bool		amcanorderbyop;
+	/* does AM support backward scanning? */
+	bool		amcanbackward;
+	/* does AM support UNIQUE indexes? */
+	bool		amcanunique;
+	/* does AM support multi-column indexes? */
+	bool		amcanmulticol;
+	/* does AM require scans to have a constraint on the first index column? */
+	bool		amoptionalkey;
+	/* does AM handle ScalarArrayOpExpr quals? */
+	bool		amsearcharray;
+	/* does AM handle IS NULL/IS NOT NULL quals? */
+	bool		amsearchnulls;
+	/* can index storage data type differ from column data type? */
+	bool		amstorage;
+	/* can an index of this type be clustered on? */
+	bool		amclusterable;
+	/* does AM handle predicate locks? */
+	bool		ampredlocks;
+	/* does AM support parallel scan? */
+	bool		amcanparallel;
+	/* does AM support columns included with clause INCLUDE? */
+	bool		amcaninclude;
+	/* does AM use maintenance_work_mem? */
+	bool		amusemaintenanceworkmem;
+	/* OR of parallel vacuum flags.  See vacuum.h for flags. */
+	uint8		amparallelvacuumoptions;
+	/* type of data stored in index, or InvalidOid if variable */
+	Oid			amkeytype;
+
+	/*
+	 * If you add new properties to either the above or the below lists, then
+	 * they should also (usually) be exposed via the property API (see
+	 * IndexAMProperty at the top of the file, and utils/adt/amutils.c).
+	 */
+
+	/* interface functions */
+	ambuild_function ambuild;
+	ambuildempty_function ambuildempty;
+	aminsert_function aminsert;
+	ambulkdelete_function ambulkdelete;
+	amvacuumcleanup_function amvacuumcleanup;
+	amcanreturn_function amcanreturn;	/* can be NULL */
+	amcostestimate_function amcostestimate;
+	amoptions_function amoptions;
+	amproperty_function amproperty; /* can be NULL */
+	ambuildphasename_function ambuildphasename; /* can be NULL */
+	amvalidate_function amvalidate;
+	ambeginscan_function ambeginscan;
+	amrescan_function amrescan;
+	amgettuple_function amgettuple; /* can be NULL */
+	amgetbitmap_function amgetbitmap;	/* can be NULL */
+	amendscan_function amendscan;
+	ammarkpos_function ammarkpos;	/* can be NULL */
+	amrestrpos_function amrestrpos; /* can be NULL */
+
+	/* interface functions to support parallel index scans */
+	amestimateparallelscan_function amestimateparallelscan; /* can be NULL */
+	aminitparallelscan_function aminitparallelscan; /* can be NULL */
+	amparallelrescan_function amparallelrescan; /* can be NULL */
+} IndexAmRoutine;
+
+
+/* Functions in access/index/amapi.c */
+extern IndexAmRoutine *GetIndexAmRoutine(Oid amhandler);
+extern IndexAmRoutine *GetIndexAmRoutineByAmId(Oid amoid, bool noerror);
+
+#endif							/* AMAPI_H */

data/ext/pg_query/include/access/attmap.h

@@ -0,0 +1,52 @@
+/*-------------------------------------------------------------------------
+ *
+ * attmap.h
+ *	  Definitions for PostgreSQL attribute mappings
+ *
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/access/attmap.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef ATTMAP_H
+#define ATTMAP_H
+
+#include "access/attnum.h"
+#include "access/tupdesc.h"
+
+/*
+ * Attribute mapping structure
+ *
+ * This maps attribute numbers between a pair of relations, designated
+ * 'input' and 'output' (most typically inheritance parent and child
+ * relations), whose common columns may have different attribute numbers.
+ * Such difference may arise due to the columns being ordered differently
+ * in the two relations or the two relations having dropped columns at
+ * different positions.
+ *
+ * 'maplen' is set to the number of attributes of the 'output' relation,
+ * taking into account any of its dropped attributes, with the corresponding
+ * elements of the 'attnums' array set to 0.
+ */
+typedef struct AttrMap
+{
+	AttrNumber *attnums;
+	int			maplen;
+} AttrMap;
+
+extern AttrMap *make_attrmap(int maplen);
+extern void free_attrmap(AttrMap *map);
+
+/* Conversion routines to build mappings */
+extern AttrMap *build_attrmap_by_name(TupleDesc indesc,
+									  TupleDesc outdesc);
+extern AttrMap *build_attrmap_by_name_if_req(TupleDesc indesc,
+											 TupleDesc outdesc);
+extern AttrMap *build_attrmap_by_position(TupleDesc indesc,
+										  TupleDesc outdesc,
+										  const char *msg);
+
+#endif							/* ATTMAP_H */

data/ext/pg_query/include/access/attnum.h

@@ -0,0 +1,64 @@
+/*-------------------------------------------------------------------------
+ *
+ * attnum.h
+ *	  POSTGRES attribute number definitions.
+ *
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/access/attnum.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef ATTNUM_H
+#define ATTNUM_H
+
+
+/*
+ * user defined attribute numbers start at 1.   -ay 2/95
+ */
+typedef int16 AttrNumber;
+
+#define InvalidAttrNumber		0
+#define MaxAttrNumber			32767
+
+/* ----------------
+ *		support macros
+ * ----------------
+ */
+/*
+ * AttributeNumberIsValid
+ *		True iff the attribute number is valid.
+ */
+#define AttributeNumberIsValid(attributeNumber) \
+	((bool) ((attributeNumber) != InvalidAttrNumber))
+
+/*
+ * AttrNumberIsForUserDefinedAttr
+ *		True iff the attribute number corresponds to an user defined attribute.
+ */
+#define AttrNumberIsForUserDefinedAttr(attributeNumber) \
+	((bool) ((attributeNumber) > 0))
+
+/*
+ * AttrNumberGetAttrOffset
+ *		Returns the attribute offset for an attribute number.
+ *
+ * Note:
+ *		Assumes the attribute number is for a user defined attribute.
+ */
+#define AttrNumberGetAttrOffset(attNum) \
+( \
+	AssertMacro(AttrNumberIsForUserDefinedAttr(attNum)), \
+	((attNum) - 1) \
+)
+
+/*
+ * AttrOffsetGetAttrNumber
+ *		Returns the attribute number for an attribute offset.
+ */
+#define AttrOffsetGetAttrNumber(attributeOffset) \
+	 ((AttrNumber) (1 + (attributeOffset)))
+
+#endif							/* ATTNUM_H */