pg_query 1.3.0 → 2.1.0

data/lib/pg_query/filter_columns.rb

@@ -1,107 +1,110 @@
-class PgQuery
-  # Returns a list of columns that the query filters by - this excludes the
-  # target list, but includes things like JOIN condition and WHERE clause.
-  #
-  # Note: This also traverses into sub-selects.
-  def filter_columns # rubocop:disable Metrics/CyclomaticComplexity
-    load_tables_and_aliases! if @aliases.nil?
+module PgQuery
+  class ParserResult
+    # Returns a list of columns that the query filters by - this excludes the
+    # target list, but includes things like JOIN condition and WHERE clause.
+    #
+    # Note: This also traverses into sub-selects.
+    def filter_columns # rubocop:disable Metrics/CyclomaticComplexity
+      load_objects! if @aliases.nil?
 
-    # Get condition items from the parsetree
-    statements = @tree.dup
-    condition_items = []
-    filter_columns = []
-    loop do
-      statement = statements.shift
-      if statement
-        if statement[RAW_STMT]
-          statements << statement[RAW_STMT][STMT_FIELD]
-        elsif statement[SELECT_STMT]
-          case statement[SELECT_STMT]['op']
-          when 0
-            if statement[SELECT_STMT][FROM_CLAUSE_FIELD]
-              # FROM subselects
-              statement[SELECT_STMT][FROM_CLAUSE_FIELD].each do |item|
-                next unless item['RangeSubselect']
-                statements << item['RangeSubselect']['subquery']
-              end
+      # Get condition items from the parsetree
+      statements = @tree.stmts.dup.to_a.map(&:stmt)
+      condition_items = []
+      filter_columns = []
+      loop do
+        statement = statements.shift
+        if statement
+          case statement.node
+          when :list
+            statements += statement.list.items
+          when :raw_stmt
+            statements << statement.raw_stmt.stmt
+          when :select_stmt
+            case statement.select_stmt.op
+            when :SETOP_NONE
+              if statement.select_stmt.from_clause
+                # FROM subselects
+                statement.select_stmt.from_clause.each do |item|
+                  next unless item['RangeSubselect']
+                  statements << item['RangeSubselect']['subquery']
+                end
 
-              # JOIN ON conditions
-              condition_items += conditions_from_join_clauses(statement[SELECT_STMT][FROM_CLAUSE_FIELD])
-            end
+                # JOIN ON conditions
+                condition_items += conditions_from_join_clauses(statement.select_stmt.from_clause)
+              end
 
-            # WHERE clause
-            condition_items << statement[SELECT_STMT]['whereClause'] if statement[SELECT_STMT]['whereClause']
+              # WHERE clause
+              condition_items << statement.select_stmt.where_clause if statement.select_stmt.where_clause
 
-            # CTEs
-            if statement[SELECT_STMT]['withClause']
-              statement[SELECT_STMT]['withClause']['WithClause']['ctes'].each do |item|
-                statements << item['CommonTableExpr']['ctequery'] if item['CommonTableExpr']
+              # CTEs
+              if statement.select_stmt.with_clause
+                statement.select_stmt.with_clause.ctes.each do |item|
+                  statements << item.common_table_expr.ctequery if item.node == :common_table_expr
+                end
               end
+            when :SETOP_UNION
+              statements << statement.select_stmt.larg if statement.select_stmt.larg
+              statements << statement.select_stmt.rarg if statement.select_stmt.rarg
             end
-          when 1
-            statements << statement[SELECT_STMT]['larg'] if statement[SELECT_STMT]['larg']
-            statements << statement[SELECT_STMT]['rarg'] if statement[SELECT_STMT]['rarg']
+          when :update_stmt
+            condition_items << statement.update_stmt.where_clause if statement.update_stmt.where_clause
+          when :delete_stmt
+            condition_items << statement.delete_stmt.where_clause if statement.delete_stmt.where_clause
           end
-        elsif statement['UpdateStmt']
-          condition_items << statement['UpdateStmt']['whereClause'] if statement['UpdateStmt']['whereClause']
-        elsif statement['DeleteStmt']
-          condition_items << statement['DeleteStmt']['whereClause'] if statement['DeleteStmt']['whereClause']
         end
-      end
 
-      # Process both JOIN and WHERE conditions here
-      next_item = condition_items.shift
-      if next_item
-        if next_item[A_EXPR]
-          %w[lexpr rexpr].each do |side|
-            expr = next_item.values[0][side]
-            next unless expr && expr.is_a?(Hash)
-            condition_items << expr
+        # Process both JOIN and WHERE conditions here
+        next_item = condition_items.shift
+        if next_item
+          case next_item.node
+          when :a_expr
+            condition_items << next_item.a_expr.lexpr if next_item.a_expr.lexpr
+            condition_items << next_item.a_expr.rexpr if next_item.a_expr.rexpr
+          when :bool_expr
+            condition_items += next_item.bool_expr.args
+          when :coalesce_expr
+            condition_items += next_item.coalesce_expr.args
+          when :row_expr
+            condition_items += next_item.row_expr.args
+          when :column_ref
+            column, table = next_item.column_ref.fields.map { |f| f.string.str }.reverse
+            filter_columns << [@aliases[table] || table, column]
+          when :null_test
+            condition_items << next_item.null_test.arg
+          when :boolean_test
+            condition_items << next_item.boolean_test.arg
+          when :func_call
+            # FIXME: This should actually be extracted as a funccall and be compared with those indices
+            condition_items += next_item.func_call.args if next_item.func_call.args
+          when :sub_link
+            condition_items << next_item.sub_link.testexpr
+            statements << next_item.sub_link.subselect
           end
-        elsif next_item[BOOL_EXPR]
-          condition_items += next_item[BOOL_EXPR]['args']
-        elsif next_item[ROW_EXPR]
-          condition_items += next_item[ROW_EXPR]['args']
-        elsif next_item[COLUMN_REF]
-          column, table = next_item[COLUMN_REF]['fields'].map { |f| f['String']['str'] }.reverse
-          filter_columns << [@aliases[table] || table, column]
-        elsif next_item[NULL_TEST]
-          condition_items << next_item[NULL_TEST]['arg']
-        elsif next_item[BOOLEAN_TEST]
-          condition_items << next_item[BOOLEAN_TEST]['arg']
-        elsif next_item[FUNC_CALL]
-          # FIXME: This should actually be extracted as a funccall and be compared with those indices
-          condition_items += next_item[FUNC_CALL]['args'] if next_item[FUNC_CALL]['args']
-        elsif next_item[SUB_LINK]
-          condition_items << next_item[SUB_LINK]['testexpr']
-          statements << next_item[SUB_LINK]['subselect']
         end
+
+        break if statements.empty? && condition_items.empty?
       end
 
-      break if statements.empty? && condition_items.empty?
+      filter_columns.uniq
     end
 
-    filter_columns.uniq
-  end
-
-  protected
+    protected
 
-  def conditions_from_join_clauses(from_clause)
-    condition_items = []
-    from_clause.each do |item|
-      next unless item[JOIN_EXPR]
+    def conditions_from_join_clauses(from_clause)
+      condition_items = []
+      from_clause.each do |item|
+        next unless item.node == :join_expr
 
-      joinexpr_items = [item[JOIN_EXPR]]
-      loop do
-        next_item = joinexpr_items.shift
-        break unless next_item
-        condition_items << next_item['quals'] if next_item['quals']
-        %w[larg rarg].each do |side|
-          next unless next_item[side][JOIN_EXPR]
-          joinexpr_items << next_item[side][JOIN_EXPR]
+        joinexpr_items = [item.join_expr]
+        loop do
+          next_item = joinexpr_items.shift
+          break unless next_item
+          condition_items << next_item.quals if next_item.quals
+          joinexpr_items << next_item.larg.join_expr if next_item.larg.node == :join_expr
+          joinexpr_items << next_item.rarg.join_expr if next_item.rarg.node == :join_expr
         end
       end
+      condition_items
     end
-    condition_items
   end
 end

data/lib/pg_query/fingerprint.rb

@@ -1,115 +1,148 @@
 require 'digest'
 
-class PgQuery
-  def fingerprint
-    hash = Digest::SHA1.new
-    fingerprint_tree(hash)
-    format('%02x', FINGERPRINT_VERSION) + hash.hexdigest
-  end
+module PgQuery
+  class ParserResult
+    def fingerprint
+      hash = FingerprintSubHash.new
+      fingerprint_tree(hash)
+      fp = PgQuery.hash_xxh3_64(hash.parts.join, FINGERPRINT_VERSION)
+      format('%016x', fp)
+    end
 
-  private
+    private
 
-  FINGERPRINT_VERSION = 2
+    FINGERPRINT_VERSION = 3
 
-  class FingerprintSubHash
-    attr_reader :parts
+    class FingerprintSubHash
+      attr_reader :parts
 
-    def initialize
-      @parts = []
-    end
+      def initialize
+        @parts = []
+      end
 
-    def update(part)
-      @parts << part
-    end
+      def update(part)
+        @parts << part
+      end
 
-    def flush_to(hash)
-      parts.each do |part|
-        hash.update part
+      def flush_to(hash)
+        parts.each do |part|
+          hash.update part
+        end
       end
     end
-  end
 
-  def ignored_fingerprint_value?(val)
-    [nil, 0, false, [], ''].include?(val)
-  end
+    def ignored_fingerprint_value?(val)
+      [nil, 0, false, [], ''].include?(val)
+    end
 
-  def fingerprint_value(val, hash, parent_node_name, parent_field_name, need_to_write_name)
-    return if ignored_fingerprint_value?(val)
+    def fingerprint_value(val, hash, parent_node_name, parent_field_name, need_to_write_name) # rubocop:disable Metrics/CyclomaticComplexity
+      subhash = FingerprintSubHash.new
+
+      if val.is_a?(Google::Protobuf::RepeatedField)
+        fingerprint_list(val, subhash, parent_node_name, parent_field_name)
+      elsif val.is_a?(List)
+        fingerprint_list(val.items, subhash, parent_node_name, parent_field_name)
+      elsif val.is_a?(Google::Protobuf::MessageExts)
+        fingerprint_node(val, subhash, parent_node_name, parent_field_name)
+      elsif !ignored_fingerprint_value?(val)
+        subhash.update val.to_s
+      end
 
-    subhash = FingerprintSubHash.new
+      return if subhash.parts.empty?
 
-    if val.is_a?(Hash)
-      fingerprint_node(val, subhash, parent_node_name, parent_field_name)
-    elsif val.is_a?(Array)
-      fingerprint_list(val, subhash, parent_node_name, parent_field_name)
-    else
-      subhash.update val.to_s
+      hash.update(parent_field_name) if need_to_write_name
+      subhash.flush_to(hash)
     end
 
-    return if subhash.parts.empty?
-
-    hash.update(parent_field_name) if need_to_write_name
-    subhash.flush_to(hash)
-  end
-
-  def fingerprint_node(node, hash, parent_node_name = nil, parent_field_name = nil) # rubocop:disable Metrics/CyclomaticComplexity
-    node_name = node.keys.first
-    return if [A_CONST, ALIAS, PARAM_REF, SET_TO_DEFAULT, INT_LIST, OID_LIST, NULL].include?(node_name)
-
-    hash.update node_name
-
-    fields = node.values.first
-    fields.sort_by { |k, _| k }.each do |field_name, val|
-      next if ignored_fingerprint_value?(val)
-
-      case field_name
-      when 'location'
-        next
-      when 'name'
-        next if node_name == RES_TARGET && parent_node_name == SELECT_STMT && parent_field_name == TARGET_LIST_FIELD
-        next if [PREPARE_STMT, EXECUTE_STMT, DEALLOCATE_STMT].include?(node_name)
-      when 'gid'
-        next if node_name == TRANSACTION_STMT
-      when 'options'
-        next if node_name == TRANSACTION_STMT
-      when 'portalname'
-        next if [DECLARE_CURSOR_STMT, FETCH_STMT, CLOSE_PORTAL_STMT].include?(node_name)
-      when 'relname'
-        next if node_name == RANGE_VAR && fields[RELPERSISTENCE_FIELD] == 't'
-      when 'stmt_len'
-        next if node_name == RAW_STMT
-      when 'stmt_location'
-        next if node_name == RAW_STMT
-      end
+    def ignored_node_type?(node)
+      [A_Const, Alias, ParamRef, SetToDefault, IntList, OidList, Null].include?(node.class) ||
+        node.is_a?(TypeCast) && (node.arg.node == :a_const || node.arg.node == :param_ref)
+    end
 
-      fingerprint_value(val, hash, node_name, field_name, true)
+    def node_protobuf_field_name_to_json_name(node_class, field)
+      node_class.descriptor.find { |d| d.name == field.to_s }.json_name
     end
-  end
 
-  def fingerprint_list(values, hash, parent_node_name, parent_field_name)
-    if [FROM_CLAUSE_FIELD, TARGET_LIST_FIELD, COLS_FIELD, REXPR_FIELD, VALUES_LISTS_FIELD].include?(parent_field_name)
-      values_subhashes = values.map do |val|
-        subhash = FingerprintSubHash.new
-        fingerprint_value(val, subhash, parent_node_name, parent_field_name, false)
-        subhash
+    def fingerprint_node(node, hash, parent_node_name = nil, parent_field_name = nil) # rubocop:disable Metrics/CyclomaticComplexity
+      return if ignored_node_type?(node)
+
+      if node.is_a?(Node)
+        return if node.node.nil?
+        node_val = node[node.node.to_s]
+        unless ignored_node_type?(node_val)
+          unless node_val.is_a?(List)
+            postgres_node_name = node_protobuf_field_name_to_json_name(node.class, node.node)
+            hash.update(postgres_node_name)
+          end
+          fingerprint_value(node_val, hash, parent_node_name, parent_field_name, false)
+        end
+        return
       end
 
-      values_subhashes.uniq!(&:parts)
-      values_subhashes.sort_by!(&:parts)
-
-      values_subhashes.each do |subhash|
-        subhash.flush_to(hash)
+      postgres_node_name = node.class.name.split('::').last
+
+      node.to_h.keys.sort.each do |field_name|
+        val = node[field_name.to_s]
+
+        postgres_field_name = node_protobuf_field_name_to_json_name(node.class, field_name)
+
+        case postgres_field_name
+        when 'location'
+          next
+        when 'name'
+          next if [PrepareStmt, ExecuteStmt, DeallocateStmt].include?(node.class)
+          next if node.is_a?(ResTarget) && parent_node_name == 'SelectStmt' && parent_field_name == 'targetList'
+        when 'gid', 'options', 'savepoint_name'
+          next if node.is_a?(TransactionStmt)
+        when 'portalname'
+          next if [DeclareCursorStmt, FetchStmt, ClosePortalStmt].include?(node.class)
+        when 'relname'
+          next if node.is_a?(RangeVar) && node.relpersistence == 't'
+          if node.is_a?(RangeVar)
+            fingerprint_value(val.gsub(/\d{2,}/, ''), hash, postgres_node_name, postgres_field_name, true)
+            next
+          end
+        when 'stmt_len'
+          next if node.is_a?(RawStmt)
+        when 'stmt_location'
+          next if node.is_a?(RawStmt)
+        when 'kind'
+          if node.is_a?(A_Expr) && (val == :AEXPR_OP_ANY || val == :AEXPR_IN)
+            fingerprint_value(:AEXPR_OP, hash, postgres_node_name, postgres_field_name, true)
+            next
+          end
+        end
+
+        fingerprint_value(val, hash, postgres_node_name, postgres_field_name, true)
       end
-    else
-      values.each do |val|
-        fingerprint_value(val, hash, parent_node_name, parent_field_name, false)
+    end
+
+    def fingerprint_list(values, hash, parent_node_name, parent_field_name)
+      if %w[fromClause targetList cols rexpr valuesLists args].include?(parent_field_name)
+        values_subhashes = values.map do |val|
+          subhash = FingerprintSubHash.new
+          fingerprint_value(val, subhash, parent_node_name, parent_field_name, false)
+          subhash
+        end
+
+        values_subhashes.uniq!(&:parts)
+        values_subhashes.sort_by! { |s| PgQuery.hash_xxh3_64(s.parts.join, FINGERPRINT_VERSION) }
+
+        values_subhashes.each do |subhash|
+          subhash.flush_to(hash)
+        end
+      else
+        values.each do |val|
+          fingerprint_value(val, hash, parent_node_name, parent_field_name, false)
+        end
       end
     end
-  end
 
-  def fingerprint_tree(hash)
-    @tree.each do |node|
-      fingerprint_node(node, hash)
+    def fingerprint_tree(hash)
+      @tree.stmts.each do |node|
+        hash.update 'RawStmt'
+        fingerprint_node(node, hash)
+      end
     end
   end
 end

data/lib/pg_query/node.rb

@@ -0,0 +1,31 @@
+module PgQuery
+  # Patch the auto-generated generic node type with additional convenience functions
+  class Node
+    def inspect
+      node ? format('<PgQuery::Node: %s: %s>', node, public_send(node).inspect) : '<PgQuery::Node>'
+    end
+
+    # Make it easier to initialize nodes from a given node child object
+    def self.from(node_field_val)
+      # This needs to match libpg_query naming for the Node message field names
+      # (see "underscore" method in libpg_query's scripts/generate_protobuf_and_funcs.rb)
+      node_field_name = node_field_val.class.name.split('::').last
+      node_field_name.gsub!(/^([A-Z\d])([A-Z][a-z])/, '\1__\2')
+      node_field_name.gsub!(/([A-Z\d]+[a-z]+)([A-Z][a-z])/, '\1_\2')
+      node_field_name.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+      node_field_name.tr!('-', '_')
+      node_field_name.downcase!
+
+      PgQuery::Node.new(node_field_name => node_field_val)
+    end
+
+    # Make it easier to initialize value nodes
+    def self.from_string(str)
+      PgQuery::Node.new(string: PgQuery::String.new(str: str))
+    end
+
+    def self.from_integer(ival)
+      PgQuery::Node.new(integer: PgQuery::Integer.new(ival: ival))
+    end
+  end
+end