pg_query 0.5.0 → 0.6.0

data/lib/pg_query.rb

@@ -3,6 +3,10 @@ require 'pg_query/parse_error'
 
 require 'pg_query/pg_query'
 require 'pg_query/parse'
+require 'pg_query/treewalker'
+
 require 'pg_query/filter_columns'
 require 'pg_query/fingerprint'
 require 'pg_query/param_refs'
+require 'pg_query/deparse'
+require 'pg_query/truncate'

data/lib/pg_query/filter_columns.rb

@@ -3,7 +3,7 @@ class PgQuery
   # target list, but includes things like JOIN condition and WHERE clause.
   #
   # Note: This also traverses into sub-selects.
-  def filter_columns
+  def filter_columns # rubocop:disable Metrics/CyclomaticComplexity
     load_tables_and_aliases! if @aliases.nil?
 
     # Get condition items from the parsetree
@@ -11,53 +11,63 @@ class PgQuery
     condition_items = []
     filter_columns = []
     loop do
-      if statement = statements.shift
-        if statement["SELECT"]
-          if statement["SELECT"]["op"] == 0
-            if statement["SELECT"]["fromClause"]
+      statement = statements.shift
+      if statement
+        if statement['SELECT']
+          if statement['SELECT']['op'] == 0
+            if statement['SELECT']['fromClause']
               # FROM subselects
-              statement["SELECT"]["fromClause"].each do |item|
-                statements << item["RANGESUBSELECT"]["subquery"] if item["RANGESUBSELECT"]
+              statement['SELECT']['fromClause'].each do |item|
+                next unless item['RANGESUBSELECT']
+                statements << item['RANGESUBSELECT']['subquery']
               end
 
               # JOIN ON conditions
-              condition_items += conditions_from_join_clauses(statement["SELECT"]["fromClause"])
+              condition_items += conditions_from_join_clauses(statement['SELECT']['fromClause'])
             end
 
             # WHERE clause
-            condition_items << statement["SELECT"]["whereClause"] if statement["SELECT"]["whereClause"]
-          elsif statement["SELECT"]["op"] == 1
-            statements << statement["SELECT"]["larg"] if statement["SELECT"]["larg"]
-            statements << statement["SELECT"]["rarg"] if statement["SELECT"]["rarg"]
+            condition_items << statement['SELECT']['whereClause'] if statement['SELECT']['whereClause']
+
+            # CTEs
+            if statement['SELECT']['withClause']
+              statement['SELECT']['withClause']['WITHCLAUSE']['ctes'].each do |item|
+                statements << item['COMMONTABLEEXPR']['ctequery'] if item['COMMONTABLEEXPR']
+              end
+            end
+          elsif statement['SELECT']['op'] == 1
+            statements << statement['SELECT']['larg'] if statement['SELECT']['larg']
+            statements << statement['SELECT']['rarg'] if statement['SELECT']['rarg']
           end
-        elsif statement["UPDATE"]
-          condition_items << statement["UPDATE"]["whereClause"] if statement["UPDATE"]["whereClause"]
-        elsif statement["DELETE FROM"]
-          condition_items << statement["DELETE FROM"]["whereClause"] if statement["DELETE FROM"]["whereClause"]
+        elsif statement['UPDATE']
+          condition_items << statement['UPDATE']['whereClause'] if statement['UPDATE']['whereClause']
+        elsif statement['DELETE FROM']
+          condition_items << statement['DELETE FROM']['whereClause'] if statement['DELETE FROM']['whereClause']
         end
       end
 
       # Process both JOIN and WHERE conditions here
-      if next_item = condition_items.shift
-        if next_item.keys[0].start_with?("AEXPR") || next_item["ANY"]
-          ["lexpr", "rexpr"].each do |side|
-            next unless expr = next_item.values[0][side]
-            next unless expr.is_a?(Hash)
+      next_item = condition_items.shift
+      if next_item
+        if next_item.keys[0].start_with?('AEXPR') || next_item['ANY']
+          %w(lexpr rexpr).each do |side|
+            expr = next_item.values[0][side]
+            next unless expr && expr.is_a?(Hash)
             condition_items << expr
           end
-        elsif next_item["ROW"]
-          condition_items += next_item["ROW"]["args"]
-        elsif next_item["COLUMNREF"]
-          column, table = next_item["COLUMNREF"]["fields"].reverse
+        elsif next_item['ROW']
+          condition_items += next_item['ROW']['args']
+        elsif next_item['COLUMNREF']
+          column, table = next_item['COLUMNREF']['fields'].reverse
           filter_columns << [@aliases[table] || table, column]
-        elsif next_item["NULLTEST"]
-          condition_items << next_item["NULLTEST"]["arg"]
-        elsif next_item["FUNCCALL"]
+        elsif next_item['NULLTEST']
+          condition_items << next_item['NULLTEST']['arg']
+        elsif next_item['FUNCCALL']
           # FIXME: This should actually be extracted as a funccall and be compared with those indices
-          condition_items += next_item["FUNCCALL"]["args"] if next_item["FUNCCALL"]["args"]
-        elsif next_item["SUBLINK"]
-          condition_items << next_item["SUBLINK"]["testexpr"]
-          statements << next_item["SUBLINK"]["subselect"]
+          condition_items += next_item['FUNCCALL']['args'] if next_item['FUNCCALL']['args']
+        elsif next_item['SUBLINK']
+          condition_items << next_item['SUBLINK']['testexpr']
+          statements << next_item['SUBLINK']['subselect']
         end
       end
 
@@ -67,18 +77,21 @@ class PgQuery
     filter_columns.uniq
   end
 
-protected
+  protected
+
   def conditions_from_join_clauses(from_clause)
     condition_items = []
     from_clause.each do |item|
-      next unless item["JOINEXPR"]
+      next unless item['JOINEXPR']
 
-      joinexpr_items = [item["JOINEXPR"]]
+      joinexpr_items = [item['JOINEXPR']]
       loop do
-        break unless next_item = joinexpr_items.shift
-        condition_items << next_item["quals"] if next_item["quals"]
-        ["larg", "rarg"].each do |side|
-          joinexpr_items << next_item[side]["JOINEXPR"] if next_item[side]["JOINEXPR"]
+        next_item = joinexpr_items.shift
+        break unless next_item
+        condition_items << next_item['quals'] if next_item['quals']
+        %w(larg rarg).each do |side|
+          next unless next_item[side]['JOINEXPR']
+          joinexpr_items << next_item[side]['JOINEXPR']
         end
       end
     end

data/lib/pg_query/fingerprint.rb

@@ -1,62 +1,37 @@
 require 'digest'
 
 class PgQuery
-  def fingerprint
+  def fingerprint # rubocop:disable Metrics/CyclomaticComplexity
     normalized_parsetree = deep_dup(parsetree)
-    exprs = normalized_parsetree.dup
-    loop do
-      expr = exprs.shift
 
-      if expr.is_a?(Hash)
-        expr.each do |k,v|
-          if v.is_a?(Hash) && ["A_CONST", "ALIAS", "PARAMREF"].include?(v.keys[0])
-            # Remove constants, aliases and param references from tree
-            expr[k] = nil
-          elsif k == "location"
-            # Remove location info in order to ignore whitespace and target list ordering
-            expr.delete(k)
-          elsif !v.nil?
-            # Remove SELECT target list names & ignore order
-            if k == "targetList" && v.is_a?(Array)
-              v.each {|v| v["RESTARGET"]["name"] = nil if v["RESTARGET"] } # Remove names
-              v.sort_by! {|v| v.to_s }
-              expr[k] = v
-            end
-
-            # Ignore INSERT cols order
-            if k == "cols" && v.is_a?(Array)
-              v.sort_by! {|v| v.to_s }
-              expr[k] = v
-            end
-
-            # Process sub-expressions
-            exprs << v
-          end
-        end
-      elsif expr.is_a?(Array)
-        exprs += expr
+    # First delete all simple elements and attributes that can be removed
+    treewalker! normalized_parsetree do |expr, k, v|
+      if v.is_a?(Hash) && %w(A_CONST ALIAS PARAMREF).include?(v.keys[0])
+        # Remove constants, aliases and param references from tree
+        expr[k] = nil
+      elsif k == 'location'
+        # Remove location info in order to ignore whitespace and target list ordering
+        expr.delete(k)
       end
-
-      break if exprs.empty?
     end
 
-    Digest::SHA1.hexdigest(normalized_parsetree.to_s)
-  end
-
-  private
-
-  def deep_dup(obj)
-    case obj
-    when Hash
-      obj.each_with_object(obj.dup) do |(key, value), hash|
-        hash[deep_dup(key)] = deep_dup(value)
+    # Now remove all unnecessary info
+    treewalker! normalized_parsetree do |expr, k, v|
+      if k == 'AEXPR IN' && v.is_a?(Hash) && v['rexpr'].is_a?(Array)
+        # Compact identical IN list elements to one
+        v['rexpr'].uniq!
+      elsif k == 'targetList' && v.is_a?(Array)
+        # Remove SELECT target list names & ignore order
+        v.each { |v2| v2['RESTARGET']['name'] = nil if v2['RESTARGET'] } # Remove names
+        v.sort_by!(&:to_s)
+        expr[k] = v
+      elsif k == 'cols' && v.is_a?(Array)
+        # Ignore INSERT cols order
+        v.sort_by!(&:to_s)
+        expr[k] = v
       end
-    when Array
-      obj.map { |it| deep_dup(it) }
-    when NilClass, FalseClass, TrueClass, Symbol, Numeric
-      obj # Can't be duplicated
-    else
-      obj.dup
     end
+
+    Digest::SHA1.hexdigest(normalized_parsetree.to_s)
   end
 end

data/lib/pg_query/param_refs.rb

@@ -1,40 +1,45 @@
 class PgQuery
-  def param_refs
+  def param_refs # rubocop:disable Metrics/CyclomaticComplexity
     results = []
-    exprs = parsetree.dup
-    loop do
-      expr = exprs.shift
-
-      if expr.is_a?(Hash)
-        expr.each do |k,v|
-          if v.is_a?(Hash)
-            if v["PARAMREF"]
-              length = 1 # FIXME: Not true when we have actual paramrefs
-              results << {"location" => v["PARAMREF"]["location"], "length" => length}
-              next
-            elsif (p = v["TYPECAST"]["arg"]["PARAMREF"] rescue false) && (t = v["TYPECAST"]["typeName"]["TYPENAME"] rescue false)
-              location = p["location"]
-              typeloc = t["location"]
-              typename = t["names"].join(".")
-              length = 1 # FIXME: Not true when we have actual paramrefs
-              if typeloc < location
-                length += location - typeloc
-                location = typeloc
-              end
-              results << {"location" => location, "length" => length, "typename" => typename}
-              next
-            end
-          end
-
-          exprs << v if !v.nil?
+
+    treewalker! parsetree do |_, _, v|
+      next unless v.is_a?(Hash)
+
+      if v['PARAMREF']
+        results << { 'location' => v['PARAMREF']['location'],
+                     'length' => param_ref_length(v['PARAMREF']) }
+      elsif v['TYPECAST']
+        next unless v['TYPECAST']['arg'] && v['TYPECAST']['typeName']
+
+        p = v['TYPECAST']['arg'].delete('PARAMREF')
+        t = v['TYPECAST']['typeName'].delete('TYPENAME')
+        next unless p && t
+
+        location = p['location']
+        typeloc  = t['location']
+        typename = t['names'].join('.')
+        length   = param_ref_length(p)
+
+        if typeloc < location
+          length += location - typeloc
+          location = typeloc
         end
-      elsif expr.is_a?(Array)
-        exprs += expr
-      end
 
-      break if exprs.empty?
+        results << { 'location' => location, 'length' => length, 'typename' => typename }
+      end
     end
-    results.sort_by! {|r| r["location"] }
+
+    results.sort_by! { |r| r['location'] }
     results
   end
+
+  private
+
+  def param_ref_length(paramref_node)
+    if paramref_node['number'] == 0
+      1 # Actually a ? replacement character
+    else
+      ('$' + paramref_node['number'].to_s).size
+    end
+  end
 end

data/lib/pg_query/parse.rb

@@ -6,8 +6,8 @@ class PgQuery
 
     begin
       parsetree = JSON.parse(parsetree, max_nesting: 1000)
-    rescue JSON::ParserError => e
-      raise ParseError.new("Failed to parse JSON", -1)
+    rescue JSON::ParserError
+      raise ParseError.new('Failed to parse JSON', -1)
     end
 
     warnings = []
@@ -22,6 +22,7 @@ class PgQuery
   attr_reader :query
   attr_reader :parsetree
   attr_reader :warnings
+
   def initialize(query, parsetree, warnings = [])
     @query = query
     @parsetree = parsetree
@@ -38,8 +39,9 @@ class PgQuery
     @aliases
   end
 
-protected
-  def load_tables_and_aliases!
+  protected
+
+  def load_tables_and_aliases! # rubocop:disable Metrics/CyclomaticComplexity
     @tables = []
     @aliases = {}
 
@@ -48,32 +50,47 @@ protected
     where_clause_items = []
 
     loop do
-      if statement = statements.shift
+      statement = statements.shift
+      if statement
         case statement.keys[0]
-        when "SELECT"
-          if statement["SELECT"]["op"] == 0
-            (statement["SELECT"]["fromClause"] || []).each do |item|
-              if item["RANGESUBSELECT"]
-                statements << item["RANGESUBSELECT"]["subquery"]
+        when 'SELECT'
+          if statement['SELECT']['op'] == 0
+            (statement['SELECT']['fromClause'] || []).each do |item|
+              if item['RANGESUBSELECT']
+                statements << item['RANGESUBSELECT']['subquery']
               else
                 from_clause_items << item
               end
             end
-          elsif statement["SELECT"]["op"] == 1
-            statements << statement["SELECT"]["larg"] if statement["SELECT"]["larg"]
-            statements << statement["SELECT"]["rarg"] if statement["SELECT"]["rarg"]
+
+            # CTEs
+            if statement['SELECT']['withClause']
+              statement['SELECT']['withClause']['WITHCLAUSE']['ctes'].each do |item|
+                statements << item['COMMONTABLEEXPR']['ctequery'] if item['COMMONTABLEEXPR']
+              end
+            end
+          elsif statement['SELECT']['op'] == 1
+            statements << statement['SELECT']['larg'] if statement['SELECT']['larg']
+            statements << statement['SELECT']['rarg'] if statement['SELECT']['rarg']
+          end
+        when 'INSERT INTO', 'UPDATE', 'DELETE FROM', 'VACUUM', 'COPY', 'ALTER TABLE', 'CREATESTMT', 'INDEXSTMT', 'RULESTMT', 'CREATETRIGSTMT'
+          from_clause_items << statement.values[0]['relation']
+        when 'VIEWSTMT'
+          from_clause_items << statement['VIEWSTMT']['view']
+          statements << statement['VIEWSTMT']['query']
+        when 'REFRESHMATVIEWSTMT'
+          from_clause_items << statement['REFRESHMATVIEWSTMT']['relation']
+        when 'EXPLAIN'
+          statements << statement['EXPLAIN']['query']
+        when 'CREATE TABLE AS'
+          if statement['CREATE TABLE AS']['into'] && statement['CREATE TABLE AS']['into']['INTOCLAUSE']['rel']
+            from_clause_items << statement['CREATE TABLE AS']['into']['INTOCLAUSE']['rel']
           end
-        when "INSERT INTO", "UPDATE", "DELETE FROM", "VACUUM", "COPY", "ALTER TABLE", "CREATESTMT", "INDEXSTMT", "RULESTMT", "CREATETRIGSTMT"
-          from_clause_items << statement.values[0]["relation"]
-        when "EXPLAIN", "VIEWSTMT"
-          statements << statement.values[0]["query"]
-        when "CREATE TABLE AS"
-          from_clause_items << statement["CREATE TABLE AS"]["into"]["INTOCLAUSE"]["rel"] rescue nil
-        when "LOCK", "TRUNCATE"
-          from_clause_items += statement.values[0]["relations"]
-        when "GRANTSTMT"
-          objects = statement["GRANTSTMT"]["objects"]
-          case statement["GRANTSTMT"]["objtype"]
+        when 'LOCK', 'TRUNCATE'
+          from_clause_items += statement.values[0]['relations']
+        when 'GRANTSTMT'
+          objects = statement['GRANTSTMT']['objects']
+          case statement['GRANTSTMT']['objtype']
           when 0 # Column
             # FIXME
           when 1 # Table
@@ -81,27 +98,29 @@ protected
           when 2 # Sequence
             # FIXME
           end
-        when "DROP"
-          objects = statement["DROP"]["objects"]
-          case statement["DROP"]["removeType"]
+        when 'DROP'
+          objects = statement['DROP']['objects']
+          case statement['DROP']['removeType']
           when 26 # Table
-            @tables += objects.map {|r| r.join('.') }
+            @tables += objects.map { |r| r.join('.') }
           when 23 # Rule
-            @tables += objects.map {|r| r[0..-2].join('.') }
+            @tables += objects.map { |r| r[0..-2].join('.') }
           when 28 # Trigger
-            @tables += objects.map {|r| r[0..-2].join('.') }
+            @tables += objects.map { |r| r[0..-2].join('.') }
           end
         end
 
-        where_clause_items << statement.values[0]["whereClause"] if !statement.empty? && statement.values[0]["whereClause"]
+        where_clause_items << statement.values[0]['whereClause'] if !statement.empty? && statement.values[0]['whereClause']
       end
 
       # Find subselects in WHERE clause
-      if next_item = where_clause_items.shift
+      next_item = where_clause_items.shift
+      if next_item
         case next_item.keys[0]
         when /^AEXPR/, 'ANY'
-          ["lexpr", "rexpr"].each do |side|
-            next unless elem = next_item.values[0][side]
+          %w(lexpr rexpr).each do |side|
+            elem = next_item.values[0][side]
+            next unless elem
             if elem.is_a?(Array)
               where_clause_items += elem
             else
@@ -109,7 +128,7 @@ protected
             end
           end
         when 'SUBLINK'
-          statements << next_item["SUBLINK"]["subselect"]
+          statements << next_item['SUBLINK']['subselect']
         end
       end
 
@@ -117,20 +136,21 @@ protected
     end
 
     loop do
-      break unless next_item = from_clause_items.shift
+      next_item = from_clause_items.shift
+      break unless next_item
 
       case next_item.keys[0]
-      when "JOINEXPR"
-        ["larg", "rarg"].each do |side|
-          from_clause_items << next_item["JOINEXPR"][side]
+      when 'JOINEXPR'
+        %w(larg rarg).each do |side|
+          from_clause_items << next_item['JOINEXPR'][side]
         end
-      when "ROW"
-        from_clause_items += next_item["ROW"]["args"]
-      when "RANGEVAR"
-        rangevar = next_item["RANGEVAR"]
-        table = [rangevar["schemaname"], rangevar["relname"]].compact.join('.')
+      when 'ROW'
+        from_clause_items += next_item['ROW']['args']
+      when 'RANGEVAR'
+        rangevar = next_item['RANGEVAR']
+        table = [rangevar['schemaname'], rangevar['relname']].compact.join('.')
         @tables << table
-        @aliases[rangevar["alias"]["ALIAS"]["aliasname"]] = table if rangevar["alias"]
+        @aliases[rangevar['alias']['ALIAS']['aliasname']] = table if rangevar['alias']
       end
     end