human-ql 1.0.0

data/lib/human-ql/tree_normalizer.rb

@@ -0,0 +1,226 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module HumanQL
+
+  # Normalizes and imposes various limitations on query abstract
+  # syntax trees (ASTs).
+  class TreeNormalizer
+
+    # Allow nested scopes?
+    # Default: false -> nested scope nodes are removed.
+    attr_accessor :nested_scope
+
+    # Allow nested :not (in other words, double negatives)?
+    # Default: false -> nested :not nodes are removed.
+    attr_accessor :nested_not
+
+    # Allow unconstrained :not?
+    # Queries containing an unsconstrained :not may be costly to
+    # execute. If false the unconstrained :not will be removed.
+    #
+    # A :not node is considered "constrained" if it has an :and
+    # ancestor with at least one contraint argument. A constraint
+    # argument is a term, phrase, or :and node matching this same
+    # criteria, or an :or node where *all* arguments match this
+    # criteria. See also #scope_can_constrain.
+    # Default: true
+    attr_accessor :unconstrained_not
+
+    # Does a scope count as a constraint?
+    # Default: true -> a scope is a constraint if its argument is a
+    # constraint.
+    # If it depends on the scope, you can override
+    # #scope_can_constrain? with this logic.
+    attr_accessor :scope_can_constrain
+
+    # Allow SCOPE within :not?
+    # * If set to `:invert` normalizes `[:not, ['SCOPE', 'a']]` to
+    #   `['SCOPE', [:not, 'a']]`.
+    # * If set to `false`, the nested scope node is removed.
+    # * For either :invert or false, the scope node is otherwise
+    #   removed if found below a :not node.
+    # Default: :invert
+    attr_accessor :not_scope
+
+    # Allow only scopes combined with :and condition?
+    # Default: false
+    attr_accessor :scope_and_only
+
+    # Allow scope at root or first level only?
+    # Default: false
+    attr_accessor :scope_at_top_only
+
+    # Construct given options that are applied via same name setters
+    # on self.
+    def initialize( opts = {} )
+      @nested_scope = false
+      @nested_not = false
+      @unconstrained_not = true
+      @scope_can_constrain = true
+      @scope_at_top_only = false
+      @scope_and_only = false
+      @not_scope = :invert
+
+      opts.each do |name,val|
+        send( name.to_s + '=', val )
+      end
+    end
+
+    # Return a new normalized AST from the given AST root node.
+    def normalize( node )
+      node = normalize_1( node, EMPTY_STACK, @unconstrained_not )
+      if ( @not_scope != true ) || @scope_and_only || @scope_at_top_only
+        node = normalize_2( node, EMPTY_STACK )
+      end
+      node
+    end
+
+    protected
+
+    def scope_can_constrain?( scope )
+      @scope_can_constrain
+    end
+
+    EMPTY_STACK = [].freeze
+
+    # Return true if node is a valid constraint
+    def constraint?( node )
+      op,*args = node
+      if ! node.is_a?( Array )
+        true
+      elsif args.empty?
+        false
+      else
+        case op
+        when :and
+          args.any? { |a| constraint?( a ) }
+        when :or
+          args.all? { |a| constraint?( a ) }
+        when :phrase
+          true
+        when String
+          scope_can_constrain?( op ) && constraint?( args.first )
+        else
+          false
+        end
+      end
+    end
+
+    private
+
+    def normalize_1( node, ops, constrained )
+      op,*args = node
+      if ! node.is_a?( Array )
+        op
+      elsif args.empty?
+        nil
+      else
+
+        case op
+        when :and
+          unless constrained
+            constrained = args.any? { |a| constraint?( a ) }
+          end
+        when String #scope
+          args = args[0,1] if args.length > 1
+          if !@nested_scope
+            outer = ops.find { |o| o.is_a?( String ) }
+            if outer == op
+              return args.first
+            elsif outer
+              return nil
+            end
+          end
+        when :not
+          args = args[0,1] if args.length > 1
+          return nil if !constrained || ( !@nested_not && ops.rindex(:not) )
+        end
+
+        a_ops = ops.dup.push( op )
+        out = []
+        args.each do |a|
+          a = normalize_1( a, a_ops, constrained )
+          if a.is_a?( Array ) && a[0] == op
+            out += a[1..-1]
+          elsif a # filter nil
+            out << a
+          end
+        end
+
+        if ( op == :and || op == :or ) && out.length < 2
+          out[0]
+        elsif out.empty?
+          nil
+        else
+          out.unshift( op )
+        end
+      end
+    end
+
+    def normalize_2( node, ops )
+      op,*args = node
+      if ! node.is_a?( Array )
+        op
+      elsif args.empty?
+        nil
+      else
+        case op
+        when String #scope
+          if @scope_at_top_only && ops.length > 1
+            return nil
+          end
+          if @scope_and_only && !ops.all? { |o| o == :and }
+            return nil
+          end
+        when :not
+          if @not_scope == :invert
+            na = args[0]
+            if na.is_a?( Array ) && na[0].is_a?( String )
+              op, na[0] = na[0], op
+              return [ op, na ]
+            end
+          end
+        end
+
+        a_ops = ops.dup.push( op )
+        out = []
+        args.each do |a|
+          a = normalize_2( a, a_ops )
+          if a.is_a?( Array ) && a[0] == op
+            out += a[1..-1]
+          elsif a # filter nil
+            out << a
+          end
+        end
+
+        if ( op == :and || op == :or ) && out.length < 2
+          out[0]
+        elsif out.empty?
+          nil
+        # If scope still found below a :not, remove it. With :invert,
+        # this implies nodes intervening
+        elsif @not_scope != true && op.is_a?( String ) && ops.rindex(:not)
+          nil
+        else
+          out.unshift( op )
+        end
+      end
+    end
+
+  end
+
+end

data/test/setup.rb

@@ -0,0 +1,24 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rubygems'
+require 'bundler/setup'
+require 'minitest/autorun'
+
+unless RUBY_PLATFORM =~ /java/
+  require 'pg'
+  require 'sequel'
+end

data/test/test_postgresql_fuzz.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require_relative 'setup.rb'
+
+require 'human-ql/postgresql_custom_parser'
+require 'human-ql/postgresql_generator'
+require 'human-ql/tree_normalizer'
+
+class TestPostgresqlFuzz < Minitest::Test
+  DB = Sequel.connect( "postgres://localhost/human_ql_test" )
+
+  PG_VERSION =
+    begin
+      v = DB &&
+          DB["select current_setting('server_version') as v"].first[:v]
+      v &&= v.split('.').map(&:to_i)
+      v || []
+    end
+
+  TC = HumanQL::PostgreSQLCustomParser.new( pg_version: PG_VERSION )
+  DN = HumanQL::TreeNormalizer.new
+
+  PG = HumanQL::PostgreSQLGenerator.new
+
+  PASSES = if $0 == __FILE__
+             100
+           else
+             1
+           end
+
+  # Assert that parsing via PG to_tsquery(generated) doesn't fail
+  def assert_pg_parse( hq )
+    ast = TC.parse( hq )
+    ast = DN.normalize( ast )
+    if ast
+      pg = PG.generate( ast )
+      begin
+        rt = DB["select to_tsquery(?) as tsquery", pg].first[:tsquery]
+        refute_nil( rt, hq )
+      rescue Sequel::DatabaseError => e
+        fail( "On query #{hq.inspect} -> #{ast.inspect}: #{ e.to_s }" )
+      end
+    else
+      pass
+    end
+  end
+
+  # Starting point query
+  GENERIC_Q = 'ape | ( boy -"cat dog" )'.freeze
+
+  # Characters which are likely to cause trouble
+  RANDOM_C = '({"\'a !:* ,^#:/-0.123e-9)<>'.freeze
+
+  PASSES.times do |i|
+    define_method( "test_fuzz_#{i}" ) do
+      1000.times do
+        s = rand( GENERIC_Q.length )
+        l = rand( GENERIC_Q.length * 2 )
+        q = GENERIC_Q[s,l]
+        20.times do
+          if rand(3) == 1
+            q[rand(q.length+1)] = fuzz
+          end
+        end
+        assert_pg_parse( q )
+      end
+    end
+  end
+
+  def fuzz
+    RANDOM_C[rand(RANDOM_C.length)]
+  end
+
+end if defined?( ::Sequel )

data/test/test_postgresql_generator.rb

@@ -0,0 +1,171 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require_relative 'setup.rb'
+
+require 'human-ql/postgresql_custom_parser'
+require 'human-ql/postgresql_generator'
+
+class TestPostgresqlGenerator < Minitest::Test
+  DB = if defined?( ::Sequel )
+         Sequel.connect( "postgres://localhost/human_ql_test" )
+       end
+
+  PG_VERSION =
+    begin
+      v = DB &&
+          DB["select current_setting('server_version') as v"].first[:v]
+      v &&= v.split('.').map(&:to_i)
+      v || []
+    end
+
+  TC = HumanQL::PostgreSQLCustomParser.new( verbose: ARGV.include?('--verbose'),
+                                            pg_version: PG_VERSION )
+  PG = HumanQL::PostgreSQLGenerator.new
+
+  def pg_gte_9_6?
+    ( PG_VERSION <=> [9,6] ) >= 0
+  end
+
+  def assert_gen( expected_pg, hq )
+    ast = TC.parse( hq )
+    pg = PG.generate( ast )
+    assert_equal( expected_pg, pg, ast )
+  end
+
+  # Assert that the round-trip representation via PG
+  # to_tsquery(generated) (and back to text) doesn't error and is as
+  # expected.
+  def assert_tsq( expected, hq )
+    if DB
+      ast = TC.parse( hq )
+      pg = PG.generate( ast )
+      rt = DB["select to_tsquery(?) as tsquery", pg].first[:tsquery]
+      assert_equal( expected, rt, ast )
+    end
+  end
+
+  def test_gen_term
+    assert_gen( 'ape', 'ape' )
+    assert_tsq( "'ape'", 'ape' )
+  end
+
+  def test_gen_and
+    assert_gen( 'ape & boy', 'ape boy' )
+    assert_tsq( "'ape' & 'boy'", 'ape boy' )
+  end
+
+  def test_gen_phrase
+    if pg_gte_9_6?
+      assert_gen( 'ape <-> boy', '"ape boy"' )
+      assert_tsq( "'ape' <-> 'boy'", '"ape boy"' )
+    else
+      assert_gen( 'ape & boy', '"ape boy"' )
+      assert_tsq( "'ape' & 'boy'", '"ape boy"' )
+    end
+  end
+
+  def test_phrase_with_danger
+    skip( "For postgresql 9.6+" ) unless pg_gte_9_6?
+    assert_gen( '_ <-> boy', '": boy"' )
+    assert_tsq( "'boy'", '": boy"' )
+  end
+
+  def test_gen_empty
+    assert_gen( nil, '' )
+  end
+
+  def test_gen_not
+    assert_gen( '!ape', '-ape' )
+    assert_tsq( "!'ape'", '-ape' )
+  end
+
+  def test_gen_not_stop
+    assert_gen( '!the', '-the' )
+    assert_tsq( "", '-the' )
+  end
+
+  def test_gen_or
+    assert_gen( '(ape | boy)', 'ape|boy' )
+    assert_tsq( "'ape' | 'boy'", 'ape|boy' )
+  end
+
+  def test_gen_or_stop
+    assert_gen( '(the | boy)', 'the|boy' )
+    assert_tsq( "'boy'", 'the|boy' )
+  end
+
+  def test_gen_not_phrase
+    skip( "For postgresql 9.6+" ) unless pg_gte_9_6?
+    assert_gen( '!(ape <-> boy)', '-"ape boy"' )
+    assert_tsq( "!( 'ape' <-> 'boy' )", '-"ape boy"' )
+  end
+
+  def test_gen_precedence_1
+    assert_gen( '(ape | boy) & cat', 'ape | boy cat' )
+    assert_tsq( "( 'ape' | 'boy' ) & 'cat'", 'ape | boy cat' )
+  end
+
+  def test_gen_precedence_2
+    assert_gen( 'ape & (boy | cat)', 'ape boy | cat' )
+    assert_tsq( "'ape' & ( 'boy' | 'cat' )", 'ape boy | cat' )
+  end
+
+  def test_gen_precedence_3
+    assert_gen( '(ape | !boy) & cat', 'ape | - boy cat' )
+    assert_tsq( "( 'ape' | !'boy' ) & 'cat'", 'ape | - boy cat' )
+  end
+
+  def test_gen_precedence_4
+    assert_gen( '(!ape | boy) & cat', '-ape | boy cat' )
+    assert_tsq( "( !'ape' | 'boy' ) & 'cat'", '-ape | boy cat' )
+  end
+
+  def test_gen_precedence_5
+    assert_gen( '(ape | boy) & !cat', 'ape | boy -cat' )
+    assert_tsq( "( 'ape' | 'boy' ) & !'cat'", 'ape | boy -cat' )
+  end
+
+  def test_gen_precedence_6
+    assert_gen( '(ape | boy) & !cat & dog', 'ape | boy -cat dog' )
+    assert_tsq( "( 'ape' | 'boy' ) & !'cat' & 'dog'", 'ape | boy -cat dog' )
+  end
+
+  def test_funk_1
+    assert_gen( "!(, & _y)", "-( , 'y -)" )
+    assert_tsq( "!'y'",     "-( ,'y -)" )
+
+    assert_gen( "!(, & _y) & c3", "|-( , 'y -)c3" )
+    assert_tsq( "!'y' & 'c3'", "|-( ,'y -)c3" )
+  end
+
+  def test_funk_2
+    if pg_gte_9_6?
+      # Crashes PG 9.6 beta 1-2, fixed in beta 3.
+      assert_tsq( "'boy' & 'cat'", "-(a -boy) & cat" )
+    else
+      # PG 9.5 doesn't normalize away the double not
+      assert_tsq( "!( !'boy' ) & 'cat'", "-(a -boy) & cat" )
+    end
+  end
+
+  def test_gen_or_not
+    assert_tsq( "'ape' & ( !'boy' | !'cat' )", "ape & ( -boy | -cat )" )
+  end
+
+end