human-ql 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 25023c260b71ece677718cfc8aad134860ced434
+  data.tar.gz: f3d8d6e3fad057bd458292b384a3b2f0ed0c5595
+SHA512:
+  metadata.gz: 83492efda22be913723385c49689f65f86f1520b4f0a3c460ff9a57e565715ef71b727ceddff7756f451fdbf9c9bf0d143ea148798ec3bc26ddb2f9472140513
+  data.tar.gz: dcf2c5edb12cb6f2cc38aff7698f89472df203585f93898023f1f7f1c4f15af67a64e66ee75eb596202c6cbb3625c88039e1b5fbeda7a72ca1e41af6c914fd12

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2016-11-8)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,17 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+lib/human-ql/base.rb
+lib/human-ql.rb
+lib/human-ql/postgresql_custom_parser.rb
+lib/human-ql/postgresql_generator.rb
+lib/human-ql/query_generator.rb
+lib/human-ql/query_parser.rb
+lib/human-ql/tree_normalizer.rb
+test/setup.rb
+test/test_postgresql_fuzz.rb
+test/test_postgresql_generator.rb
+test/test_query_generator.rb
+test/test_query_parser.rb
+test/test_tree_normalizer.rb

data/README.rdoc

@@ -0,0 +1,59 @@
+= HumanQL
+
+* http://github.com/dekellum/human-ql
+* http://rdoc.gravitext.com/human-ql/
+
+== Description
+
+Human Query Language for full text search engines. Provides a lenient
+parser and associated tools for a self-contained and search-engine
+agnostic query language suitable for use by end users. Lenient in that
+is will produce a parse tree for any input, given a default operator
+and by generally ignoring any unparsable syntax. Suitable for use by
+end users in that it supports potentially several operator variants
+and a query language not unlike some major web search and other
+commercial search engines.
+
+The query language supports the following features at a high level:
+
+* Boolean operators: AND (infix), OR (infix), NOT (prefix) with an
+  implied default operator and precedence rules,
+  e.g. "boy or girl -infant"
+
+* Optional parenthesis for explicitly denoting precedence.
+
+* Quoted phrases (for proximity matching)
+
+* Declarable prefix scopes, e.g. "TITLE:(car or bike)"
+
+The main components are each highly customizable:
+
+HumanQL::QueryParser — Parses any arbitrary input string and outputs an
+Abstract Syntax Tree (AST)
+
+HumanQL::TreeNormalizer — Normalizes and imposes limits on an AST,
+e.g. avoids pathological queries.
+
+HumanQL::QueryGenerator — Given an AST, generates a Human Query
+Language string.
+
+HumanQL::PostgreSQLGenerator — Given an AST, generate strings suitable
+for passing to PostgreSQL's to_tsquery function.
+
+Other generators are possible.
+
+== License
+
+Copyright (c) 2016 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,14 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'bundler/setup'
+require 'rjack-tarpit'
+
+RJack::TarPit.new( 'human-ql' ).define_tasks
+
+desc "Upload RDOC to Amazon S3 (rdoc.gravitext.com/human-ql, Oregon)"
+task :publish_rdoc => [ :clean, :rerdoc ] do
+  sh <<-SH
+    aws s3 sync --acl public-read doc/ s3://rdoc.gravitext.com/human-ql/
+  SH
+end

data/lib/human-ql.rb

@@ -0,0 +1,17 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'human-ql/base'

data/lib/human-ql/base.rb

@@ -0,0 +1,21 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module HumanQL
+
+  VERSION='1.0.0'
+
+end

data/lib/human-ql/postgresql_custom_parser.rb

@@ -0,0 +1,67 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'human-ql/query_parser'
+
+module HumanQL
+
+  # Extends the generic QueryParser with additional special character
+  # filtering so as to avoid syntax errors in PostgreSQL to_tsquery()
+  # for any known input.  Note that this is still a parser for the
+  # HumanQL query language, not anything implemented in PostgreSQL.
+  class PostgreSQLCustomParser < QueryParser
+
+    # Construct given options to set via base clase or as specified
+    # below.
+    #
+    # === Options
+    #
+    # :pg_version:: A version string ("9.5.5", "9.6.1") or integer
+    #               array ( [9,6,1]) indicating the target PostgreSQL
+    #               version. Phrase support starts in 9.6 so quoted
+    #               phrases are ignored before that.
+    #
+    def initialize(opts = {})
+      opts = opts.dup
+      pg_version = opts.delete(:pg_version)
+      if pg_version.is_a?( String )
+        pg_version = pg_version.split( '.' ).map( &:to_i )
+      end
+      pg_version ||= []
+
+      super
+
+      # Phrase support starts in 9.6
+      if ( pg_version <=> [9,6] ) >= 0
+        @term_rejects = /[()|&:*!'<>]/
+      else
+        # Disable quote tokens and reject DQUOTE as token character
+        self.lquote = nil
+        self.rquote = nil
+        @term_rejects = /[()|&:*!'"<>]/
+      end
+
+    end
+
+    # Replace term_rejects characters with '_' which is punctuation
+    # (or effectively, whitespace) in tsquery with tested
+    # dictionaries.
+    def norm_term( t )
+      t.gsub( @term_rejects, '_' )
+    end
+  end
+
+end

data/lib/human-ql/postgresql_generator.rb

@@ -0,0 +1,83 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module HumanQL
+
+  # Generate query strings suitable for passing to PostgreSQL's
+  # to_tsquery function, from a HumanQL abstract syntax tree (AST).
+  #
+  # In order to guarantee valid output for any human input, the AST
+  # should be created using PostgreSQLCustomParser and normalized via
+  # TreeNormalizer (with minimal defaults).
+  #
+  # Any scope's provided in the parser should have been handled and
+  # stripped out of the AST, as PostgreSQL is not expected to have a
+  # direct equivalent in tsquery syntax.
+  class PostgreSQLGenerator
+
+    #--
+    # From https://www.postgresql.org/docs/9.6/static/datatype-textsearch.html
+    # > In the absence of parentheses, '!' (NOT) binds most tightly,
+    # > and '&' (AND) and '<->' (FOLLOWED BY) both bind more tightly
+    # > than | (OR).
+    #++
+
+    AND = ' & '.freeze
+    OR = ' | '.freeze
+    NOT = '!'.freeze
+    NEAR = ' <-> '.freeze
+
+    # Given the root node of the AST, return a string in PostgreSQL
+    # tsquery syntax.
+    def generate( node )
+      op,*args = node
+      if ! node.is_a?( Array )
+        op
+      elsif args.empty?
+        nil
+      else
+        case op
+        when :and
+          terms_join( args, AND )
+        when :or
+          pwrap( terms_join( args, OR ) )
+        when :not
+          if args[0].is_a?( Array )
+            NOT + pwrap( generate( args[0] ) )
+          else
+            NOT + args[0]
+          end
+        when :phrase
+          terms_join( args, NEAR )
+        else
+          raise "Unsupported op: #{node.inspect}"
+        end
+      end
+    end
+
+    protected
+
+    def terms_join( args, op )
+      args.map { |a| generate( a ) }.join( op )
+    end
+
+    def pwrap( inner )
+      '(' + inner + ')'
+    end
+
+  end
+
+end

data/lib/human-ql/query_generator.rb

@@ -0,0 +1,163 @@
+#--
+# Copyright (c) 2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module HumanQL
+
+  # Generate a Human Query Language String from an abstract syntax
+  # tree (AST). This allows query simplification (e.g. via
+  # TreeNormalizer) and re-writing queries.
+  class QueryGenerator
+
+    # The AND operator (if not default).
+    # Default: ' and '
+    attr_accessor :and
+
+    # The OR operator (if not default).
+    # Default: ' or '
+    attr_accessor :or
+
+    # The NOT operator.
+    # Default: '-'
+    attr_accessor :not
+
+    # SPACE delimiter.
+    # Default: ' '
+    attr_accessor :space
+
+    # Left quote character for phrases.
+    # Default: '"'
+    attr_accessor :lquote
+
+    # Right quote character for phrases.
+    # Default: '"'
+    attr_accessor :rquote
+
+    # COLON character used a prefix delimiter.
+    # Default: ':'
+    attr_accessor :colon
+
+    # Left parenthesis character.
+    # Default: '('
+    attr_accessor :lparen
+
+    # Right parenthesis character.
+    # Default: ')'
+    attr_accessor :rparen
+
+    # The default operator (:and or :or). If set, will output a :space
+    # instead of the operator.
+    # Default: nil
+    attr_accessor :default_op
+
+    # Hash of operators to precedence integer values, as per
+    # QueryParser#precedence. If set, outputs parentheses only when
+    # precedence dictates that it is necessary.
+    # Default: nil
+    attr_accessor :precedence
+
+    # Set #default_op and #precedence from the given QueryParser, as a
+    # convenience.
+    def parser=( p )
+      @default_op = p.default_op
+      @precedence = p.precedence
+    end
+
+    # Construct given options which are interpreted as attribute names
+    # to set.
+    def initialize( opts = {} )
+      @and = ' and '.freeze
+      @or = ' or '.freeze
+      @not = '-'.freeze
+      @space = ' '.freeze
+      @lquote = @rquote = '"'.freeze
+      @colon = ':'.freeze
+      @lparen = '('.freeze
+      @rparen = ')'.freeze
+      @default_op = nil
+      @precedence = nil
+
+      opts.each do |name,val|
+        send( name.to_s + '=', val )
+      end
+    end
+
+    # Given the root node of the AST, return a String in Human Query
+    # Language syntax.
+    def generate( node )
+      op,*args = node
+      if ! node.is_a?( Array )
+        op
+      elsif args.empty?
+        nil
+      else
+        case op
+        when :and
+          terms_join( args, :and )
+        when :or
+          terms_join( args, :or )
+        when :not
+          @not + pwrap_gen( args[0], op )
+        when :phrase
+          @lquote + args.join( @space ) + @rquote
+        when String
+          op + @colon + pwrap_gen( args[0], op )
+        else
+          raise "Unsupported op: #{node.inspect}"
+        end
+      end
+    end
+
+    protected
+
+    def terms_join( args, op )
+      args = args.map { |a| pwrap_gen( a, op ) }
+      if op == @default_op
+        args.join( @space )
+      elsif op == :and
+        args.join( @and )
+      elsif op == :or
+        args.join( @or )
+      end
+    end
+
+    def pwrap_gen( node, parent_op )
+      if node.is_a?( Array )
+        op = node[0]
+        if precedence_lte?( parent_op, op )
+          generate( node )
+        else
+          pwrap( generate( node ) )
+        end
+      else
+        node
+      end
+    end
+
+    def pwrap( inner )
+      @lparen + inner + @rparen
+    end
+
+    def precedence_lte?( op1, op2 )
+      if @precedence
+        @precedence[op1] <= @precedence[op2]
+      else
+        false
+      end
+    end
+
+  end
+
+end