human-ql 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/History.rdoc +9 -0
- data/README.rdoc +1 -1
- data/lib/human-ql.rb +1 -1
- data/lib/human-ql/base.rb +2 -2
- data/lib/human-ql/postgresql_custom_parser.rb +43 -12
- data/lib/human-ql/postgresql_generator.rb +1 -1
- data/lib/human-ql/query_generator.rb +1 -1
- data/lib/human-ql/query_parser.rb +1 -1
- data/lib/human-ql/tree_normalizer.rb +1 -1
- data/test/setup.rb +1 -1
- data/test/test_postgresql_fuzz.rb +1 -1
- data/test/test_postgresql_generator.rb +8 -6
- data/test/test_query_generator.rb +1 -1
- data/test/test_query_parser.rb +1 -1
- data/test/test_tree_normalizer.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 51ffc03488bc280dbdb361e86b029a56c6deac74
|
|
4
|
+
data.tar.gz: d02aac690117c9849b08a89558dece5c06db8a6c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c5f2be15113a044056f332b114c8ce768e71c3a3db4699812df2b46334446a68ceaec2dbd081044f707cf208b4722c565bd2bfaa0ccf24b7601bb65be0ee7679
|
|
7
|
+
data.tar.gz: c4a773aedd6a5aeb7a5517115f1a266be27edab6a1741db739cfcc7585bda4594b30f824e4f19e8c2b8d54c3beb61038933bfa580a1da83d86d95406bbf139c7
|
data/History.rdoc
CHANGED
|
@@ -1,2 +1,11 @@
|
|
|
1
|
+
=== 1.0.1 (2017-3-21)
|
|
2
|
+
|
|
3
|
+
* Drop more tokens, keep more characters in PostgreSQLCustomParser,
|
|
4
|
+
which originally treated more of the PG special (unsafe) characters
|
|
5
|
+
as whitespace. A result of these changes is that its less likely to
|
|
6
|
+
see replacement characters in generated output.
|
|
7
|
+
|
|
8
|
+
* Account for Postgresql 9.6.2 change in tests
|
|
9
|
+
|
|
1
10
|
=== 1.0.0 (2016-11-8)
|
|
2
11
|
* Initial release.
|
data/README.rdoc
CHANGED
|
@@ -44,7 +44,7 @@ Other generators are possible.
|
|
|
44
44
|
|
|
45
45
|
== License
|
|
46
46
|
|
|
47
|
-
Copyright (c) 2016 David Kellum
|
|
47
|
+
Copyright (c) 2016-2017 David Kellum
|
|
48
48
|
|
|
49
49
|
Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
50
50
|
may not use this file except in compliance with the License. You
|
data/lib/human-ql.rb
CHANGED
data/lib/human-ql/base.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#--
|
|
2
|
-
# Copyright (c) 2016 David Kellum
|
|
2
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
5
5
|
# may not use this file except in compliance with the License. You may
|
|
@@ -16,6 +16,6 @@
|
|
|
16
16
|
|
|
17
17
|
module HumanQL
|
|
18
18
|
|
|
19
|
-
VERSION='1.0.
|
|
19
|
+
VERSION='1.0.1'
|
|
20
20
|
|
|
21
21
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
1
3
|
#--
|
|
2
|
-
# Copyright (c) 2016 David Kellum
|
|
4
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
3
5
|
#
|
|
4
6
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
5
7
|
# may not use this file except in compliance with the License. You may
|
|
@@ -19,11 +21,17 @@ require 'human-ql/query_parser'
|
|
|
19
21
|
module HumanQL
|
|
20
22
|
|
|
21
23
|
# Extends the generic QueryParser with additional special character
|
|
22
|
-
# filtering so as to avoid syntax errors
|
|
23
|
-
# for any known input. Note that this is
|
|
24
|
-
# HumanQL query language, not anything
|
|
24
|
+
# and token filtering so as to avoid syntax errors with to_tsquery()
|
|
25
|
+
# (via PostgreSQLGenerator) for any known input. Note that this is
|
|
26
|
+
# still a parser for the HumanQL query language, not anything
|
|
27
|
+
# implemented in PostgreSQL.
|
|
25
28
|
class PostgreSQLCustomParser < QueryParser
|
|
26
29
|
|
|
30
|
+
# U+2019 RIGHT SINGLE QUOTATION MARK
|
|
31
|
+
RSQMARK = '’'.freeze
|
|
32
|
+
|
|
33
|
+
UNDERSCORE = '_'.freeze
|
|
34
|
+
|
|
27
35
|
# Construct given options to set via base clase or as specified
|
|
28
36
|
# below.
|
|
29
37
|
#
|
|
@@ -32,7 +40,7 @@ module HumanQL
|
|
|
32
40
|
# :pg_version:: A version string ("9.5.5", "9.6.1") or integer
|
|
33
41
|
# array ( [9,6,1]) indicating the target PostgreSQL
|
|
34
42
|
# version. Phrase support starts in 9.6 so quoted
|
|
35
|
-
# phrases are ignored before that.
|
|
43
|
+
# phrases are ignored before that. Default: < 9.6
|
|
36
44
|
#
|
|
37
45
|
def initialize(opts = {})
|
|
38
46
|
opts = opts.dup
|
|
@@ -46,21 +54,44 @@ module HumanQL
|
|
|
46
54
|
|
|
47
55
|
# Phrase support starts in 9.6
|
|
48
56
|
if ( pg_version <=> [9,6] ) >= 0
|
|
49
|
-
|
|
57
|
+
# Handle what PG-sensitive chracters we can early as
|
|
58
|
+
# whitespace. This can't include anything part of HumanQL,
|
|
59
|
+
# e.g. ':' as used for scopes, so deal with the remainder
|
|
60
|
+
# below.
|
|
61
|
+
self.spaces = /[[:space:]*!<>]+/.freeze
|
|
50
62
|
else
|
|
51
|
-
# Disable quote tokens
|
|
63
|
+
# Disable quote tokens
|
|
52
64
|
self.lquote = nil
|
|
53
65
|
self.rquote = nil
|
|
54
|
-
|
|
66
|
+
# As above but add DQUOTE as well.
|
|
67
|
+
self.spaces = /[[:space:]*!<>"]+/.freeze
|
|
55
68
|
end
|
|
56
69
|
|
|
70
|
+
# Use by custom #norm_phrase_tokens as a superset of the
|
|
71
|
+
# #lparen, #rparen token patterns removed by default. In
|
|
72
|
+
# PostgreSQL, the '|' and '&' still need to be filtered. Other
|
|
73
|
+
# freestanding punctuation tokens are best removed entirely.
|
|
74
|
+
@phrase_token_rejects = /\A[()|&':]\z/.freeze
|
|
75
|
+
|
|
76
|
+
# SQUOTE is a problem only when at beginning of term.
|
|
77
|
+
@lead_squote = /\A'/.freeze
|
|
78
|
+
|
|
79
|
+
# COLON is always a problem, but since its also part of Human QL
|
|
80
|
+
# (scopes) it can't be included earlier in #spaces. Also per
|
|
81
|
+
# scope parsing rules, its not always made a separate token.
|
|
82
|
+
@term_rejects = /:/.freeze
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def norm_phrase_tokens( tokens )
|
|
86
|
+
tokens.
|
|
87
|
+
reject { |t| @phrase_token_rejects === t }.
|
|
88
|
+
map { |t| norm_term( t ) }
|
|
57
89
|
end
|
|
58
90
|
|
|
59
|
-
# Replace
|
|
60
|
-
# (or effectively, whitespace) in tsquery with tested
|
|
61
|
-
# dictionaries.
|
|
91
|
+
# Replace various problem single characters with alt. characters.
|
|
62
92
|
def norm_term( t )
|
|
63
|
-
t.
|
|
93
|
+
t.sub( @lead_squote, RSQMARK ).
|
|
94
|
+
gsub( @term_rejects, UNDERSCORE )
|
|
64
95
|
end
|
|
65
96
|
end
|
|
66
97
|
|
data/test/setup.rb
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# coding: utf-8
|
|
2
3
|
|
|
3
4
|
#--
|
|
4
|
-
# Copyright (c) 2016 David Kellum
|
|
5
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
5
6
|
#
|
|
6
7
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
7
8
|
# may not use this file except in compliance with the License. You may
|
|
@@ -82,7 +83,7 @@ class TestPostgresqlGenerator < Minitest::Test
|
|
|
82
83
|
|
|
83
84
|
def test_phrase_with_danger
|
|
84
85
|
skip( "For postgresql 9.6+" ) unless pg_gte_9_6?
|
|
85
|
-
assert_gen( '
|
|
86
|
+
assert_gen( 'boy', '": boy"' )
|
|
86
87
|
assert_tsq( "'boy'", '": boy"' )
|
|
87
88
|
end
|
|
88
89
|
|
|
@@ -147,15 +148,16 @@ class TestPostgresqlGenerator < Minitest::Test
|
|
|
147
148
|
end
|
|
148
149
|
|
|
149
150
|
def test_funk_1
|
|
150
|
-
assert_gen( "!(, &
|
|
151
|
+
assert_gen( "!(, & ’y)", "-( , 'y -)" )
|
|
151
152
|
assert_tsq( "!'y'", "-( ,'y -)" )
|
|
152
|
-
|
|
153
|
-
assert_gen( "!(, & _y) & c3", "|-( , 'y -)c3" )
|
|
153
|
+
assert_gen( "!(, & ’y) & c3", "|-( , 'y -)c3" )
|
|
154
154
|
assert_tsq( "!'y' & 'c3'", "|-( ,'y -)c3" )
|
|
155
155
|
end
|
|
156
156
|
|
|
157
157
|
def test_funk_2
|
|
158
|
-
if
|
|
158
|
+
if ( PG_VERSION <=> [9,6,2] ) >= 0
|
|
159
|
+
assert_tsq( "!!'boy' & 'cat'", "-(a -boy) & cat" )
|
|
160
|
+
elsif pg_gte_9_6?
|
|
159
161
|
# Crashes PG 9.6 beta 1-2, fixed in beta 3.
|
|
160
162
|
assert_tsq( "'boy' & 'cat'", "-(a -boy) & cat" )
|
|
161
163
|
else
|
data/test/test_query_parser.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: human-ql
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Kellum
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-03-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: minitest
|
|
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
133
133
|
version: '0'
|
|
134
134
|
requirements: []
|
|
135
135
|
rubyforge_project:
|
|
136
|
-
rubygems_version: 2.6.
|
|
136
|
+
rubygems_version: 2.6.10
|
|
137
137
|
signing_key:
|
|
138
138
|
specification_version: 4
|
|
139
139
|
summary: Human Query Language for full text search engines. Provides a lenient parser
|