human-ql 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.rdoc +9 -0
- data/README.rdoc +1 -1
- data/lib/human-ql.rb +1 -1
- data/lib/human-ql/base.rb +2 -2
- data/lib/human-ql/postgresql_custom_parser.rb +43 -12
- data/lib/human-ql/postgresql_generator.rb +1 -1
- data/lib/human-ql/query_generator.rb +1 -1
- data/lib/human-ql/query_parser.rb +1 -1
- data/lib/human-ql/tree_normalizer.rb +1 -1
- data/test/setup.rb +1 -1
- data/test/test_postgresql_fuzz.rb +1 -1
- data/test/test_postgresql_generator.rb +8 -6
- data/test/test_query_generator.rb +1 -1
- data/test/test_query_parser.rb +1 -1
- data/test/test_tree_normalizer.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 51ffc03488bc280dbdb361e86b029a56c6deac74
|
|
4
|
+
data.tar.gz: d02aac690117c9849b08a89558dece5c06db8a6c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c5f2be15113a044056f332b114c8ce768e71c3a3db4699812df2b46334446a68ceaec2dbd081044f707cf208b4722c565bd2bfaa0ccf24b7601bb65be0ee7679
|
|
7
|
+
data.tar.gz: c4a773aedd6a5aeb7a5517115f1a266be27edab6a1741db739cfcc7585bda4594b30f824e4f19e8c2b8d54c3beb61038933bfa580a1da83d86d95406bbf139c7
|
data/History.rdoc
CHANGED
|
@@ -1,2 +1,11 @@
|
|
|
1
|
+
=== 1.0.1 (2017-3-21)
|
|
2
|
+
|
|
3
|
+
* Drop more tokens, keep more characters in PostgreSQLCustomParser,
|
|
4
|
+
which originally treated more of the PG special (unsafe) characters
|
|
5
|
+
as whitespace. A result of these changes is that its less likely to
|
|
6
|
+
see replacement characters in generated output.
|
|
7
|
+
|
|
8
|
+
* Account for Postgresql 9.6.2 change in tests
|
|
9
|
+
|
|
1
10
|
=== 1.0.0 (2016-11-8)
|
|
2
11
|
* Initial release.
|
data/README.rdoc
CHANGED
|
@@ -44,7 +44,7 @@ Other generators are possible.
|
|
|
44
44
|
|
|
45
45
|
== License
|
|
46
46
|
|
|
47
|
-
Copyright (c) 2016 David Kellum
|
|
47
|
+
Copyright (c) 2016-2017 David Kellum
|
|
48
48
|
|
|
49
49
|
Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
50
50
|
may not use this file except in compliance with the License. You
|
data/lib/human-ql.rb
CHANGED
data/lib/human-ql/base.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#--
|
|
2
|
-
# Copyright (c) 2016 David Kellum
|
|
2
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
5
5
|
# may not use this file except in compliance with the License. You may
|
|
@@ -16,6 +16,6 @@
|
|
|
16
16
|
|
|
17
17
|
module HumanQL
|
|
18
18
|
|
|
19
|
-
VERSION='1.0.
|
|
19
|
+
VERSION='1.0.1'
|
|
20
20
|
|
|
21
21
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
1
3
|
#--
|
|
2
|
-
# Copyright (c) 2016 David Kellum
|
|
4
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
3
5
|
#
|
|
4
6
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
5
7
|
# may not use this file except in compliance with the License. You may
|
|
@@ -19,11 +21,17 @@ require 'human-ql/query_parser'
|
|
|
19
21
|
module HumanQL
|
|
20
22
|
|
|
21
23
|
# Extends the generic QueryParser with additional special character
|
|
22
|
-
# filtering so as to avoid syntax errors
|
|
23
|
-
# for any known input. Note that this is
|
|
24
|
-
# HumanQL query language, not anything
|
|
24
|
+
# and token filtering so as to avoid syntax errors with to_tsquery()
|
|
25
|
+
# (via PostgreSQLGenerator) for any known input. Note that this is
|
|
26
|
+
# still a parser for the HumanQL query language, not anything
|
|
27
|
+
# implemented in PostgreSQL.
|
|
25
28
|
class PostgreSQLCustomParser < QueryParser
|
|
26
29
|
|
|
30
|
+
# U+2019 RIGHT SINGLE QUOTATION MARK
|
|
31
|
+
RSQMARK = '’'.freeze
|
|
32
|
+
|
|
33
|
+
UNDERSCORE = '_'.freeze
|
|
34
|
+
|
|
27
35
|
# Construct given options to set via base clase or as specified
|
|
28
36
|
# below.
|
|
29
37
|
#
|
|
@@ -32,7 +40,7 @@ module HumanQL
|
|
|
32
40
|
# :pg_version:: A version string ("9.5.5", "9.6.1") or integer
|
|
33
41
|
# array ( [9,6,1]) indicating the target PostgreSQL
|
|
34
42
|
# version. Phrase support starts in 9.6 so quoted
|
|
35
|
-
# phrases are ignored before that.
|
|
43
|
+
# phrases are ignored before that. Default: < 9.6
|
|
36
44
|
#
|
|
37
45
|
def initialize(opts = {})
|
|
38
46
|
opts = opts.dup
|
|
@@ -46,21 +54,44 @@ module HumanQL
|
|
|
46
54
|
|
|
47
55
|
# Phrase support starts in 9.6
|
|
48
56
|
if ( pg_version <=> [9,6] ) >= 0
|
|
49
|
-
|
|
57
|
+
# Handle what PG-sensitive chracters we can early as
|
|
58
|
+
# whitespace. This can't include anything part of HumanQL,
|
|
59
|
+
# e.g. ':' as used for scopes, so deal with the remainder
|
|
60
|
+
# below.
|
|
61
|
+
self.spaces = /[[:space:]*!<>]+/.freeze
|
|
50
62
|
else
|
|
51
|
-
# Disable quote tokens
|
|
63
|
+
# Disable quote tokens
|
|
52
64
|
self.lquote = nil
|
|
53
65
|
self.rquote = nil
|
|
54
|
-
|
|
66
|
+
# As above but add DQUOTE as well.
|
|
67
|
+
self.spaces = /[[:space:]*!<>"]+/.freeze
|
|
55
68
|
end
|
|
56
69
|
|
|
70
|
+
# Use by custom #norm_phrase_tokens as a superset of the
|
|
71
|
+
# #lparen, #rparen token patterns removed by default. In
|
|
72
|
+
# PostgreSQL, the '|' and '&' still need to be filtered. Other
|
|
73
|
+
# freestanding punctuation tokens are best removed entirely.
|
|
74
|
+
@phrase_token_rejects = /\A[()|&':]\z/.freeze
|
|
75
|
+
|
|
76
|
+
# SQUOTE is a problem only when at beginning of term.
|
|
77
|
+
@lead_squote = /\A'/.freeze
|
|
78
|
+
|
|
79
|
+
# COLON is always a problem, but since its also part of Human QL
|
|
80
|
+
# (scopes) it can't be included earlier in #spaces. Also per
|
|
81
|
+
# scope parsing rules, its not always made a separate token.
|
|
82
|
+
@term_rejects = /:/.freeze
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def norm_phrase_tokens( tokens )
|
|
86
|
+
tokens.
|
|
87
|
+
reject { |t| @phrase_token_rejects === t }.
|
|
88
|
+
map { |t| norm_term( t ) }
|
|
57
89
|
end
|
|
58
90
|
|
|
59
|
-
# Replace
|
|
60
|
-
# (or effectively, whitespace) in tsquery with tested
|
|
61
|
-
# dictionaries.
|
|
91
|
+
# Replace various problem single characters with alt. characters.
|
|
62
92
|
def norm_term( t )
|
|
63
|
-
t.
|
|
93
|
+
t.sub( @lead_squote, RSQMARK ).
|
|
94
|
+
gsub( @term_rejects, UNDERSCORE )
|
|
64
95
|
end
|
|
65
96
|
end
|
|
66
97
|
|
data/test/setup.rb
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# coding: utf-8
|
|
2
3
|
|
|
3
4
|
#--
|
|
4
|
-
# Copyright (c) 2016 David Kellum
|
|
5
|
+
# Copyright (c) 2016-2017 David Kellum
|
|
5
6
|
#
|
|
6
7
|
# Licensed under the Apache License, Version 2.0 (the "License"); you
|
|
7
8
|
# may not use this file except in compliance with the License. You may
|
|
@@ -82,7 +83,7 @@ class TestPostgresqlGenerator < Minitest::Test
|
|
|
82
83
|
|
|
83
84
|
def test_phrase_with_danger
|
|
84
85
|
skip( "For postgresql 9.6+" ) unless pg_gte_9_6?
|
|
85
|
-
assert_gen( '
|
|
86
|
+
assert_gen( 'boy', '": boy"' )
|
|
86
87
|
assert_tsq( "'boy'", '": boy"' )
|
|
87
88
|
end
|
|
88
89
|
|
|
@@ -147,15 +148,16 @@ class TestPostgresqlGenerator < Minitest::Test
|
|
|
147
148
|
end
|
|
148
149
|
|
|
149
150
|
def test_funk_1
|
|
150
|
-
assert_gen( "!(, &
|
|
151
|
+
assert_gen( "!(, & ’y)", "-( , 'y -)" )
|
|
151
152
|
assert_tsq( "!'y'", "-( ,'y -)" )
|
|
152
|
-
|
|
153
|
-
assert_gen( "!(, & _y) & c3", "|-( , 'y -)c3" )
|
|
153
|
+
assert_gen( "!(, & ’y) & c3", "|-( , 'y -)c3" )
|
|
154
154
|
assert_tsq( "!'y' & 'c3'", "|-( ,'y -)c3" )
|
|
155
155
|
end
|
|
156
156
|
|
|
157
157
|
def test_funk_2
|
|
158
|
-
if
|
|
158
|
+
if ( PG_VERSION <=> [9,6,2] ) >= 0
|
|
159
|
+
assert_tsq( "!!'boy' & 'cat'", "-(a -boy) & cat" )
|
|
160
|
+
elsif pg_gte_9_6?
|
|
159
161
|
# Crashes PG 9.6 beta 1-2, fixed in beta 3.
|
|
160
162
|
assert_tsq( "'boy' & 'cat'", "-(a -boy) & cat" )
|
|
161
163
|
else
|
data/test/test_query_parser.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: human-ql
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Kellum
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-03-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: minitest
|
|
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
133
133
|
version: '0'
|
|
134
134
|
requirements: []
|
|
135
135
|
rubyforge_project:
|
|
136
|
-
rubygems_version: 2.6.
|
|
136
|
+
rubygems_version: 2.6.10
|
|
137
137
|
signing_key:
|
|
138
138
|
specification_version: 4
|
|
139
139
|
summary: Human Query Language for full text search engines. Provides a lenient parser
|