query_helper 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "query_helper"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,33 @@
1
+ class QueryHelper
2
+ class Associations
3
+ def self.process_association_params(associations)
4
+ associations ||= []
5
+ associations.class == String ? [associations.to_sym] : associations
6
+ end
7
+
8
+ def self.load_associations(payload:, associations: [], as_json_options: {})
9
+ as_json_options ||= {}
10
+ as_json_options[:include] = as_json_options[:include] || json_associations(associations)
11
+ ActiveRecord::Associations::Preloader.new.preload(payload, associations)
12
+ payload.as_json(as_json_options)
13
+ end
14
+
15
+ def self.json_associations(associations)
16
+ associations ||= []
17
+ associations = associations.is_a?(Array) ? associations : [associations]
18
+ associations.inject([]) do |translated, association|
19
+ if association.is_a?(Symbol) || association.is_a?(String)
20
+ translated << association.to_sym
21
+ elsif association.is_a?(Array)
22
+ translated << association.map(&:to_sym)
23
+ elsif association.is_a?(Hash)
24
+ translated_hash = {}
25
+ association.each do |key, value|
26
+ translated_hash[key.to_sym] = { include: json_associations(value) }
27
+ end
28
+ translated << translated_hash
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,56 @@
1
+ require "query_helper/sql_parser"
2
+
3
+ class QueryHelper
4
+ class ColumnMap
5
+
6
+ def self.create_column_mappings(custom_mappings:, query:, model:)
7
+ parser = SqlParser.new(query)
8
+ maps = create_from_hash(custom_mappings)
9
+
10
+ parser.find_aliases.each do |m|
11
+ maps << m if maps.select{|x| x.alias_name == m.alias_name}.empty?
12
+ end
13
+
14
+ model.attribute_names.each do |attribute|
15
+ if maps.select{|x| x.alias_name == attribute}.empty?
16
+ maps << ColumnMap.new(alias_name: attribute, sql_expression: "#{model.to_s.downcase.pluralize}.#{attribute}")
17
+ end
18
+ end
19
+
20
+ maps
21
+ end
22
+
23
+ def self.create_from_hash(hash)
24
+ map = []
25
+ hash.each do |k,v|
26
+ alias_name = k
27
+ aggregate = false
28
+ if v.class == String
29
+ sql_expression = v
30
+ elsif v.class == Hash
31
+ sql_expression = v[:sql_expression]
32
+ aggregate = v[:aggregate]
33
+ end
34
+ map << self.new(
35
+ alias_name: alias_name,
36
+ sql_expression: sql_expression,
37
+ aggregate: aggregate
38
+ )
39
+ end
40
+ map
41
+ end
42
+
43
+ attr_accessor :alias_name, :sql_expression, :aggregate
44
+
45
+ def initialize(
46
+ alias_name:,
47
+ sql_expression:,
48
+ aggregate: false
49
+ )
50
+ @alias_name = alias_name
51
+ @sql_expression = sql_expression
52
+ @aggregate = aggregate
53
+ end
54
+
55
+ end
56
+ end
@@ -0,0 +1,112 @@
1
+ require "query_helper/invalid_query_error"
2
+
3
+ class QueryHelper
4
+ class Filter
5
+
6
+ attr_accessor :operator, :criterion, :comparate, :operator_code, :bind_variable, :aggregate
7
+
8
+ def initialize(
9
+ operator_code:,
10
+ criterion:,
11
+ comparate:,
12
+ aggregate: false
13
+ )
14
+ @operator_code = operator_code
15
+ @criterion = criterion # Converts to a string to be inserted into sql.
16
+ @comparate = comparate
17
+ @aggregate = aggregate
18
+ @bind_variable = ('a'..'z').to_a.shuffle[0,20].join.to_sym
19
+
20
+ translate_operator_code()
21
+ mofify_criterion()
22
+ modify_comparate()
23
+ validate_criterion()
24
+ end
25
+
26
+ def sql_string
27
+ case operator_code
28
+ when "in", "notin"
29
+ "#{comparate} #{operator} (:#{bind_variable})"
30
+ when "null"
31
+ "#{comparate} #{operator}"
32
+ else
33
+ "#{comparate} #{operator} :#{bind_variable}"
34
+ end
35
+
36
+ end
37
+
38
+ private
39
+
40
+ def translate_operator_code
41
+ @operator = case operator_code
42
+ when "gte"
43
+ ">="
44
+ when "lte"
45
+ "<="
46
+ when "gt"
47
+ ">"
48
+ when "lt"
49
+ "<"
50
+ when "eql"
51
+ "="
52
+ when "noteql"
53
+ "!="
54
+ when "in"
55
+ "in"
56
+ when "like"
57
+ "like"
58
+ when "notin"
59
+ "not in"
60
+ when "null"
61
+ if criterion.to_s == "true"
62
+ "is null"
63
+ else
64
+ "is not null"
65
+ end
66
+ else
67
+ raise InvalidQueryError.new("Invalid operator code: '#{operator_code}'")
68
+ end
69
+ end
70
+
71
+ def mofify_criterion
72
+ # lowercase strings for comparison
73
+ @criterion.downcase! if criterion.class == String && criterion.scan(/[a-zA-Z]/).any?
74
+
75
+ # turn the criterion into an array for in and notin comparisons
76
+ @criterion = criterion.split(",") if ["in", "notin"].include?(operator_code) && criterion.class == String
77
+ end
78
+
79
+ def modify_comparate
80
+ # lowercase strings for comparison
81
+ @comparate = "lower(#{@comparate})" if criterion.class == String && criterion.scan(/[a-zA-Z]/).any? && !["true", "false"].include?(criterion)
82
+ end
83
+
84
+ def validate_criterion
85
+ case operator_code
86
+ when "gte", "lte", "gt", "lt"
87
+ begin
88
+ Time.parse(criterion.to_s)
89
+ rescue
90
+ begin
91
+ Date.parse(criterion.to_s)
92
+ rescue
93
+ begin
94
+ Float(criterion.to_s)
95
+ rescue
96
+ invalid_criterion_error()
97
+ end
98
+ end
99
+ end
100
+ when "in", "notin"
101
+ invalid_criterion_error() unless criterion.class == Array
102
+ when "null"
103
+ invalid_criterion_error() unless ["true", "false"].include?(criterion.to_s)
104
+ end
105
+ true
106
+ end
107
+
108
+ def invalid_criterion_error
109
+ raise InvalidQueryError.new("'#{criterion}' is not a valid criterion for the '#{@operator}' operator")
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,3 @@
1
+ class QueryHelper
2
+ class InvalidQueryError < StandardError; end
3
+ end
@@ -0,0 +1,41 @@
1
+ require 'active_support/concern'
2
+ require "query_helper/sql_filter"
3
+
4
+ class QueryHelper
5
+ module QueryHelperConcern
6
+ extend ActiveSupport::Concern
7
+
8
+ included do
9
+ def query_helper
10
+ @query_helper
11
+ end
12
+
13
+ def create_query_helper
14
+ @query_helper = QueryHelper.new(**query_helper_params, api_payload: true)
15
+ end
16
+
17
+ def create_query_helper_filter
18
+ filter_values = params[:filter].permit!.to_h
19
+ QueryHelper::SqlFilter.new(filter_values: filter_values)
20
+ end
21
+
22
+ def create_query_helper_sort
23
+ QueryHelper::SqlSort.new(sort_string: params[:sort])
24
+ end
25
+
26
+ def create_query_helper_associations
27
+
28
+ end
29
+
30
+ def query_helper_params
31
+ helpers = {}
32
+ helpers[:page] = params[:page] if params[:page]
33
+ helpers[:per_page] = params[:per_page] if params[:per_page]
34
+ helpers[:sql_filter] = create_query_helper_filter() if params[:filter]
35
+ helpers[:sql_sort] = create_query_helper_sort() if params[:sort]
36
+ helpers[:associations] = create_query_helper_associations() if params[:include]
37
+ helpers
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,43 @@
1
+ require "query_helper/invalid_query_error"
2
+
3
+ class QueryHelper
4
+ class SqlFilter
5
+
6
+ attr_accessor :filter_values, :column_maps
7
+
8
+ def initialize(filter_values: [], column_maps: [])
9
+ @column_maps = column_maps
10
+ @filter_values = filter_values
11
+ end
12
+
13
+ def create_filters
14
+ @filters = []
15
+
16
+ @filter_values.each do |comparate_alias, criteria|
17
+ # Find the sql mapping if it exists
18
+ map = @column_maps.find { |m| m.alias_name == comparate_alias }
19
+ raise InvalidQueryError.new("cannot filter by #{comparate_alias}") unless map
20
+
21
+ # create the filter
22
+ @filters << QueryHelper::Filter.new(
23
+ operator_code: criteria.keys.first,
24
+ criterion: criteria.values.first,
25
+ comparate: map.sql_expression,
26
+ aggregate: map.aggregate
27
+ )
28
+ end
29
+ end
30
+
31
+ def where_clauses
32
+ @filters.select{ |f| f.aggregate == false }.map(&:sql_string)
33
+ end
34
+
35
+ def having_clauses
36
+ @filters.select{ |f| f.aggregate == true }.map(&:sql_string)
37
+ end
38
+
39
+ def bind_variables
40
+ Hash[@filters.collect { |f| [f.bind_variable, f.criterion] }]
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,62 @@
1
+ require "query_helper/sql_parser"
2
+
3
+ class QueryHelper
4
+ class SqlManipulator
5
+
6
+ attr_accessor :sql
7
+
8
+ def initialize(
9
+ sql:,
10
+ where_clauses: nil,
11
+ having_clauses: nil,
12
+ order_by_clauses: nil,
13
+ include_limit_clause: false
14
+ )
15
+ @parser = SqlParser.new(sql)
16
+ @sql = @parser.sql.dup
17
+ @where_clauses = where_clauses
18
+ @having_clauses = having_clauses
19
+ @order_by_clauses = order_by_clauses
20
+ @include_limit_clause = include_limit_clause
21
+ end
22
+
23
+ def build
24
+ insert_having_clauses()
25
+ insert_where_clauses()
26
+ insert_total_count_select_clause()
27
+ insert_order_by_and_limit_clause()
28
+ @sql.squish
29
+ end
30
+
31
+ private
32
+
33
+ def insert_total_count_select_clause
34
+ return unless @include_limit_clause
35
+ total_count_clause = " ,count(*) over () as _query_full_count "
36
+ @sql.insert(@parser.insert_select_index, total_count_clause)
37
+ # Potentially update parser here
38
+ end
39
+
40
+ def insert_where_clauses
41
+ return unless @where_clauses.length > 0
42
+ begin_string = @parser.where_included? ? "and" : "where"
43
+ filter_string = @where_clauses.join(" and ")
44
+ " #{begin_string} #{filter_string} "
45
+ @sql.insert(@parser.insert_where_index, " #{begin_string} #{filter_string} ")
46
+ end
47
+
48
+ def insert_having_clauses
49
+ return unless @having_clauses.length > 0
50
+ begin_string = @parser.having_included? ? "and" : "having"
51
+ filter_string = @having_clauses.join(" and ")
52
+ @sql.insert(@parser.insert_having_index, " #{begin_string} #{filter_string} ")
53
+ end
54
+
55
+ def insert_order_by_and_limit_clause
56
+ @sql.slice!(@parser.limit_clause) if @parser.limit_included? # remove existing limit clause
57
+ @sql.slice!(@parser.order_by_clause) if @parser.order_by_included? # remove existing order by clause
58
+ @sql += " order by #{@order_by_clauses.join(", ")} " if @order_by_clauses.length > 0
59
+ @sql += " limit :limit offset :offset " if @include_limit_clause
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,189 @@
1
+ require "query_helper/invalid_query_error"
2
+ require "query_helper/column_map"
3
+
4
+ class QueryHelper
5
+ class SqlParser
6
+
7
+ attr_accessor :sql
8
+
9
+ def initialize(sql)
10
+ update(sql)
11
+ end
12
+
13
+ def update(sql)
14
+ @sql = sql
15
+ remove_comments()
16
+ white_out()
17
+ end
18
+
19
+ def remove_comments
20
+ # Remove SQL inline comments (/* */) and line comments (--)
21
+ @sql = @sql.gsub(/\/\*(.*?)\*\//, '').gsub(/--(.*)$/, '')
22
+ @sql.squish!
23
+ end
24
+
25
+ def white_out
26
+ # Replace everything between () and '' and ""
27
+ # This will allow us to ignore subqueries, common table expressions,
28
+ # regex, custom strings, etc. when determining injection points
29
+ # and performing other manipulations
30
+ @white_out_sql = @sql.dup
31
+ while @white_out_sql.scan(/\"[^""]*\"|\'[^'']*\'|\([^()]*\)/).length > 0 do
32
+ @white_out_sql.scan(/\"[^""]*\"|\'[^'']*\'|\([^()]*\)/).each { |s| @white_out_sql.gsub!(s,s.gsub(/./, '*')) }
33
+ end
34
+ end
35
+
36
+ def select_index(position=:start)
37
+ regex = /( |^)[Ss][Ee][Ll][Ee][Cc][Tt] / # space or new line at beginning of select
38
+ find_index(regex, position)
39
+ end
40
+
41
+ def from_index(position=:start)
42
+ regex = / [Ff][Rr][Oo][Mm] /
43
+ find_index(regex, position)
44
+ end
45
+
46
+ def where_index(position=:start)
47
+ regex = / [Ww][Hh][Ee][Rr][Ee] /
48
+ find_index(regex, position)
49
+ end
50
+
51
+ def group_by_index(position=:start)
52
+ regex = / [Gg][Rr][Oo][Uu][Pp] [Bb][Yy] /
53
+ find_index(regex, position)
54
+ end
55
+
56
+ def having_index(position=:start)
57
+ regex = / [Hh][Aa][Vv][Ii][Nn][Gg] /
58
+ find_index(regex, position)
59
+ end
60
+
61
+ def order_by_index(position=:start)
62
+ regex = / [Oo][Rr][Dd][Ee][Rr] [Bb][Yy] /
63
+ find_index(regex, position)
64
+ end
65
+
66
+ def limit_index(position=:start)
67
+ regex = / [Ll][Ii][Mm][Ii][Tt] /
68
+ find_index(regex, position)
69
+ end
70
+
71
+ def select_included?
72
+ !select_index.nil?
73
+ end
74
+
75
+ def from_included?
76
+ !from_index.nil?
77
+ end
78
+
79
+ def where_included?
80
+ !where_index.nil?
81
+ end
82
+
83
+ def group_by_included?
84
+ !group_by_index.nil?
85
+ end
86
+
87
+ def having_included?
88
+ !having_index.nil?
89
+ end
90
+
91
+ def order_by_included?
92
+ !order_by_index.nil?
93
+ end
94
+
95
+ def limit_included?
96
+ !limit_index.nil?
97
+ end
98
+
99
+ def insert_select_index
100
+ from_index() || where_index() || group_by_index() || order_by_index() || limit_index() || @sql.length
101
+ end
102
+
103
+ def insert_join_index
104
+ where_index() || group_by_index() || order_by_index() || limit_index() || @sql.length
105
+ end
106
+
107
+ def insert_where_index
108
+ group_by_index() || order_by_index() || limit_index() || @sql.length
109
+ end
110
+
111
+ def insert_having_index
112
+ # raise InvalidQueryError.new("Cannot calculate insert_having_index because the query has no group by clause") unless group_by_included?
113
+ order_by_index() || limit_index() || @sql.length
114
+ end
115
+
116
+ def insert_order_by_index
117
+ # raise InvalidQueryError.new("This query already includes an order by clause") if order_by_included?
118
+ limit_index() || @sql.length
119
+ end
120
+
121
+ def insert_limit_index
122
+ # raise InvalidQueryError.new("This query already includes a limit clause") if limit_included?
123
+ @sql.length
124
+ end
125
+
126
+ def select_clause
127
+ @sql[select_index()..insert_select_index()].strip if select_included?
128
+ end
129
+
130
+ def from_clause
131
+ @sql[from_index()..insert_join_index()].strip if from_included?
132
+ end
133
+
134
+ def where_clause
135
+ @sql[where_index()..insert_where_index()].strip if where_included?
136
+ end
137
+
138
+ # def group_by_clause
139
+ # @sql[group_by_index()..insert_group_by_index()] if group_by_included?
140
+ # end
141
+
142
+ def having_clause
143
+ @sql[having_index()..insert_having_index()].strip if having_included?
144
+ end
145
+
146
+ def order_by_clause
147
+ @sql[order_by_index()..insert_order_by_index()].strip if order_by_included?
148
+ end
149
+
150
+ def limit_clause
151
+ @sql[limit_index()..insert_limit_index()].strip if limit_included?
152
+ end
153
+
154
+ def find_aliases
155
+ # Determine alias expression combos. White out sql used in case there
156
+ # are any custom strings or subqueries in the select clause
157
+ white_out_selects = @white_out_sql[select_index(:end)..from_index()]
158
+ selects = @sql[select_index(:end)..from_index()]
159
+ comma_split_points = white_out_selects.each_char.with_index.map{|char, i| i if char == ','}.compact
160
+ comma_split_points.unshift(-1) # We need the first select clause to start out with a 'split'
161
+ column_maps = white_out_selects.split(",").each_with_index.map do |x,i|
162
+ sql_alias = x.squish.split(" as ")[1] || x.squish.split(" AS ")[1] || x.squish.split(".")[1] # look for custom defined aliases or table.column notation
163
+ # sql_alias = nil unless /^[a-zA-Z_]+$/.match?(sql_alias) # only allow aliases with letters and underscores
164
+ sql_expression = if x.split(" as ")[1]
165
+ expression_length = x.split(" as ")[0].length
166
+ selects[comma_split_points[i] + 1, expression_length]
167
+ elsif x.squish.split(" AS ")[1]
168
+ expression_length = x.split(" AS ")[0].length
169
+ selects[comma_split_points[i] + 1, expression_length]
170
+ elsif x.squish.split(".")[1]
171
+ selects[comma_split_points[i] + 1, x.length]
172
+ end
173
+ ColumnMap.new(
174
+ alias_name: sql_alias,
175
+ sql_expression: sql_expression.squish,
176
+ aggregate: /(array_agg|avg|bit_and|bit_or|bool_and|bool_or|count|every|json_agg|jsonb_agg|json_object_agg|jsonb_object_agg|max|min|string_agg|sum|xmlagg)\((.*)\)/.match?(sql_expression)
177
+ ) if sql_alias
178
+ end
179
+ column_maps.compact
180
+ end
181
+
182
+ private
183
+
184
+ def find_index(regex, position=:start)
185
+ start_position = @white_out_sql.rindex(regex)
186
+ return position == :start ? start_position : start_position + @white_out_sql[regex].size()
187
+ end
188
+ end
189
+ end