forest_admin_datasource_pylon 1.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/README.md +179 -0
- data/Rakefile +6 -0
- data/forest_admin_datasource_pylon.gemspec +36 -0
- data/lib/forest_admin_datasource_pylon/client/writes.rb +88 -0
- data/lib/forest_admin_datasource_pylon/client.rb +436 -0
- data/lib/forest_admin_datasource_pylon/collections/account/api_filters.rb +43 -0
- data/lib/forest_admin_datasource_pylon/collections/account/schema_definition.rb +91 -0
- data/lib/forest_admin_datasource_pylon/collections/account/serializer.rb +21 -0
- data/lib/forest_admin_datasource_pylon/collections/account.rb +47 -0
- data/lib/forest_admin_datasource_pylon/collections/base_collection.rb +563 -0
- data/lib/forest_admin_datasource_pylon/collections/contact/api_filters.rb +38 -0
- data/lib/forest_admin_datasource_pylon/collections/contact/schema_definition.rb +93 -0
- data/lib/forest_admin_datasource_pylon/collections/contact/serializer.rb +20 -0
- data/lib/forest_admin_datasource_pylon/collections/contact.rb +45 -0
- data/lib/forest_admin_datasource_pylon/collections/cursor_collection.rb +131 -0
- data/lib/forest_admin_datasource_pylon/collections/fetch_all_collection.rb +192 -0
- data/lib/forest_admin_datasource_pylon/collections/issue/api_filters.rb +41 -0
- data/lib/forest_admin_datasource_pylon/collections/issue/id_lookup_reader.rb +88 -0
- data/lib/forest_admin_datasource_pylon/collections/issue/messages_embedder.rb +89 -0
- data/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb +122 -0
- data/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb +26 -0
- data/lib/forest_admin_datasource_pylon/collections/issue.rb +128 -0
- data/lib/forest_admin_datasource_pylon/collections/record_serialization.rb +84 -0
- data/lib/forest_admin_datasource_pylon/collections/relation_embedder.rb +101 -0
- data/lib/forest_admin_datasource_pylon/collections/team.rb +49 -0
- data/lib/forest_admin_datasource_pylon/collections/user.rb +69 -0
- data/lib/forest_admin_datasource_pylon/collections/writes.rb +417 -0
- data/lib/forest_admin_datasource_pylon/configuration.rb +84 -0
- data/lib/forest_admin_datasource_pylon/datasource.rb +53 -0
- data/lib/forest_admin_datasource_pylon/issue_enums.rb +20 -0
- data/lib/forest_admin_datasource_pylon/pagination/cursor_walker.rb +103 -0
- data/lib/forest_admin_datasource_pylon/plugins/close_issue/messages.rb +62 -0
- data/lib/forest_admin_datasource_pylon/plugins/close_issue.rb +141 -0
- data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/form_builder.rb +173 -0
- data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/payload.rb +72 -0
- data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification.rb +175 -0
- data/lib/forest_admin_datasource_pylon/plugins/issue_targets.rb +51 -0
- data/lib/forest_admin_datasource_pylon/query/condition_tree_translator.rb +151 -0
- data/lib/forest_admin_datasource_pylon/query/filter_value.rb +135 -0
- data/lib/forest_admin_datasource_pylon/query/operator_maps.rb +108 -0
- data/lib/forest_admin_datasource_pylon/rate_limiter.rb +139 -0
- data/lib/forest_admin_datasource_pylon/rate_limits.rb +96 -0
- data/lib/forest_admin_datasource_pylon/retry_policy.rb +86 -0
- data/lib/forest_admin_datasource_pylon/schema/custom_fields_introspector.rb +203 -0
- data/lib/forest_admin_datasource_pylon/throttle.rb +21 -0
- data/lib/forest_admin_datasource_pylon/version.rb +3 -0
- data/lib/forest_admin_datasource_pylon.rb +70 -0
- metadata +152 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
module ForestAdminDatasourcePylon
|
|
2
|
+
module Query
|
|
3
|
+
# See https://docs.usepylon.com/pylon-docs/developer/api/api-reference/issues
|
|
4
|
+
#
|
|
5
|
+
# Translates a Forest condition tree into the structured JSON filter of
|
|
6
|
+
# `POST /issues/search`:
|
|
7
|
+
#
|
|
8
|
+
# leaf -> { 'field' => …, 'operator' => …, 'value' | 'values' => … }
|
|
9
|
+
# branch -> { 'operator' => 'and' | 'or', 'subfilters' => [...] }
|
|
10
|
+
#
|
|
11
|
+
# Pylon nests sub-filters, so — unlike the Zendesk query string — OR is
|
|
12
|
+
# translated natively instead of being rejected.
|
|
13
|
+
#
|
|
14
|
+
# Anything the API cannot express raises UnsupportedOperatorError: a filter
|
|
15
|
+
# that is dropped returns unfiltered rows which look filtered. The wire
|
|
16
|
+
# format of the values, and the refusals that go with it, live in FilterValue.
|
|
17
|
+
class ConditionTreeTranslator
|
|
18
|
+
Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch
|
|
19
|
+
Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
|
|
20
|
+
|
|
21
|
+
# Pylon rejects sub-filters nested deeper than three levels.
|
|
22
|
+
MAX_DEPTH = 3
|
|
23
|
+
|
|
24
|
+
LIST_OPERATORS = %w[in not_in].freeze
|
|
25
|
+
VALUELESS_OPERATORS = %w[is_set is_unset].freeze
|
|
26
|
+
|
|
27
|
+
# The comparisons Pylon reads as a moment in time: what tells FilterValue
|
|
28
|
+
# that a bare date is a date, and not a piece of text a field happens to
|
|
29
|
+
# hold. Read off the emitted operator rather than off the column type,
|
|
30
|
+
# which the translator does not see -- and it is the operator that decides
|
|
31
|
+
# the format anyway.
|
|
32
|
+
TIME_OPERATORS = %w[time_is_after time_is_before].freeze
|
|
33
|
+
|
|
34
|
+
def self.call(condition_tree, api_filters: {}, timezone: nil)
|
|
35
|
+
return nil if condition_tree.nil?
|
|
36
|
+
|
|
37
|
+
new(api_filters: api_filters, timezone: timezone).translate(condition_tree)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize(api_filters: {}, timezone: nil)
|
|
41
|
+
@api_filters = api_filters || {}
|
|
42
|
+
@value = FilterValue.new(timezone: timezone)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def translate(node, depth = 1)
|
|
46
|
+
case node
|
|
47
|
+
when Branch then translate_branch(node, depth)
|
|
48
|
+
when Leaf then translate_leaf(node)
|
|
49
|
+
else
|
|
50
|
+
raise UnsupportedOperatorError, "Unknown condition node: #{node.class}"
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def translate_branch(branch, depth)
|
|
57
|
+
conditions = Array(branch.conditions)
|
|
58
|
+
if conditions.empty?
|
|
59
|
+
raise UnsupportedOperatorError, "Condition tree aggregator '#{branch.aggregator}' carries no condition."
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Validated before the unwrap below, so a branch is refused on the
|
|
63
|
+
# aggregator it carries rather than on how many conditions it holds.
|
|
64
|
+
operator = aggregator(branch)
|
|
65
|
+
|
|
66
|
+
# A lone condition needs no wrapper, and spending no nesting level on it
|
|
67
|
+
# keeps trees the agent builds one branch at a time under Pylon's cap.
|
|
68
|
+
return translate(conditions.first, depth) if conditions.size == 1
|
|
69
|
+
|
|
70
|
+
raise_too_deep(depth) if depth > MAX_DEPTH
|
|
71
|
+
|
|
72
|
+
{ 'operator' => operator,
|
|
73
|
+
'subfilters' => conditions.map { |condition| translate(condition, depth + 1) } }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def aggregator(branch)
|
|
77
|
+
value = branch.aggregator.to_s.downcase
|
|
78
|
+
return value if %w[and or].include?(value)
|
|
79
|
+
|
|
80
|
+
raise UnsupportedOperatorError,
|
|
81
|
+
"Unknown condition tree aggregator #{branch.aggregator.inspect}; expected 'And' or 'Or'."
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def translate_leaf(leaf)
|
|
85
|
+
spec = @api_filters[leaf.field]
|
|
86
|
+
raise_unfilterable_field(leaf.field) unless spec
|
|
87
|
+
|
|
88
|
+
operator = spec[:ops][leaf.operator]
|
|
89
|
+
raise_unsupported_operator(leaf, spec) unless operator
|
|
90
|
+
ensure_filterable_absence!(leaf, spec)
|
|
91
|
+
|
|
92
|
+
with_value({ 'field' => (spec[:param] || leaf.field).to_s, 'operator' => operator }, operator, leaf)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def with_value(filter, operator, leaf)
|
|
96
|
+
return filter if VALUELESS_OPERATORS.include?(operator)
|
|
97
|
+
return filter.merge('values' => @value.list(leaf)) if LIST_OPERATORS.include?(operator)
|
|
98
|
+
|
|
99
|
+
filter.merge('value' => @value.single(leaf, time: TIME_OPERATORS.include?(operator)))
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# `present`, `blank` and `missing` are advertised on every field carrying
|
|
103
|
+
# an equality or a membership filter: the agent derives them from those
|
|
104
|
+
# above the datasource and rewrites them into a comparison with an empty
|
|
105
|
+
# value. Only a field the API reference documents `is_set` / `is_unset` on
|
|
106
|
+
# can answer one, and it answers it through those operators, never through
|
|
107
|
+
# the rewritten comparison -- which Pylon would match against the empty
|
|
108
|
+
# value as if it were a value of its own.
|
|
109
|
+
#
|
|
110
|
+
# Refused here rather than in FilterValue, which sees the empty value but
|
|
111
|
+
# not whether the field has a presence filter to answer it with.
|
|
112
|
+
def ensure_filterable_absence!(leaf, spec)
|
|
113
|
+
return unless absence_condition?(leaf)
|
|
114
|
+
return if spec[:ops].values.any? { |candidate| VALUELESS_OPERATORS.include?(candidate) }
|
|
115
|
+
|
|
116
|
+
raise UnsupportedOperatorError,
|
|
117
|
+
"Pylon cannot filter '#{leaf.field}' for absence: the field carries no is_set / is_unset filter " \
|
|
118
|
+
'in the Pylon API reference, so a present, blank or missing condition on it cannot be translated. ' \
|
|
119
|
+
'Filter for absence on a field that does, or filter on a value instead.'
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The shape the absence operators are rewritten into: a nil value, or a
|
|
123
|
+
# list holding nothing but blanks. An empty list is not one of them -- it
|
|
124
|
+
# comes from a filter carrying no value at all, which FilterValue reports.
|
|
125
|
+
def absence_condition?(leaf)
|
|
126
|
+
return true if leaf.value.nil?
|
|
127
|
+
return false unless leaf.value.is_a?(Array) && leaf.value.any?
|
|
128
|
+
|
|
129
|
+
leaf.value.all? { |value| value.nil? || value.to_s.empty? }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def raise_too_deep(depth)
|
|
133
|
+
raise UnsupportedOperatorError,
|
|
134
|
+
"Pylon rejects a filter nested deeper than #{MAX_DEPTH} levels (reached #{depth}); " \
|
|
135
|
+
'flatten the segment or the filter.'
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def raise_unfilterable_field(field)
|
|
139
|
+
raise UnsupportedOperatorError,
|
|
140
|
+
"Pylon cannot filter on '#{field}'; add it to the collection's api_filters " \
|
|
141
|
+
'after checking it against the Pylon API reference.'
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def raise_unsupported_operator(leaf, spec)
|
|
145
|
+
raise UnsupportedOperatorError,
|
|
146
|
+
"Operator '#{leaf.operator}' is not supported on field '#{leaf.field}'. " \
|
|
147
|
+
"Supported: #{spec[:ops].keys.join(", ")}."
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
require 'date'
|
|
2
|
+
require 'active_support/core_ext/time/zones'
|
|
3
|
+
|
|
4
|
+
module ForestAdminDatasourcePylon
|
|
5
|
+
module Query
|
|
6
|
+
# How a Forest filter value reaches the wire, and every way it can fail to.
|
|
7
|
+
# Split from the translator, which knows the shape of the condition tree but
|
|
8
|
+
# not the format of what the leaves carry.
|
|
9
|
+
class FilterValue
|
|
10
|
+
# A date carrying no time of day, which is what the frontend sends for a
|
|
11
|
+
# Dateonly column -- a shape only a custom field has, no native column
|
|
12
|
+
# being typed that way.
|
|
13
|
+
DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/
|
|
14
|
+
|
|
15
|
+
# The identifier is stored stripped, not only checked stripped: kept as it
|
|
16
|
+
# came, a `" Europe/Paris "` passes the blank guard and then fails the zone
|
|
17
|
+
# lookup, and `format_date` falls back to UTC -- a day boundary quietly off
|
|
18
|
+
# by the offset, which is the failure this guard exists to prevent.
|
|
19
|
+
def initialize(timezone: nil)
|
|
20
|
+
identifier = timezone.to_s.strip
|
|
21
|
+
|
|
22
|
+
@timezone = identifier.empty? ? 'UTC' : identifier
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# `time` says the operator this value travels with is one of Pylon's time
|
|
26
|
+
# comparisons, which is what decides whether a bare date is a date or a
|
|
27
|
+
# piece of text: the same string on a text field is a value of its own.
|
|
28
|
+
def single(leaf, time: false)
|
|
29
|
+
raise_nil_value(leaf.field) if leaf.value.nil?
|
|
30
|
+
|
|
31
|
+
format(leaf.value, time: time, field: leaf.field)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Dropping the blanks would silently answer a different question: `not_in
|
|
35
|
+
# [nil, 'open']` was asked to exclude the blank records and would come
|
|
36
|
+
# back including them. An empty list is just as bad the other way round,
|
|
37
|
+
# translating to a filter matching everything.
|
|
38
|
+
#
|
|
39
|
+
# No time comparison takes a list, so nothing here is read as a date.
|
|
40
|
+
def list(leaf)
|
|
41
|
+
values = Array(leaf.value)
|
|
42
|
+
raise_empty_list(leaf) if values.empty?
|
|
43
|
+
raise_blank_in_list(leaf) if values.any? { |value| value.nil? || value.to_s.empty? }
|
|
44
|
+
|
|
45
|
+
values.map { |value| format(value, field: leaf.field) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# Booleans travel as they are: the filter is JSON, not a query string, so
|
|
51
|
+
# only the dates and the numbers the agent widened need a wire format.
|
|
52
|
+
def format(value, time: false, field: nil)
|
|
53
|
+
case value
|
|
54
|
+
when Time, DateTime then value.to_time.utc.iso8601
|
|
55
|
+
when Date then format_date(value)
|
|
56
|
+
when Float then format_float(value, field)
|
|
57
|
+
when String then time ? format_time_string(value) : value
|
|
58
|
+
else value
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The agent casts every Number column with `to_f`
|
|
63
|
+
# (`ConditionTreeParser.cast_to_type`), so an integer custom field would be
|
|
64
|
+
# filtered with `42.0` -- a form none of its values carry. A float with
|
|
65
|
+
# nothing after the point travels as the integer it is; a decimal keeps its
|
|
66
|
+
# own.
|
|
67
|
+
#
|
|
68
|
+
# A cast that overflowed to Infinity, or a NaN, is refused rather than
|
|
69
|
+
# passed on: `to_i` raises on both, and so does the JSON encoder a step
|
|
70
|
+
# later, either way as a 500 naming nothing the operator can act on.
|
|
71
|
+
def format_float(value, field)
|
|
72
|
+
raise_out_of_range(value, field) unless value.finite?
|
|
73
|
+
|
|
74
|
+
value == value.to_i ? value.to_i : value
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# `time_is_after` receives a timestamp everywhere else in this datasource,
|
|
78
|
+
# a native date column being read and filtered as one: a Dateonly custom
|
|
79
|
+
# field cannot be the one field sending the same operator another shape.
|
|
80
|
+
# The bound is the one a Ruby `Date` already gets -- midnight in the
|
|
81
|
+
# timezone of the caller.
|
|
82
|
+
#
|
|
83
|
+
# A string this operator cannot read as a date is left to Pylon, which
|
|
84
|
+
# names what it refuses better than a guess here would.
|
|
85
|
+
def format_time_string(value)
|
|
86
|
+
return value unless DATE_ONLY.match?(value)
|
|
87
|
+
|
|
88
|
+
format_date(Date.parse(value))
|
|
89
|
+
rescue Date::Error
|
|
90
|
+
value
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Only reached by a condition tree built in Ruby -- a segment or a scope
|
|
94
|
+
# written as code -- and by the bare date above. Everything else coming
|
|
95
|
+
# through HTTP arrives as an ISO8601 timestamp, already expressed in the
|
|
96
|
+
# timezone of the caller: the agent casts a date filter with `value.to_s`,
|
|
97
|
+
# and the toolkit formats the bounds it derives from Today / Previous*
|
|
98
|
+
# itself.
|
|
99
|
+
def format_date(value)
|
|
100
|
+
Time.use_zone(@timezone) { Time.zone.local(value.year, value.month, value.day).utc.iso8601 }
|
|
101
|
+
rescue ArgumentError
|
|
102
|
+
ForestAdminDatasourcePylon.logger.warn(
|
|
103
|
+
"[forest_admin_datasource_pylon] unknown timezone '#{@timezone}', falling back to UTC"
|
|
104
|
+
)
|
|
105
|
+
value.strftime('%Y-%m-%dT00:00:00Z')
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# A filter carrying a nil value reads as a presence check on most APIs,
|
|
109
|
+
# which is silently the wrong query.
|
|
110
|
+
def raise_nil_value(field)
|
|
111
|
+
raise UnsupportedOperatorError,
|
|
112
|
+
"Filter value on '#{field}' is nil; use the PRESENT or BLANK operator to filter for absence."
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def raise_blank_in_list(leaf)
|
|
116
|
+
raise UnsupportedOperatorError,
|
|
117
|
+
"Operator '#{leaf.operator}' on field '#{leaf.field}' was given a list holding a blank value; " \
|
|
118
|
+
'Pylon matches an absent value through is_set / is_unset only, so filter for absence with the ' \
|
|
119
|
+
'PRESENT or BLANK operator rather than listing nil or an empty string.'
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def raise_empty_list(leaf)
|
|
123
|
+
raise UnsupportedOperatorError,
|
|
124
|
+
"Operator '#{leaf.operator}' on field '#{leaf.field}' was given an empty list; " \
|
|
125
|
+
'pass at least one value.'
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def raise_out_of_range(value, field)
|
|
129
|
+
raise UnsupportedOperatorError,
|
|
130
|
+
"Filter value on '#{field}' is #{value}, which is not a number Pylon can be asked for; " \
|
|
131
|
+
'filter with a value inside the range of a double.'
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
module ForestAdminDatasourcePylon
|
|
2
|
+
module Query
|
|
3
|
+
# The operator maps the Pylon search endpoints share. A map spells each
|
|
4
|
+
# Forest operator as the Pylon operator honouring it; a collection's
|
|
5
|
+
# `API_FILTERS` table then assembles, field by field, the maps its endpoint
|
|
6
|
+
# accepts according to the API reference.
|
|
7
|
+
#
|
|
8
|
+
# Sharing the maps is what keeps those tables readable as the allow-lists
|
|
9
|
+
# they transcribe, and keeps one wire spelling from being fixed in one
|
|
10
|
+
# collection and left wrong in the next.
|
|
11
|
+
module OperatorMaps
|
|
12
|
+
Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
|
|
13
|
+
|
|
14
|
+
# `not_equal` is deliberately absent: the toolkit derives it from
|
|
15
|
+
# `not_in`, so declaring it would only add a second spelling.
|
|
16
|
+
EQUALITY = { Operators::EQUAL => 'equals',
|
|
17
|
+
Operators::IN => 'in',
|
|
18
|
+
Operators::NOT_IN => 'not_in' }.freeze
|
|
19
|
+
|
|
20
|
+
# MISSING is mapped as well although Pylon spells absence one way only:
|
|
21
|
+
# the toolkit rewrites it into `equal nil` when it is left out, which the
|
|
22
|
+
# translator cannot express, so a field the API does check for absence
|
|
23
|
+
# would refuse the very filter it can answer.
|
|
24
|
+
PRESENCE = { Operators::PRESENT => 'is_set',
|
|
25
|
+
Operators::BLANK => 'is_unset',
|
|
26
|
+
Operators::MISSING => 'is_unset' }.freeze
|
|
27
|
+
|
|
28
|
+
# Declaring the bare comparisons rather than before/after is what lets
|
|
29
|
+
# the toolkit rewrite Today / PreviousWeek / ... into a pair of bounds,
|
|
30
|
+
# which is also why `time_range` never has to be emitted.
|
|
31
|
+
TIME = { Operators::GREATER_THAN => 'time_is_after',
|
|
32
|
+
Operators::LESS_THAN => 'time_is_before' }.freeze
|
|
33
|
+
|
|
34
|
+
# Pylon exposes a single substring operator and documents no case
|
|
35
|
+
# semantics for it, so both Forest spellings map onto that one operator --
|
|
36
|
+
# or the UI would offer a case-sensitive "contains" and a case-insensitive
|
|
37
|
+
# one behaving identically.
|
|
38
|
+
SUBSTRING = { Operators::CONTAINS => 'string_contains',
|
|
39
|
+
Operators::I_CONTAINS => 'string_contains' }.freeze
|
|
40
|
+
|
|
41
|
+
# The same, for the endpoints that also accept the negation. A field
|
|
42
|
+
# filtered through SUBSTRING alone must not advertise it: Pylon rejects
|
|
43
|
+
# `string_does_not_contain` where it is not documented.
|
|
44
|
+
FULL_TEXT = SUBSTRING.merge(Operators::NOT_CONTAINS => 'string_does_not_contain',
|
|
45
|
+
Operators::NOT_I_CONTAINS => 'string_does_not_contain').freeze
|
|
46
|
+
|
|
47
|
+
# A list-valued field -- `tags`, `domains`: `in` matches it against
|
|
48
|
+
# several candidates at once.
|
|
49
|
+
#
|
|
50
|
+
# Pylon also accepts `contains` / `does_not_contain` on such a field, and
|
|
51
|
+
# they are deliberately left out: the columns are typed `Json`, the only
|
|
52
|
+
# type the toolkit has for a list, and `Rules` allows a Json column the
|
|
53
|
+
# base and array operators alone. A declared `contains` would be refused
|
|
54
|
+
# by `ConditionTreeValidator` on the way in -- "the given operator
|
|
55
|
+
# 'contains' is not allowed with the columnType schema: 'Json'" -- so the
|
|
56
|
+
# UI would offer a filter that errors instead of one Pylon answers.
|
|
57
|
+
# Typing the columns `['String']` is not the way out either: no branch of
|
|
58
|
+
# `get_allowed_operators_for_column_type` reads an array type, and the
|
|
59
|
+
# validator raises a NoMethodError on it. Reaching those two operators
|
|
60
|
+
# takes a toolkit change, which is not this datasource's to make here.
|
|
61
|
+
MEMBERSHIP = { Operators::IN => 'in',
|
|
62
|
+
Operators::NOT_IN => 'not_in' }.freeze
|
|
63
|
+
|
|
64
|
+
# A custom field is filtered through its slug, so its operators come from
|
|
65
|
+
# the column the integrator declared rather than from a table; every
|
|
66
|
+
# search endpoint accepts this same set on one.
|
|
67
|
+
CUSTOM_FIELD_OPS = EQUALITY.merge(PRESENCE).merge(TIME).merge(FULL_TEXT).freeze
|
|
68
|
+
|
|
69
|
+
# Extended by a collection's `ApiFilters` module, whose `API_FILTERS` is
|
|
70
|
+
# the single source of truth for what its endpoint filters: the schema
|
|
71
|
+
# derives every column's `filter_operators` from it, so no collection
|
|
72
|
+
# declares a filter the translator would then refuse.
|
|
73
|
+
#
|
|
74
|
+
# One family escapes those tables. The agent derives `present`, `blank` and
|
|
75
|
+
# `missing` from an equality or a membership filter, above the datasource,
|
|
76
|
+
# and rewrites them into a comparison with an empty value. Only a field
|
|
77
|
+
# carrying PRESENCE can answer one -- Pylon matches an absent value through
|
|
78
|
+
# `is_set` / `is_unset` alone -- so on every other field the translator
|
|
79
|
+
# refuses the rewritten condition and names the filter to change, rather
|
|
80
|
+
# than sending a comparison Pylon would answer as if the empty value were
|
|
81
|
+
# a value of its own.
|
|
82
|
+
module Table
|
|
83
|
+
def forest_operators(field)
|
|
84
|
+
self::API_FILTERS.dig(field, :ops)&.keys || []
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Read off the extending module rather than off this one, so the
|
|
88
|
+
# `CUSTOM_FIELD_OPS` a collection declares is the single source both
|
|
89
|
+
# this spelling and `allowed_custom_field_operators` come from: an
|
|
90
|
+
# endpoint accepting less on a custom field narrows one constant.
|
|
91
|
+
def for_custom_field(schema)
|
|
92
|
+
{ ops: self::CUSTOM_FIELD_OPS.slice(*Array(schema&.filter_operators)) }
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The table of a collection whose endpoint filters nothing server-side: no
|
|
97
|
+
# field, and no operator on a custom field either. It is the default of
|
|
98
|
+
# `BaseCollection#filter_table`, so a collection read whole and filtered
|
|
99
|
+
# in memory needs no table of its own.
|
|
100
|
+
module EmptyTable
|
|
101
|
+
extend Table
|
|
102
|
+
|
|
103
|
+
API_FILTERS = {}.freeze
|
|
104
|
+
CUSTOM_FIELD_OPS = {}.freeze
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
module ForestAdminDatasourcePylon
|
|
2
|
+
# Spaces requests out so the budget of an endpoint is spent rather than
|
|
3
|
+
# exceeded: a sliding window per endpoint, and a wait until the next slot when
|
|
4
|
+
# the window is full.
|
|
5
|
+
#
|
|
6
|
+
# This sits in front of the 429 retry rather than replacing it. The retry
|
|
7
|
+
# answers a 429 Pylon already sent, which costs a full rate-limit window to
|
|
8
|
+
# recover from (Retry-After runs up to 60s); waiting half a second for a slot
|
|
9
|
+
# is the same throughput at a fraction of the latency. The retry stays as the
|
|
10
|
+
# backstop for what this cannot see — another process, or another agent, on
|
|
11
|
+
# the same token.
|
|
12
|
+
#
|
|
13
|
+
# One limiter per Configuration, so per token: Pylon meters the token, and two
|
|
14
|
+
# agents holding different ones do not share a budget.
|
|
15
|
+
class RateLimiter
|
|
16
|
+
WINDOW = 60.0
|
|
17
|
+
|
|
18
|
+
# How long one attempt may be held back before it is let through anyway. The
|
|
19
|
+
# limiter exists to avoid a 429, not to guarantee one never happens: past
|
|
20
|
+
# this point the window is saturated by more than this agent's own traffic,
|
|
21
|
+
# so queueing behind it would trade a retry the client already handles for a
|
|
22
|
+
# request the operator watches spin. It goes out, and the warning says why.
|
|
23
|
+
#
|
|
24
|
+
# Per attempt, not per request: `retry` replays up to `max_retries` times and
|
|
25
|
+
# each replay asks for a slot of its own, so a request can spend this bound
|
|
26
|
+
# once per attempt, on top of the backoff `retry` waits itself. None of it
|
|
27
|
+
# runs under the Faraday timeout, which only covers the adapter.
|
|
28
|
+
#
|
|
29
|
+
# The region this smooths is narrow, and it is worth being plain about it.
|
|
30
|
+
# On a stream over budget the waits accumulate rather than settling, so the
|
|
31
|
+
# bound is crossed sooner the further over it sits: measured against a
|
|
32
|
+
# 120/min endpoint, a stream 1% over budget throttles its first thousand
|
|
33
|
+
# requests and then lets roughly one in twelve through unthrottled, one 5%
|
|
34
|
+
# over gives up after two hundred and lets half through, and past ~10% over
|
|
35
|
+
# the bound is crossed as soon as the window fills — request 121 — after
|
|
36
|
+
# which almost nothing is throttled at all. A burst arriving at once books
|
|
37
|
+
# its next slot a whole window out and goes straight through from the first
|
|
38
|
+
# request past the budget. Under real saturation the 429 retry is the
|
|
39
|
+
# defence, not this; what this buys is the region just over budget, and a
|
|
40
|
+
# budget the code knows rather than one it discovers as a 429.
|
|
41
|
+
DEFAULT_MAX_WAIT = 5.0
|
|
42
|
+
|
|
43
|
+
attr_reader :max_wait, :window
|
|
44
|
+
|
|
45
|
+
def initialize(limits: RateLimits, window: WINDOW, max_wait: DEFAULT_MAX_WAIT, clock: nil, sleeper: nil)
|
|
46
|
+
@limits = limits
|
|
47
|
+
@window = window.to_f
|
|
48
|
+
@max_wait = max_wait.to_f
|
|
49
|
+
@clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
50
|
+
@sleeper = sleeper || ->(seconds) { sleep(seconds) }
|
|
51
|
+
@mutex = Mutex.new
|
|
52
|
+
@slots = {}
|
|
53
|
+
@warned = {}
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Blocks until the endpoint has room, then returns. Called once per attempt,
|
|
57
|
+
# retries included: a replayed request spends the budget a first one did.
|
|
58
|
+
def acquire(method, path)
|
|
59
|
+
rule = @limits.for(method, path)
|
|
60
|
+
wait, warn = @mutex.synchronize { reserve(rule) }
|
|
61
|
+
|
|
62
|
+
warn_saturated(rule, wait) if warn
|
|
63
|
+
return if wait <= 0 || wait > @max_wait
|
|
64
|
+
|
|
65
|
+
@sleeper.call(wait)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# The reservation happens under the lock and the waiting outside it: holding
|
|
71
|
+
# the mutex across the sleep would serialize every thread behind the slowest
|
|
72
|
+
# one, and threads waiting on unrelated endpoints have no reason to queue.
|
|
73
|
+
#
|
|
74
|
+
# What is recorded is the moment the request will be made, not the moment it
|
|
75
|
+
# was asked for, so concurrent callers each take a distinct slot and spread
|
|
76
|
+
# out instead of all waking onto the same one.
|
|
77
|
+
#
|
|
78
|
+
# Returns the wait the caller owes and whether this is the bypass worth a log
|
|
79
|
+
# line — both settled here, the second being shared state like the first.
|
|
80
|
+
def reserve(rule)
|
|
81
|
+
taken = (@slots[rule.name] ||= [])
|
|
82
|
+
now = @clock.call
|
|
83
|
+
expire(taken, now)
|
|
84
|
+
|
|
85
|
+
slot = next_slot(taken, rule.limit, now)
|
|
86
|
+
wait = slot - now
|
|
87
|
+
bypass = wait > @max_wait
|
|
88
|
+
# Past the bound the request goes out now, so the slot it books is now:
|
|
89
|
+
# recording the one it declined to wait for would meter a request nobody
|
|
90
|
+
# ever made and push the whole window further out.
|
|
91
|
+
insert(taken, bypass ? now : slot)
|
|
92
|
+
|
|
93
|
+
[wait, bypass && first_warning?(rule, now)]
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The bookings that have left the window are its leading run, the list being
|
|
97
|
+
# kept ordered.
|
|
98
|
+
def expire(taken, now)
|
|
99
|
+
taken.shift(taken.bsearch_index { |at| at > now - @window } || taken.size)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Now while the window still has room, otherwise a window past the limit-th
|
|
103
|
+
# most recent booking, which is the one that has to fall out of it first —
|
|
104
|
+
# an index that only reads as such on an ordered list.
|
|
105
|
+
def next_slot(taken, limit, now)
|
|
106
|
+
return now if taken.size < limit
|
|
107
|
+
|
|
108
|
+
taken[taken.size - limit] + @window
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# A booking has one position in an ordered list, so it goes there rather than
|
|
112
|
+
# onto the end followed by a sort: `now` lands before the slots already
|
|
113
|
+
# reserved further out, and the list is what `next_slot` reads an index off.
|
|
114
|
+
def insert(taken, booking)
|
|
115
|
+
taken.insert(taken.bsearch_index { |at| at >= booking } || taken.size, booking)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# One line per endpoint per window. What the warning reports is a saturation
|
|
119
|
+
# that lasts, so a line per request puts one on every request it describes —
|
|
120
|
+
# a thousand of them for a couple of minutes over budget, burying the first,
|
|
121
|
+
# which is the only one the operator needed.
|
|
122
|
+
def first_warning?(rule, now)
|
|
123
|
+
last = @warned[rule.name]
|
|
124
|
+
return false if last && now - last < @window
|
|
125
|
+
|
|
126
|
+
@warned[rule.name] = now
|
|
127
|
+
true
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def warn_saturated(rule, wait)
|
|
131
|
+
ForestAdminDatasourcePylon.logger.warn(
|
|
132
|
+
"[forest_admin_datasource_pylon] #{rule.name} is at its budget of #{rule.limit} requests per " \
|
|
133
|
+
"#{@window.round}s; the next slot is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \
|
|
134
|
+
'Letting the request through — Pylon may answer 429, which the client retries. Further requests over ' \
|
|
135
|
+
"this budget are let through too, and this says so once per #{@window.round}s."
|
|
136
|
+
)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
module ForestAdminDatasourcePylon
|
|
2
|
+
# The per-endpoint budgets Pylon documents, in requests per minute, read off
|
|
3
|
+
# the API reference (docs.usepylon.com/pylon-docs/developer/api/api-reference,
|
|
4
|
+
# one page per resource — each endpoint states its own "Rate limit:" line).
|
|
5
|
+
#
|
|
6
|
+
# Pylon meters per endpoint, not per token, so every rule owns its own window:
|
|
7
|
+
# two endpoints allowing 300 each are 600 requests, and pooling them would
|
|
8
|
+
# throttle at half the budget the API grants.
|
|
9
|
+
#
|
|
10
|
+
# An endpoint absent from the table falls back to DEFAULT_LIMIT, the lowest
|
|
11
|
+
# figure documented anywhere on the API. An undocumented quota is not an
|
|
12
|
+
# absent one, and the generous guess is the one an operator discovers as a 429.
|
|
13
|
+
#
|
|
14
|
+
# Every endpoint the client calls has a rule, writes included: the budget of a
|
|
15
|
+
# write is no more uniform than that of a read — `POST /accounts` is granted
|
|
16
|
+
# ten times `POST /issues`, and `PATCH /accounts/{id}` two and a half times
|
|
17
|
+
# `PATCH /issues/{id}` — so leaving one to the fallback throttles it at a
|
|
18
|
+
# fraction of what Pylon grants, which is the failure this table exists to
|
|
19
|
+
# avoid rather than a safe default.
|
|
20
|
+
class RateLimits
|
|
21
|
+
DEFAULT_LIMIT = 30
|
|
22
|
+
|
|
23
|
+
# Ids are escaped before being joined to a path, so a segment never carries
|
|
24
|
+
# a slash and this matches exactly one of them.
|
|
25
|
+
ID = '[^/]+'.freeze
|
|
26
|
+
|
|
27
|
+
Rule = Struct.new(:name, :limit, keyword_init: true)
|
|
28
|
+
|
|
29
|
+
# Every rule is anchored, so `/issues`, `/issues/{id}` and
|
|
30
|
+
# `/issues/{id}/messages` are three distinct buckets rather than three
|
|
31
|
+
# readings of the same prefix, and the order of this list carries no meaning.
|
|
32
|
+
#
|
|
33
|
+
# `name` is what the window is keyed on and what a log line shows, so it
|
|
34
|
+
# spells the endpoint rather than its budget: two endpoints sharing a figure
|
|
35
|
+
# must not share a window.
|
|
36
|
+
RULES = [
|
|
37
|
+
['post /issues/search', :post, %r{\A/issues/search\z}, 120],
|
|
38
|
+
['post /accounts/search', :post, %r{\A/accounts/search\z}, 120],
|
|
39
|
+
['post /contacts/search', :post, %r{\A/contacts/search\z}, 120],
|
|
40
|
+
['get /issues/:id/messages', :get, %r{\A/issues/#{ID}/messages\z}, 120],
|
|
41
|
+
['get /issues', :get, %r{\A/issues\z}, 30],
|
|
42
|
+
['post /issues', :post, %r{\A/issues\z}, 30],
|
|
43
|
+
['get /issues/:id', :get, %r{\A/issues/#{ID}\z}, 300],
|
|
44
|
+
['patch /issues/:id', :patch, %r{\A/issues/#{ID}\z}, 120],
|
|
45
|
+
['get /accounts', :get, %r{\A/accounts\z}, 300],
|
|
46
|
+
['get /accounts/:id', :get, %r{\A/accounts/#{ID}\z}, 300],
|
|
47
|
+
['get /contacts', :get, %r{\A/contacts\z}, 300],
|
|
48
|
+
['get /contacts/:id', :get, %r{\A/contacts/#{ID}\z}, 300],
|
|
49
|
+
['get /users', :get, %r{\A/users\z}, 300],
|
|
50
|
+
['get /users/:id', :get, %r{\A/users/#{ID}\z}, 300],
|
|
51
|
+
['get /teams', :get, %r{\A/teams\z}, 300],
|
|
52
|
+
['get /teams/:id', :get, %r{\A/teams/#{ID}\z}, 300],
|
|
53
|
+
['get /custom-fields', :get, %r{\A/custom-fields\z}, 300],
|
|
54
|
+
['get /me', :get, %r{\A/me\z}, 300],
|
|
55
|
+
['post /accounts', :post, %r{\A/accounts\z}, 300],
|
|
56
|
+
['patch /accounts/:id', :patch, %r{\A/accounts/#{ID}\z}, 300],
|
|
57
|
+
['post /contacts', :post, %r{\A/contacts\z}, 300],
|
|
58
|
+
['patch /contacts/:id', :patch, %r{\A/contacts/#{ID}\z}, 300],
|
|
59
|
+
['delete /issues/:id', :delete, %r{\A/issues/#{ID}\z}, 120],
|
|
60
|
+
['patch /teams/:id', :patch, %r{\A/teams/#{ID}\z}, 120],
|
|
61
|
+
['patch /users/:id', :patch, %r{\A/users/#{ID}\z}, 120],
|
|
62
|
+
['delete /accounts/:id', :delete, %r{\A/accounts/#{ID}\z}, 30],
|
|
63
|
+
['delete /contacts/:id', :delete, %r{\A/contacts/#{ID}\z}, 30],
|
|
64
|
+
['post /teams', :post, %r{\A/teams\z}, 30]
|
|
65
|
+
].map { |name, verb, pattern, limit| [verb, pattern, Rule.new(name: name, limit: limit)] }.freeze
|
|
66
|
+
|
|
67
|
+
class << self
|
|
68
|
+
def for(method, path)
|
|
69
|
+
verb = method.to_s.downcase.to_sym
|
|
70
|
+
normalized = normalize(path)
|
|
71
|
+
|
|
72
|
+
found = RULES.find { |rule_verb, pattern, _rule| rule_verb == verb && pattern.match?(normalized) }
|
|
73
|
+
found ? found[2] : fallback(verb, normalized)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
# Faraday hands back the path of the resolved URL, which carries the
|
|
79
|
+
# leading slash the rules are written against and, on a base url mounted
|
|
80
|
+
# under a subpath, whatever precedes it.
|
|
81
|
+
def normalize(path)
|
|
82
|
+
stripped = path.to_s.chomp('/')
|
|
83
|
+
stripped.start_with?('/') ? stripped : "/#{stripped}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# An unlisted endpoint is bucketed by its first segment rather than by its
|
|
87
|
+
# full path: keying on the path would open a window per record id, so a
|
|
88
|
+
# fan-out over a hundred records would meter as a hundred endpoints each
|
|
89
|
+
# one request in.
|
|
90
|
+
def fallback(verb, path)
|
|
91
|
+
segment = path.split('/').reject(&:empty?).first
|
|
92
|
+
Rule.new(name: "#{verb} /#{segment} (undocumented)", limit: DEFAULT_LIMIT)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|