forest_admin_datasource_pylon 1.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/README.md +179 -0
  4. data/Rakefile +6 -0
  5. data/forest_admin_datasource_pylon.gemspec +36 -0
  6. data/lib/forest_admin_datasource_pylon/client/writes.rb +88 -0
  7. data/lib/forest_admin_datasource_pylon/client.rb +436 -0
  8. data/lib/forest_admin_datasource_pylon/collections/account/api_filters.rb +43 -0
  9. data/lib/forest_admin_datasource_pylon/collections/account/schema_definition.rb +91 -0
  10. data/lib/forest_admin_datasource_pylon/collections/account/serializer.rb +21 -0
  11. data/lib/forest_admin_datasource_pylon/collections/account.rb +47 -0
  12. data/lib/forest_admin_datasource_pylon/collections/base_collection.rb +563 -0
  13. data/lib/forest_admin_datasource_pylon/collections/contact/api_filters.rb +38 -0
  14. data/lib/forest_admin_datasource_pylon/collections/contact/schema_definition.rb +93 -0
  15. data/lib/forest_admin_datasource_pylon/collections/contact/serializer.rb +20 -0
  16. data/lib/forest_admin_datasource_pylon/collections/contact.rb +45 -0
  17. data/lib/forest_admin_datasource_pylon/collections/cursor_collection.rb +131 -0
  18. data/lib/forest_admin_datasource_pylon/collections/fetch_all_collection.rb +192 -0
  19. data/lib/forest_admin_datasource_pylon/collections/issue/api_filters.rb +41 -0
  20. data/lib/forest_admin_datasource_pylon/collections/issue/id_lookup_reader.rb +88 -0
  21. data/lib/forest_admin_datasource_pylon/collections/issue/messages_embedder.rb +89 -0
  22. data/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb +122 -0
  23. data/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb +26 -0
  24. data/lib/forest_admin_datasource_pylon/collections/issue.rb +128 -0
  25. data/lib/forest_admin_datasource_pylon/collections/record_serialization.rb +84 -0
  26. data/lib/forest_admin_datasource_pylon/collections/relation_embedder.rb +101 -0
  27. data/lib/forest_admin_datasource_pylon/collections/team.rb +49 -0
  28. data/lib/forest_admin_datasource_pylon/collections/user.rb +69 -0
  29. data/lib/forest_admin_datasource_pylon/collections/writes.rb +417 -0
  30. data/lib/forest_admin_datasource_pylon/configuration.rb +84 -0
  31. data/lib/forest_admin_datasource_pylon/datasource.rb +53 -0
  32. data/lib/forest_admin_datasource_pylon/issue_enums.rb +20 -0
  33. data/lib/forest_admin_datasource_pylon/pagination/cursor_walker.rb +103 -0
  34. data/lib/forest_admin_datasource_pylon/plugins/close_issue/messages.rb +62 -0
  35. data/lib/forest_admin_datasource_pylon/plugins/close_issue.rb +141 -0
  36. data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/form_builder.rb +173 -0
  37. data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/payload.rb +72 -0
  38. data/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification.rb +175 -0
  39. data/lib/forest_admin_datasource_pylon/plugins/issue_targets.rb +51 -0
  40. data/lib/forest_admin_datasource_pylon/query/condition_tree_translator.rb +151 -0
  41. data/lib/forest_admin_datasource_pylon/query/filter_value.rb +135 -0
  42. data/lib/forest_admin_datasource_pylon/query/operator_maps.rb +108 -0
  43. data/lib/forest_admin_datasource_pylon/rate_limiter.rb +139 -0
  44. data/lib/forest_admin_datasource_pylon/rate_limits.rb +96 -0
  45. data/lib/forest_admin_datasource_pylon/retry_policy.rb +86 -0
  46. data/lib/forest_admin_datasource_pylon/schema/custom_fields_introspector.rb +203 -0
  47. data/lib/forest_admin_datasource_pylon/throttle.rb +21 -0
  48. data/lib/forest_admin_datasource_pylon/version.rb +3 -0
  49. data/lib/forest_admin_datasource_pylon.rb +70 -0
  50. metadata +152 -0
@@ -0,0 +1,151 @@
1
+ module ForestAdminDatasourcePylon
2
+ module Query
3
+ # See https://docs.usepylon.com/pylon-docs/developer/api/api-reference/issues
4
+ #
5
+ # Translates a Forest condition tree into the structured JSON filter of
6
+ # `POST /issues/search`:
7
+ #
8
+ # leaf -> { 'field' => …, 'operator' => …, 'value' | 'values' => … }
9
+ # branch -> { 'operator' => 'and' | 'or', 'subfilters' => [...] }
10
+ #
11
+ # Pylon nests sub-filters, so — unlike the Zendesk query string — OR is
12
+ # translated natively instead of being rejected.
13
+ #
14
+ # Anything the API cannot express raises UnsupportedOperatorError: a filter
15
+ # that is dropped returns unfiltered rows which look filtered. The wire
16
+ # format of the values, and the refusals that go with it, live in FilterValue.
17
+ class ConditionTreeTranslator
18
+ Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch
19
+ Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
20
+
21
+ # Pylon rejects sub-filters nested deeper than three levels.
22
+ MAX_DEPTH = 3
23
+
24
+ LIST_OPERATORS = %w[in not_in].freeze
25
+ VALUELESS_OPERATORS = %w[is_set is_unset].freeze
26
+
27
+ # The comparisons Pylon reads as a moment in time: what tells FilterValue
28
+ # that a bare date is a date, and not a piece of text a field happens to
29
+ # hold. Read off the emitted operator rather than off the column type,
30
+ # which the translator does not see -- and it is the operator that decides
31
+ # the format anyway.
32
+ TIME_OPERATORS = %w[time_is_after time_is_before].freeze
33
+
34
+ def self.call(condition_tree, api_filters: {}, timezone: nil)
35
+ return nil if condition_tree.nil?
36
+
37
+ new(api_filters: api_filters, timezone: timezone).translate(condition_tree)
38
+ end
39
+
40
+ def initialize(api_filters: {}, timezone: nil)
41
+ @api_filters = api_filters || {}
42
+ @value = FilterValue.new(timezone: timezone)
43
+ end
44
+
45
+ def translate(node, depth = 1)
46
+ case node
47
+ when Branch then translate_branch(node, depth)
48
+ when Leaf then translate_leaf(node)
49
+ else
50
+ raise UnsupportedOperatorError, "Unknown condition node: #{node.class}"
51
+ end
52
+ end
53
+
54
+ private
55
+
56
+ def translate_branch(branch, depth)
57
+ conditions = Array(branch.conditions)
58
+ if conditions.empty?
59
+ raise UnsupportedOperatorError, "Condition tree aggregator '#{branch.aggregator}' carries no condition."
60
+ end
61
+
62
+ # Validated before the unwrap below, so a branch is refused on the
63
+ # aggregator it carries rather than on how many conditions it holds.
64
+ operator = aggregator(branch)
65
+
66
+ # A lone condition needs no wrapper, and spending no nesting level on it
67
+ # keeps trees the agent builds one branch at a time under Pylon's cap.
68
+ return translate(conditions.first, depth) if conditions.size == 1
69
+
70
+ raise_too_deep(depth) if depth > MAX_DEPTH
71
+
72
+ { 'operator' => operator,
73
+ 'subfilters' => conditions.map { |condition| translate(condition, depth + 1) } }
74
+ end
75
+
76
+ def aggregator(branch)
77
+ value = branch.aggregator.to_s.downcase
78
+ return value if %w[and or].include?(value)
79
+
80
+ raise UnsupportedOperatorError,
81
+ "Unknown condition tree aggregator #{branch.aggregator.inspect}; expected 'And' or 'Or'."
82
+ end
83
+
84
+ def translate_leaf(leaf)
85
+ spec = @api_filters[leaf.field]
86
+ raise_unfilterable_field(leaf.field) unless spec
87
+
88
+ operator = spec[:ops][leaf.operator]
89
+ raise_unsupported_operator(leaf, spec) unless operator
90
+ ensure_filterable_absence!(leaf, spec)
91
+
92
+ with_value({ 'field' => (spec[:param] || leaf.field).to_s, 'operator' => operator }, operator, leaf)
93
+ end
94
+
95
+ def with_value(filter, operator, leaf)
96
+ return filter if VALUELESS_OPERATORS.include?(operator)
97
+ return filter.merge('values' => @value.list(leaf)) if LIST_OPERATORS.include?(operator)
98
+
99
+ filter.merge('value' => @value.single(leaf, time: TIME_OPERATORS.include?(operator)))
100
+ end
101
+
102
+ # `present`, `blank` and `missing` are advertised on every field carrying
103
+ # an equality or a membership filter: the agent derives them from those
104
+ # above the datasource and rewrites them into a comparison with an empty
105
+ # value. Only a field the API reference documents `is_set` / `is_unset` on
106
+ # can answer one, and it answers it through those operators, never through
107
+ # the rewritten comparison -- which Pylon would match against the empty
108
+ # value as if it were a value of its own.
109
+ #
110
+ # Refused here rather than in FilterValue, which sees the empty value but
111
+ # not whether the field has a presence filter to answer it with.
112
+ def ensure_filterable_absence!(leaf, spec)
113
+ return unless absence_condition?(leaf)
114
+ return if spec[:ops].values.any? { |candidate| VALUELESS_OPERATORS.include?(candidate) }
115
+
116
+ raise UnsupportedOperatorError,
117
+ "Pylon cannot filter '#{leaf.field}' for absence: the field carries no is_set / is_unset filter " \
118
+ 'in the Pylon API reference, so a present, blank or missing condition on it cannot be translated. ' \
119
+ 'Filter for absence on a field that does, or filter on a value instead.'
120
+ end
121
+
122
+ # The shape the absence operators are rewritten into: a nil value, or a
123
+ # list holding nothing but blanks. An empty list is not one of them -- it
124
+ # comes from a filter carrying no value at all, which FilterValue reports.
125
+ def absence_condition?(leaf)
126
+ return true if leaf.value.nil?
127
+ return false unless leaf.value.is_a?(Array) && leaf.value.any?
128
+
129
+ leaf.value.all? { |value| value.nil? || value.to_s.empty? }
130
+ end
131
+
132
+ def raise_too_deep(depth)
133
+ raise UnsupportedOperatorError,
134
+ "Pylon rejects a filter nested deeper than #{MAX_DEPTH} levels (reached #{depth}); " \
135
+ 'flatten the segment or the filter.'
136
+ end
137
+
138
+ def raise_unfilterable_field(field)
139
+ raise UnsupportedOperatorError,
140
+ "Pylon cannot filter on '#{field}'; add it to the collection's api_filters " \
141
+ 'after checking it against the Pylon API reference.'
142
+ end
143
+
144
+ def raise_unsupported_operator(leaf, spec)
145
+ raise UnsupportedOperatorError,
146
+ "Operator '#{leaf.operator}' is not supported on field '#{leaf.field}'. " \
147
+ "Supported: #{spec[:ops].keys.join(", ")}."
148
+ end
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,135 @@
1
+ require 'date'
2
+ require 'active_support/core_ext/time/zones'
3
+
4
+ module ForestAdminDatasourcePylon
5
+ module Query
6
+ # How a Forest filter value reaches the wire, and every way it can fail to.
7
+ # Split from the translator, which knows the shape of the condition tree but
8
+ # not the format of what the leaves carry.
9
+ class FilterValue
10
+ # A date carrying no time of day, which is what the frontend sends for a
11
+ # Dateonly column -- a shape only a custom field has, no native column
12
+ # being typed that way.
13
+ DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/
14
+
15
+ # The identifier is stored stripped, not only checked stripped: kept as it
16
+ # came, a `" Europe/Paris "` passes the blank guard and then fails the zone
17
+ # lookup, and `format_date` falls back to UTC -- a day boundary quietly off
18
+ # by the offset, which is the failure this guard exists to prevent.
19
+ def initialize(timezone: nil)
20
+ identifier = timezone.to_s.strip
21
+
22
+ @timezone = identifier.empty? ? 'UTC' : identifier
23
+ end
24
+
25
+ # `time` says the operator this value travels with is one of Pylon's time
26
+ # comparisons, which is what decides whether a bare date is a date or a
27
+ # piece of text: the same string on a text field is a value of its own.
28
+ def single(leaf, time: false)
29
+ raise_nil_value(leaf.field) if leaf.value.nil?
30
+
31
+ format(leaf.value, time: time, field: leaf.field)
32
+ end
33
+
34
+ # Dropping the blanks would silently answer a different question: `not_in
35
+ # [nil, 'open']` was asked to exclude the blank records and would come
36
+ # back including them. An empty list is just as bad the other way round,
37
+ # translating to a filter matching everything.
38
+ #
39
+ # No time comparison takes a list, so nothing here is read as a date.
40
+ def list(leaf)
41
+ values = Array(leaf.value)
42
+ raise_empty_list(leaf) if values.empty?
43
+ raise_blank_in_list(leaf) if values.any? { |value| value.nil? || value.to_s.empty? }
44
+
45
+ values.map { |value| format(value, field: leaf.field) }
46
+ end
47
+
48
+ private
49
+
50
+ # Booleans travel as they are: the filter is JSON, not a query string, so
51
+ # only the dates and the numbers the agent widened need a wire format.
52
+ def format(value, time: false, field: nil)
53
+ case value
54
+ when Time, DateTime then value.to_time.utc.iso8601
55
+ when Date then format_date(value)
56
+ when Float then format_float(value, field)
57
+ when String then time ? format_time_string(value) : value
58
+ else value
59
+ end
60
+ end
61
+
62
+ # The agent casts every Number column with `to_f`
63
+ # (`ConditionTreeParser.cast_to_type`), so an integer custom field would be
64
+ # filtered with `42.0` -- a form none of its values carry. A float with
65
+ # nothing after the point travels as the integer it is; a decimal keeps its
66
+ # own.
67
+ #
68
+ # A cast that overflowed to Infinity, or a NaN, is refused rather than
69
+ # passed on: `to_i` raises on both, and so does the JSON encoder a step
70
+ # later, either way as a 500 naming nothing the operator can act on.
71
+ def format_float(value, field)
72
+ raise_out_of_range(value, field) unless value.finite?
73
+
74
+ value == value.to_i ? value.to_i : value
75
+ end
76
+
77
+ # `time_is_after` receives a timestamp everywhere else in this datasource,
78
+ # a native date column being read and filtered as one: a Dateonly custom
79
+ # field cannot be the one field sending the same operator another shape.
80
+ # The bound is the one a Ruby `Date` already gets -- midnight in the
81
+ # timezone of the caller.
82
+ #
83
+ # A string this operator cannot read as a date is left to Pylon, which
84
+ # names what it refuses better than a guess here would.
85
+ def format_time_string(value)
86
+ return value unless DATE_ONLY.match?(value)
87
+
88
+ format_date(Date.parse(value))
89
+ rescue Date::Error
90
+ value
91
+ end
92
+
93
+ # Only reached by a condition tree built in Ruby -- a segment or a scope
94
+ # written as code -- and by the bare date above. Everything else coming
95
+ # through HTTP arrives as an ISO8601 timestamp, already expressed in the
96
+ # timezone of the caller: the agent casts a date filter with `value.to_s`,
97
+ # and the toolkit formats the bounds it derives from Today / Previous*
98
+ # itself.
99
+ def format_date(value)
100
+ Time.use_zone(@timezone) { Time.zone.local(value.year, value.month, value.day).utc.iso8601 }
101
+ rescue ArgumentError
102
+ ForestAdminDatasourcePylon.logger.warn(
103
+ "[forest_admin_datasource_pylon] unknown timezone '#{@timezone}', falling back to UTC"
104
+ )
105
+ value.strftime('%Y-%m-%dT00:00:00Z')
106
+ end
107
+
108
+ # A filter carrying a nil value reads as a presence check on most APIs,
109
+ # which is silently the wrong query.
110
+ def raise_nil_value(field)
111
+ raise UnsupportedOperatorError,
112
+ "Filter value on '#{field}' is nil; use the PRESENT or BLANK operator to filter for absence."
113
+ end
114
+
115
+ def raise_blank_in_list(leaf)
116
+ raise UnsupportedOperatorError,
117
+ "Operator '#{leaf.operator}' on field '#{leaf.field}' was given a list holding a blank value; " \
118
+ 'Pylon matches an absent value through is_set / is_unset only, so filter for absence with the ' \
119
+ 'PRESENT or BLANK operator rather than listing nil or an empty string.'
120
+ end
121
+
122
+ def raise_empty_list(leaf)
123
+ raise UnsupportedOperatorError,
124
+ "Operator '#{leaf.operator}' on field '#{leaf.field}' was given an empty list; " \
125
+ 'pass at least one value.'
126
+ end
127
+
128
+ def raise_out_of_range(value, field)
129
+ raise UnsupportedOperatorError,
130
+ "Filter value on '#{field}' is #{value}, which is not a number Pylon can be asked for; " \
131
+ 'filter with a value inside the range of a double.'
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,108 @@
1
+ module ForestAdminDatasourcePylon
2
+ module Query
3
+ # The operator maps the Pylon search endpoints share. A map spells each
4
+ # Forest operator as the Pylon operator honouring it; a collection's
5
+ # `API_FILTERS` table then assembles, field by field, the maps its endpoint
6
+ # accepts according to the API reference.
7
+ #
8
+ # Sharing the maps is what keeps those tables readable as the allow-lists
9
+ # they transcribe, and keeps one wire spelling from being fixed in one
10
+ # collection and left wrong in the next.
11
+ module OperatorMaps
12
+ Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
13
+
14
+ # `not_equal` is deliberately absent: the toolkit derives it from
15
+ # `not_in`, so declaring it would only add a second spelling.
16
+ EQUALITY = { Operators::EQUAL => 'equals',
17
+ Operators::IN => 'in',
18
+ Operators::NOT_IN => 'not_in' }.freeze
19
+
20
+ # MISSING is mapped as well although Pylon spells absence one way only:
21
+ # the toolkit rewrites it into `equal nil` when it is left out, which the
22
+ # translator cannot express, so a field the API does check for absence
23
+ # would refuse the very filter it can answer.
24
+ PRESENCE = { Operators::PRESENT => 'is_set',
25
+ Operators::BLANK => 'is_unset',
26
+ Operators::MISSING => 'is_unset' }.freeze
27
+
28
+ # Declaring the bare comparisons rather than before/after is what lets
29
+ # the toolkit rewrite Today / PreviousWeek / ... into a pair of bounds,
30
+ # which is also why `time_range` never has to be emitted.
31
+ TIME = { Operators::GREATER_THAN => 'time_is_after',
32
+ Operators::LESS_THAN => 'time_is_before' }.freeze
33
+
34
+ # Pylon exposes a single substring operator and documents no case
35
+ # semantics for it, so both Forest spellings map onto that one operator --
36
+ # or the UI would offer a case-sensitive "contains" and a case-insensitive
37
+ # one behaving identically.
38
+ SUBSTRING = { Operators::CONTAINS => 'string_contains',
39
+ Operators::I_CONTAINS => 'string_contains' }.freeze
40
+
41
+ # The same, for the endpoints that also accept the negation. A field
42
+ # filtered through SUBSTRING alone must not advertise it: Pylon rejects
43
+ # `string_does_not_contain` where it is not documented.
44
+ FULL_TEXT = SUBSTRING.merge(Operators::NOT_CONTAINS => 'string_does_not_contain',
45
+ Operators::NOT_I_CONTAINS => 'string_does_not_contain').freeze
46
+
47
+ # A list-valued field -- `tags`, `domains`: `in` matches it against
48
+ # several candidates at once.
49
+ #
50
+ # Pylon also accepts `contains` / `does_not_contain` on such a field, and
51
+ # they are deliberately left out: the columns are typed `Json`, the only
52
+ # type the toolkit has for a list, and `Rules` allows a Json column the
53
+ # base and array operators alone. A declared `contains` would be refused
54
+ # by `ConditionTreeValidator` on the way in -- "the given operator
55
+ # 'contains' is not allowed with the columnType schema: 'Json'" -- so the
56
+ # UI would offer a filter that errors instead of one Pylon answers.
57
+ # Typing the columns `['String']` is not the way out either: no branch of
58
+ # `get_allowed_operators_for_column_type` reads an array type, and the
59
+ # validator raises a NoMethodError on it. Reaching those two operators
60
+ # takes a toolkit change, which is not this datasource's to make here.
61
+ MEMBERSHIP = { Operators::IN => 'in',
62
+ Operators::NOT_IN => 'not_in' }.freeze
63
+
64
+ # A custom field is filtered through its slug, so its operators come from
65
+ # the column the integrator declared rather than from a table; every
66
+ # search endpoint accepts this same set on one.
67
+ CUSTOM_FIELD_OPS = EQUALITY.merge(PRESENCE).merge(TIME).merge(FULL_TEXT).freeze
68
+
69
+ # Extended by a collection's `ApiFilters` module, whose `API_FILTERS` is
70
+ # the single source of truth for what its endpoint filters: the schema
71
+ # derives every column's `filter_operators` from it, so no collection
72
+ # declares a filter the translator would then refuse.
73
+ #
74
+ # One family escapes those tables. The agent derives `present`, `blank` and
75
+ # `missing` from an equality or a membership filter, above the datasource,
76
+ # and rewrites them into a comparison with an empty value. Only a field
77
+ # carrying PRESENCE can answer one -- Pylon matches an absent value through
78
+ # `is_set` / `is_unset` alone -- so on every other field the translator
79
+ # refuses the rewritten condition and names the filter to change, rather
80
+ # than sending a comparison Pylon would answer as if the empty value were
81
+ # a value of its own.
82
+ module Table
83
+ def forest_operators(field)
84
+ self::API_FILTERS.dig(field, :ops)&.keys || []
85
+ end
86
+
87
+ # Read off the extending module rather than off this one, so the
88
+ # `CUSTOM_FIELD_OPS` a collection declares is the single source both
89
+ # this spelling and `allowed_custom_field_operators` come from: an
90
+ # endpoint accepting less on a custom field narrows one constant.
91
+ def for_custom_field(schema)
92
+ { ops: self::CUSTOM_FIELD_OPS.slice(*Array(schema&.filter_operators)) }
93
+ end
94
+ end
95
+
96
+ # The table of a collection whose endpoint filters nothing server-side: no
97
+ # field, and no operator on a custom field either. It is the default of
98
+ # `BaseCollection#filter_table`, so a collection read whole and filtered
99
+ # in memory needs no table of its own.
100
+ module EmptyTable
101
+ extend Table
102
+
103
+ API_FILTERS = {}.freeze
104
+ CUSTOM_FIELD_OPS = {}.freeze
105
+ end
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,139 @@
1
+ module ForestAdminDatasourcePylon
2
+ # Spaces requests out so the budget of an endpoint is spent rather than
3
+ # exceeded: a sliding window per endpoint, and a wait until the next slot when
4
+ # the window is full.
5
+ #
6
+ # This sits in front of the 429 retry rather than replacing it. The retry
7
+ # answers a 429 Pylon already sent, which costs a full rate-limit window to
8
+ # recover from (Retry-After runs up to 60s); waiting half a second for a slot
9
+ # is the same throughput at a fraction of the latency. The retry stays as the
10
+ # backstop for what this cannot see — another process, or another agent, on
11
+ # the same token.
12
+ #
13
+ # One limiter per Configuration, so per token: Pylon meters the token, and two
14
+ # agents holding different ones do not share a budget.
15
+ class RateLimiter
16
+ WINDOW = 60.0
17
+
18
+ # How long one attempt may be held back before it is let through anyway. The
19
+ # limiter exists to avoid a 429, not to guarantee one never happens: past
20
+ # this point the window is saturated by more than this agent's own traffic,
21
+ # so queueing behind it would trade a retry the client already handles for a
22
+ # request the operator watches spin. It goes out, and the warning says why.
23
+ #
24
+ # Per attempt, not per request: `retry` replays up to `max_retries` times and
25
+ # each replay asks for a slot of its own, so a request can spend this bound
26
+ # once per attempt, on top of the backoff `retry` waits itself. None of it
27
+ # runs under the Faraday timeout, which only covers the adapter.
28
+ #
29
+ # The region this smooths is narrow, and it is worth being plain about it.
30
+ # On a stream over budget the waits accumulate rather than settling, so the
31
+ # bound is crossed sooner the further over it sits: measured against a
32
+ # 120/min endpoint, a stream 1% over budget throttles its first thousand
33
+ # requests and then lets roughly one in twelve through unthrottled, one 5%
34
+ # over gives up after two hundred and lets half through, and past ~10% over
35
+ # the bound is crossed as soon as the window fills — request 121 — after
36
+ # which almost nothing is throttled at all. A burst arriving at once books
37
+ # its next slot a whole window out and goes straight through from the first
38
+ # request past the budget. Under real saturation the 429 retry is the
39
+ # defence, not this; what this buys is the region just over budget, and a
40
+ # budget the code knows rather than one it discovers as a 429.
41
+ DEFAULT_MAX_WAIT = 5.0
42
+
43
+ attr_reader :max_wait, :window
44
+
45
+ def initialize(limits: RateLimits, window: WINDOW, max_wait: DEFAULT_MAX_WAIT, clock: nil, sleeper: nil)
46
+ @limits = limits
47
+ @window = window.to_f
48
+ @max_wait = max_wait.to_f
49
+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
50
+ @sleeper = sleeper || ->(seconds) { sleep(seconds) }
51
+ @mutex = Mutex.new
52
+ @slots = {}
53
+ @warned = {}
54
+ end
55
+
56
+ # Blocks until the endpoint has room, then returns. Called once per attempt,
57
+ # retries included: a replayed request spends the budget a first one did.
58
+ def acquire(method, path)
59
+ rule = @limits.for(method, path)
60
+ wait, warn = @mutex.synchronize { reserve(rule) }
61
+
62
+ warn_saturated(rule, wait) if warn
63
+ return if wait <= 0 || wait > @max_wait
64
+
65
+ @sleeper.call(wait)
66
+ end
67
+
68
+ private
69
+
70
+ # The reservation happens under the lock and the waiting outside it: holding
71
+ # the mutex across the sleep would serialize every thread behind the slowest
72
+ # one, and threads waiting on unrelated endpoints have no reason to queue.
73
+ #
74
+ # What is recorded is the moment the request will be made, not the moment it
75
+ # was asked for, so concurrent callers each take a distinct slot and spread
76
+ # out instead of all waking onto the same one.
77
+ #
78
+ # Returns the wait the caller owes and whether this is the bypass worth a log
79
+ # line — both settled here, the second being shared state like the first.
80
+ def reserve(rule)
81
+ taken = (@slots[rule.name] ||= [])
82
+ now = @clock.call
83
+ expire(taken, now)
84
+
85
+ slot = next_slot(taken, rule.limit, now)
86
+ wait = slot - now
87
+ bypass = wait > @max_wait
88
+ # Past the bound the request goes out now, so the slot it books is now:
89
+ # recording the one it declined to wait for would meter a request nobody
90
+ # ever made and push the whole window further out.
91
+ insert(taken, bypass ? now : slot)
92
+
93
+ [wait, bypass && first_warning?(rule, now)]
94
+ end
95
+
96
+ # The bookings that have left the window are its leading run, the list being
97
+ # kept ordered.
98
+ def expire(taken, now)
99
+ taken.shift(taken.bsearch_index { |at| at > now - @window } || taken.size)
100
+ end
101
+
102
+ # Now while the window still has room, otherwise a window past the limit-th
103
+ # most recent booking, which is the one that has to fall out of it first —
104
+ # an index that only reads as such on an ordered list.
105
+ def next_slot(taken, limit, now)
106
+ return now if taken.size < limit
107
+
108
+ taken[taken.size - limit] + @window
109
+ end
110
+
111
+ # A booking has one position in an ordered list, so it goes there rather than
112
+ # onto the end followed by a sort: `now` lands before the slots already
113
+ # reserved further out, and the list is what `next_slot` reads an index off.
114
+ def insert(taken, booking)
115
+ taken.insert(taken.bsearch_index { |at| at >= booking } || taken.size, booking)
116
+ end
117
+
118
+ # One line per endpoint per window. What the warning reports is a saturation
119
+ # that lasts, so a line per request puts one on every request it describes —
120
+ # a thousand of them for a couple of minutes over budget, burying the first,
121
+ # which is the only one the operator needed.
122
+ def first_warning?(rule, now)
123
+ last = @warned[rule.name]
124
+ return false if last && now - last < @window
125
+
126
+ @warned[rule.name] = now
127
+ true
128
+ end
129
+
130
+ def warn_saturated(rule, wait)
131
+ ForestAdminDatasourcePylon.logger.warn(
132
+ "[forest_admin_datasource_pylon] #{rule.name} is at its budget of #{rule.limit} requests per " \
133
+ "#{@window.round}s; the next slot is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \
134
+ 'Letting the request through — Pylon may answer 429, which the client retries. Further requests over ' \
135
+ "this budget are let through too, and this says so once per #{@window.round}s."
136
+ )
137
+ end
138
+ end
139
+ end
@@ -0,0 +1,96 @@
1
+ module ForestAdminDatasourcePylon
2
+ # The per-endpoint budgets Pylon documents, in requests per minute, read off
3
+ # the API reference (docs.usepylon.com/pylon-docs/developer/api/api-reference,
4
+ # one page per resource — each endpoint states its own "Rate limit:" line).
5
+ #
6
+ # Pylon meters per endpoint, not per token, so every rule owns its own window:
7
+ # two endpoints allowing 300 each are 600 requests, and pooling them would
8
+ # throttle at half the budget the API grants.
9
+ #
10
+ # An endpoint absent from the table falls back to DEFAULT_LIMIT, the lowest
11
+ # figure documented anywhere on the API. An undocumented quota is not an
12
+ # absent one, and the generous guess is the one an operator discovers as a 429.
13
+ #
14
+ # Every endpoint the client calls has a rule, writes included: the budget of a
15
+ # write is no more uniform than that of a read — `POST /accounts` is granted
16
+ # ten times `POST /issues`, and `PATCH /accounts/{id}` two and a half times
17
+ # `PATCH /issues/{id}` — so leaving one to the fallback throttles it at a
18
+ # fraction of what Pylon grants, which is the failure this table exists to
19
+ # avoid rather than a safe default.
20
+ class RateLimits
21
+ DEFAULT_LIMIT = 30
22
+
23
+ # Ids are escaped before being joined to a path, so a segment never carries
24
+ # a slash and this matches exactly one of them.
25
+ ID = '[^/]+'.freeze
26
+
27
+ Rule = Struct.new(:name, :limit, keyword_init: true)
28
+
29
+ # Every rule is anchored, so `/issues`, `/issues/{id}` and
30
+ # `/issues/{id}/messages` are three distinct buckets rather than three
31
+ # readings of the same prefix, and the order of this list carries no meaning.
32
+ #
33
+ # `name` is what the window is keyed on and what a log line shows, so it
34
+ # spells the endpoint rather than its budget: two endpoints sharing a figure
35
+ # must not share a window.
36
+ RULES = [
37
+ ['post /issues/search', :post, %r{\A/issues/search\z}, 120],
38
+ ['post /accounts/search', :post, %r{\A/accounts/search\z}, 120],
39
+ ['post /contacts/search', :post, %r{\A/contacts/search\z}, 120],
40
+ ['get /issues/:id/messages', :get, %r{\A/issues/#{ID}/messages\z}, 120],
41
+ ['get /issues', :get, %r{\A/issues\z}, 30],
42
+ ['post /issues', :post, %r{\A/issues\z}, 30],
43
+ ['get /issues/:id', :get, %r{\A/issues/#{ID}\z}, 300],
44
+ ['patch /issues/:id', :patch, %r{\A/issues/#{ID}\z}, 120],
45
+ ['get /accounts', :get, %r{\A/accounts\z}, 300],
46
+ ['get /accounts/:id', :get, %r{\A/accounts/#{ID}\z}, 300],
47
+ ['get /contacts', :get, %r{\A/contacts\z}, 300],
48
+ ['get /contacts/:id', :get, %r{\A/contacts/#{ID}\z}, 300],
49
+ ['get /users', :get, %r{\A/users\z}, 300],
50
+ ['get /users/:id', :get, %r{\A/users/#{ID}\z}, 300],
51
+ ['get /teams', :get, %r{\A/teams\z}, 300],
52
+ ['get /teams/:id', :get, %r{\A/teams/#{ID}\z}, 300],
53
+ ['get /custom-fields', :get, %r{\A/custom-fields\z}, 300],
54
+ ['get /me', :get, %r{\A/me\z}, 300],
55
+ ['post /accounts', :post, %r{\A/accounts\z}, 300],
56
+ ['patch /accounts/:id', :patch, %r{\A/accounts/#{ID}\z}, 300],
57
+ ['post /contacts', :post, %r{\A/contacts\z}, 300],
58
+ ['patch /contacts/:id', :patch, %r{\A/contacts/#{ID}\z}, 300],
59
+ ['delete /issues/:id', :delete, %r{\A/issues/#{ID}\z}, 120],
60
+ ['patch /teams/:id', :patch, %r{\A/teams/#{ID}\z}, 120],
61
+ ['patch /users/:id', :patch, %r{\A/users/#{ID}\z}, 120],
62
+ ['delete /accounts/:id', :delete, %r{\A/accounts/#{ID}\z}, 30],
63
+ ['delete /contacts/:id', :delete, %r{\A/contacts/#{ID}\z}, 30],
64
+ ['post /teams', :post, %r{\A/teams\z}, 30]
65
+ ].map { |name, verb, pattern, limit| [verb, pattern, Rule.new(name: name, limit: limit)] }.freeze
66
+
67
+ class << self
68
+ def for(method, path)
69
+ verb = method.to_s.downcase.to_sym
70
+ normalized = normalize(path)
71
+
72
+ found = RULES.find { |rule_verb, pattern, _rule| rule_verb == verb && pattern.match?(normalized) }
73
+ found ? found[2] : fallback(verb, normalized)
74
+ end
75
+
76
+ private
77
+
78
+ # Faraday hands back the path of the resolved URL, which carries the
79
+ # leading slash the rules are written against and, on a base url mounted
80
+ # under a subpath, whatever precedes it.
81
+ def normalize(path)
82
+ stripped = path.to_s.chomp('/')
83
+ stripped.start_with?('/') ? stripped : "/#{stripped}"
84
+ end
85
+
86
+ # An unlisted endpoint is bucketed by its first segment rather than by its
87
+ # full path: keying on the path would open a window per record id, so a
88
+ # fan-out over a hundred records would meter as a hundred endpoints each
89
+ # one request in.
90
+ def fallback(verb, path)
91
+ segment = path.split('/').reject(&:empty?).first
92
+ Rule.new(name: "#{verb} /#{segment} (undocumented)", limit: DEFAULT_LIMIT)
93
+ end
94
+ end
95
+ end
96
+ end