forest_admin_datasource_intercom 1.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/LICENSE +202 -0
  4. data/README.md +811 -0
  5. data/Rakefile +6 -0
  6. data/exe/forest_admin_intercom_probe +287 -0
  7. data/forest_admin_datasource_intercom.gemspec +38 -0
  8. data/lib/forest_admin_datasource_intercom/cache.rb +84 -0
  9. data/lib/forest_admin_datasource_intercom/client.rb +546 -0
  10. data/lib/forest_admin_datasource_intercom/collections/admin.rb +83 -0
  11. data/lib/forest_admin_datasource_intercom/collections/base_collection.rb +161 -0
  12. data/lib/forest_admin_datasource_intercom/collections/company/serializer.rb +44 -0
  13. data/lib/forest_admin_datasource_intercom/collections/company.rb +69 -0
  14. data/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb +106 -0
  15. data/lib/forest_admin_datasource_intercom/collections/contact.rb +266 -0
  16. data/lib/forest_admin_datasource_intercom/collections/contact_identity.rb +77 -0
  17. data/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb +75 -0
  18. data/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb +40 -0
  19. data/lib/forest_admin_datasource_intercom/collections/conversation.rb +153 -0
  20. data/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +363 -0
  21. data/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb +55 -0
  22. data/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb +196 -0
  23. data/lib/forest_admin_datasource_intercom/collections/offset_collection.rb +311 -0
  24. data/lib/forest_admin_datasource_intercom/collections/records_by_id.rb +97 -0
  25. data/lib/forest_admin_datasource_intercom/collections/relations.rb +294 -0
  26. data/lib/forest_admin_datasource_intercom/collections/team.rb +70 -0
  27. data/lib/forest_admin_datasource_intercom/collections/team_membership.rb +51 -0
  28. data/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb +130 -0
  29. data/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb +70 -0
  30. data/lib/forest_admin_datasource_intercom/collections/ticket.rb +143 -0
  31. data/lib/forest_admin_datasource_intercom/collections/ticket_state.rb +40 -0
  32. data/lib/forest_admin_datasource_intercom/collections/ticket_type.rb +46 -0
  33. data/lib/forest_admin_datasource_intercom/collections/timeline.rb +40 -0
  34. data/lib/forest_admin_datasource_intercom/configuration.rb +226 -0
  35. data/lib/forest_admin_datasource_intercom/datasource.rb +78 -0
  36. data/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb +123 -0
  37. data/lib/forest_admin_datasource_intercom/query/caller_zone.rb +48 -0
  38. data/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb +169 -0
  39. data/lib/forest_admin_datasource_intercom/query/day_bounds.rb +75 -0
  40. data/lib/forest_admin_datasource_intercom/query/filter_value.rb +177 -0
  41. data/lib/forest_admin_datasource_intercom/query/operator_table.rb +54 -0
  42. data/lib/forest_admin_datasource_intercom/query/search_fields.rb +225 -0
  43. data/lib/forest_admin_datasource_intercom/query/search_fields.yml +663 -0
  44. data/lib/forest_admin_datasource_intercom/rate_limiter.rb +148 -0
  45. data/lib/forest_admin_datasource_intercom/retry_policy.rb +95 -0
  46. data/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb +52 -0
  47. data/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb +88 -0
  48. data/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb +133 -0
  49. data/lib/forest_admin_datasource_intercom/throttle.rb +19 -0
  50. data/lib/forest_admin_datasource_intercom/version.rb +3 -0
  51. data/lib/forest_admin_datasource_intercom.rb +57 -0
  52. metadata +180 -0
@@ -0,0 +1,363 @@
1
+ module ForestAdminDatasourceIntercom
2
+ module Collections
3
+ # Cursor-paginated tier: only a page is in hand, so nothing is filtered, sorted or counted in
4
+ # memory. Whatever the search DSL cannot express is refused, never served unfiltered.
5
+ # Long by line count only: half of it is the refusals and their remedies.
6
+ class CursorCollection < BaseCollection # rubocop:disable Metrics/ClassLength
7
+ include RecordsById
8
+
9
+ # Countable, and exactly: unlike the pages a walk collected, `total_count`
10
+ # is the whole dataset the filter names.
11
+ def initialize(datasource, name)
12
+ super
13
+ enable_count
14
+ end
15
+
16
+ # One request per id here, so a relation pointing at this tier resolves a
17
+ # slice at a time and is refused past what a slice may hold -- see
18
+ # `Relations#target_rows`. The collection that reads its ids in bulk
19
+ # widens both; see `Contact`.
20
+ def ids_per_read = max_id_reads
21
+ def max_resolvable_ids = max_id_reads
22
+
23
+ # The read scope spans the whole page rather than any one step of it: the
24
+ # enrichment and the relation embed below ask Intercom for the same
25
+ # contacts, from two places that cannot see each other. See
26
+ # `Client#with_read_scope`.
27
+ def list(caller, filter, projection)
28
+ client.with_read_scope do
29
+ records = fetch_records(caller, filter, server_sort(filter))
30
+ # Serialized whole and projected afterwards rather than the other way
31
+ # round: a projection reaching through a relation names no foreign
32
+ # key, and the key is where the relation is read from.
33
+ serialized = records.map { |record| serialize(record) }
34
+ rows = serialized.map { |record| project(record, projection) }
35
+
36
+ enrich(records, rows, projection)
37
+ embed_relations(caller, serialized, rows, projection)
38
+ rows
39
+ end
40
+ end
41
+
42
+ # Count only, and never a group: Intercom exposes no aggregate endpoint,
43
+ # and grouping over the pages a walk happened to collect would look exact
44
+ # while answering a fraction. Refused here rather than through the
45
+ # contract's NotImplementedError, which reads as an oversight.
46
+ def aggregate(caller, filter, aggregation, _limit = nil)
47
+ refuse_unsupported_aggregation!(aggregation)
48
+
49
+ [{ 'group' => {}, 'value' => count_records(caller, filter) }]
50
+ end
51
+
52
+ protected
53
+
54
+ def list_endpoint = raise(NotImplementedError, "#{self.class} did not implement list_endpoint")
55
+ def record_endpoint = list_endpoint
56
+ def list_key = 'data'
57
+ def read_params = {}
58
+
59
+ # The row of the measured table this collection is filtered through: what
60
+ # its columns may advertise, and what the translator is allowed to write.
61
+ def searchable = raise(NotImplementedError, "#{self.class} did not implement searchable")
62
+
63
+ def search_endpoint
64
+ @search_endpoint ||= Query::SearchFields.fetch(searchable)
65
+ end
66
+
67
+ # The column a free-text search is answered on, for a collection whose
68
+ # endpoint has one. Nil elsewhere, and a search is then refused rather than
69
+ # answered by a page that ignored it.
70
+ def search_column = nil
71
+
72
+ def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize")
73
+
74
+ # Hook for what a row needs beyond its own payload. Left empty here: what
75
+ # it costs is the collection's business, not this base's.
76
+ def enrich(_records, _rows, _projection); end
77
+
78
+ # Bounded per collection rather than by the API maximum: Intercom offers no
79
+ # field selection, so a collection whose rows carry their whole timeline
80
+ # pays for it by the page. See Ticket.
81
+ def max_page_size = Client::MAX_PER_PAGE
82
+
83
+ # One page of the collection. A listing for conversations, a search for
84
+ # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint
85
+ # and its shape belong to the collection, while walking it does not.
86
+ def read_page(per_page:, cursor:, query: nil, sort: nil)
87
+ size = [per_page, max_page_size].min
88
+ # The listing endpoint neither filters nor sorts, and a collection whose
89
+ # records are only reachable through the search has no listing at all.
90
+ # Either way what routes the read to the search is a query, so the one
91
+ # that matches everything stands in for the condition there is none of.
92
+ query ||= match_all_query if sort || searchable_only?
93
+
94
+ if query.nil?
95
+ client.list_page(list_endpoint, per_page: size, starting_after: cursor,
96
+ params: read_params, list_key: list_key)
97
+ else
98
+ client.search_page(search_endpoint.path, query: query, per_page: size, starting_after: cursor,
99
+ params: read_params, list_key: list_key, sort: sort)
100
+ end
101
+ end
102
+
103
+ # Operators come from the measured table only, so no column offers a filter the translator
104
+ # would refuse. Only `/contacts/search` honours a sort; the other search endpoints silently
105
+ # ignore one, so their columns are never sortable.
106
+ def add_column(name, type, is_primary_key: false)
107
+ add_field(name, ColumnSchema.new(column_type: type,
108
+ filter_operators: column_operators(name, is_primary_key),
109
+ is_primary_key: is_primary_key,
110
+ is_read_only: true,
111
+ is_sortable: sortable_column?(name),
112
+ is_groupable: false))
113
+ end
114
+
115
+ def sortable_column?(name)
116
+ search_endpoint.field(name)&.sortable? == true
117
+ end
118
+
119
+ # The Intercom query standing in for the condition a read has none of.
120
+ # Two collections need one and for different reasons: Intercom exposes no
121
+ # `GET /tickets` at all, and an order is applied by the search endpoint
122
+ # alone. Nil where the listing endpoint answers both, which is where a
123
+ # sort is reported as ignored rather than dropped.
124
+ def match_all_query = nil
125
+
126
+ def searchable_only? = false
127
+
128
+ # Bounded, unlike the tier read whole: one more than a group may hold is
129
+ # all it takes to know the fan-out will not fit, and reading further would
130
+ # walk a whole collection to refuse it afterwards.
131
+ def match_page
132
+ ForestAdminDatasourceToolkit::Components::Query::Page.new(
133
+ offset: 0, limit: Query::ConditionTreeTranslator::MAX_GROUP_SIZE + 1
134
+ )
135
+ end
136
+
137
+ def walker
138
+ @walker ||= Pagination::CursorWalker.new
139
+ end
140
+
141
+ # How many records a bulk read by id may fetch. A hook rather than the
142
+ # constant, since the collection that reads its ids in bulk answers far
143
+ # more of them for the same request count.
144
+ def max_id_reads = MAX_ID_READS
145
+
146
+ # Columns answered by a route of the collection's own rather than by its
147
+ # search endpoint, with the operators that route takes.
148
+ def routed_operators = {}
149
+
150
+ private
151
+
152
+ def column_operators(name, is_primary_key)
153
+ return [Operators::EQUAL, Operators::IN] if is_primary_key
154
+ return routed_operators[name] if routed_operators.key?(name)
155
+
156
+ field = search_endpoint.field(name)
157
+
158
+ field ? Query::OperatorTable.forest_operators(field) : []
159
+ end
160
+
161
+ def fetch_records(caller, filter, sort = nil)
162
+ ids = id_lookup(filter)
163
+ # The window is cut out of the ids rather than out of the records they
164
+ # read: Intercom reads them one request each, so paging after the read
165
+ # would pay for a whole page to hand back a slice of it -- and page 2 of
166
+ # a set larger than the cap would come back empty, the records it names
167
+ # having been dropped by the truncation before the window was applied.
168
+ if ids
169
+ # Which is also why an order cannot be honoured on this route: the
170
+ # window is cut in the order the ids were named, so sorting what comes
171
+ # back would order a slice picked by something else. Reported rather
172
+ # than dropped in silence, like every other order this tier cannot
173
+ # apply -- `server_sort` stayed quiet, having found a column Intercom
174
+ # does sort.
175
+ warn_unordered_ids(sort) if sort
176
+ return records_by_ids(page_window(ids, filter))
177
+ end
178
+
179
+ query = translate(caller, filter)
180
+ # A condition through a relation the target matched no record with names
181
+ # no row, and Intercom's DSL cannot say so: the read is skipped rather
182
+ # than sent as a filter that would come back with everything.
183
+ return [] if query == NOTHING
184
+
185
+ listed_records(filter, query, sort)
186
+ end
187
+
188
+ # The Intercom query a filter comes down to, or nil for a list view, which
189
+ # walks the listing endpoint instead. The free-text search is folded into
190
+ # the condition tree rather than added to the translated query: written as
191
+ # one tree, it is checked against the nesting Intercom allows like every
192
+ # other condition, instead of adding a level nothing counted.
193
+ def translate(caller, filter)
194
+ tree = rewrite_relation_conditions(caller, combined_tree(filter)) do |key, ids, leaf|
195
+ relation_group(key, ids, leaf)
196
+ end
197
+ return NOTHING if tree == NOTHING
198
+
199
+ Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name,
200
+ timezone: timezone_for(caller),
201
+ attribute_columns: attribute_column_names)
202
+ end
203
+
204
+ # An OR of equalities: the DSL has no IN on foreign keys, and caps a group at 15 conditions,
205
+ # so a larger match is refused here where the relation can be named. A RelationBranch so it
206
+ # can be inlined into a same-aggregator parent, saving a nesting level.
207
+ def relation_group(key, ids, leaf)
208
+ refuse_fan_out!(leaf, key, ids) if ids.size > Query::ConditionTreeTranslator::MAX_GROUP_SIZE
209
+ return Leaf.new(key, Operators::EQUAL, ids.first) if ids.size == 1
210
+
211
+ RelationBranch.new('Or', ids.map { |id| Leaf.new(key, Operators::EQUAL, id) }, leaf.field)
212
+ end
213
+
214
+ # Inlining a relation group into its parent trades a level of nesting for
215
+ # width, and Intercom bounds both. Past the conditions a group may hold,
216
+ # the nested form is the one it answers.
217
+ def absorb_relation_group?(size)
218
+ size <= Query::ConditionTreeTranslator::MAX_GROUP_SIZE
219
+ end
220
+
221
+ # A relation this endpoint filters nothing through. It is navigable all the
222
+ # same -- the read costs nothing, the target being read whole -- and saying
223
+ # which of the two it is, is the whole point of the message.
224
+ def relation_key_filterable?(key) = !search_endpoint.field(key).nil?
225
+
226
+ def check_relation_filterable!(leaf, relation)
227
+ key = relation.foreign_key
228
+ refuse_unfilterable_key!(leaf, key) unless relation_key_filterable?(key)
229
+ end
230
+
231
+ def refuse_unfilterable_key!(leaf, key)
232
+ raise UnsupportedOperatorError,
233
+ "#{name} cannot filter #{leaf.field.inspect}: the relation resolves to #{key.inspect}, on " \
234
+ "which #{search_endpoint.path} takes no filter. The relation is there to be read and " \
235
+ "navigated; filter on one of: #{search_endpoint.filterable_columns.join(", ")}."
236
+ end
237
+
238
+ def refuse_fan_out!(leaf, key, _ids)
239
+ # "more than", never a count: the target is read one record past what a
240
+ # group may hold, so what is known is that it does not fit -- printing
241
+ # 16 where a workspace holds three thousand would read as a number the
242
+ # operator could go and narrow by one.
243
+ raise UnsupportedOperatorError,
244
+ "#{name} cannot filter #{leaf.field.inspect}: it names more than " \
245
+ "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} records, " \
246
+ "#{search_endpoint.path} answers #{key.inspect} one value at a time, and Intercom takes " \
247
+ "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} conditions per group. Narrow the condition " \
248
+ "on the relation, or filter on #{key.inspect} itself."
249
+ end
250
+
251
+ def combined_tree(filter)
252
+ conditions = [filter&.condition_tree, search_condition(filter)].compact
253
+ return conditions.first if conditions.size < 2
254
+
255
+ ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory.intersect(conditions)
256
+ end
257
+
258
+ # A free-text search reaches Intercom as a condition on the one column its
259
+ # endpoint matches text on -- per word, not as a substring, which the
260
+ # README says rather than the interface implying otherwise.
261
+ def search_condition(filter)
262
+ return nil if blank_search?(filter)
263
+
264
+ refuse_search! if search_column.nil?
265
+
266
+ Leaf.new(search_column, Operators::CONTAINS, filter.search.to_s.strip)
267
+ end
268
+
269
+ # The records of this tier carry bodies written by end customers, so every
270
+ # read of one asks Intercom for plain text (R10). `BaseCollection` reads
271
+ # this on the way to the record endpoint.
272
+ def record_read_params = read_params
273
+
274
+ def listed_records(filter, query, sort = nil)
275
+ offset, limit = translate_page(filter&.page)
276
+
277
+ walker.walk(offset: offset, limit: limit) do |per_page, cursor|
278
+ read_page(per_page: per_page, cursor: cursor, query: query, sort: sort)
279
+ end
280
+ end
281
+
282
+ # A filter with no page asks for every record it matched; the walker reads
283
+ # that as the nil limit it bounds with its own caps.
284
+ def translate_page(page)
285
+ return [0, nil] if page.nil?
286
+
287
+ [page.offset.to_i.clamp(0, nil), page_limit(page)]
288
+ end
289
+
290
+ # Exact, and one request: `total_count` counts what the filter names, not
291
+ # what a page happened to hold. An id lookup counts the records it found,
292
+ # which is cheaper still.
293
+ def count_records(caller, filter)
294
+ ids = id_lookup(filter)
295
+ return count_by_ids(ids) if ids
296
+
297
+ query = translate(caller, filter)
298
+ return 0 if query == NOTHING
299
+
300
+ exact_count(read_page(per_page: 1, cursor: nil, query: query))
301
+ end
302
+
303
+ def refuse_search!
304
+ raise UnsupportedOperatorError,
305
+ "#{name} cannot answer a free-text search: #{search_endpoint.path} matches values field by field, " \
306
+ 'and this collection exposes no text column it searches. Filter on a column instead of searching.'
307
+ end
308
+
309
+ # The order Intercom will really apply, written the way the client sends
310
+ # it -- or nil, and the order the operator asked for is then reported
311
+ # rather than dropped in silence.
312
+ #
313
+ # The ascending primary-key sort the agent injects when a request names
314
+ # none is neither honoured nor reported: it is not an order anybody asked
315
+ # for, and `/contacts/search` does not sort on an id anyway.
316
+ def server_sort(filter)
317
+ clauses = Array(filter&.sort)
318
+ return nil if clauses.empty? || default_pk_sort?(clauses)
319
+
320
+ honoured = honourable_sort(clauses)
321
+ warn_ignored_sort(clauses) if honoured.nil?
322
+ honoured
323
+ end
324
+
325
+ # One clause, on a column the measured table says the endpoint sorts.
326
+ # Intercom takes a single `{ field, order }` and nothing composite, so a
327
+ # second clause is not half-honoured: honouring the first alone would
328
+ # order the page by something the operator did not ask for.
329
+ def honourable_sort(clauses)
330
+ return nil unless clauses.size == 1
331
+
332
+ clause = clauses.first
333
+ field = search_endpoint.field(sort_field(clause).to_s)
334
+ return nil unless field&.sortable?
335
+
336
+ { field: field.field, ascending: ascending?(clause) }
337
+ end
338
+
339
+ # Intercom accepts a `sort` on the other two endpoints and ignores it
340
+ # without a word -- measured -- so an order asked for and not applied has
341
+ # to be reported here or nowhere.
342
+ def warn_ignored_sort(clauses)
343
+ ForestAdminDatasourceIntercom.logger.warn(
344
+ "[forest_admin_datasource_intercom] #{name} was asked to sort on " \
345
+ "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, which Intercom does not sort this " \
346
+ 'collection on -- and it ignores a sort it refuses without reporting it. The rows come back in the ' \
347
+ 'order the API imposes.'
348
+ )
349
+ end
350
+
351
+ # `sort` here is the clause `server_sort` found the endpoint does honour,
352
+ # which is why nothing has reported it yet: it is this route, not the
353
+ # column, that cannot carry it.
354
+ def warn_unordered_ids(sort)
355
+ ForestAdminDatasourceIntercom.logger.warn(
356
+ "[forest_admin_datasource_intercom] #{name} was asked to sort on #{sort[:field].inspect} while reading " \
357
+ 'records by id, which Intercom reads by id and in no order. The rows come back in the order the ids ' \
358
+ 'were named.'
359
+ )
360
+ end
361
+ end
362
+ end
363
+ end
@@ -0,0 +1,55 @@
1
+ module ForestAdminDatasourceIntercom
2
+ module Collections
3
+ # An attribute whose name is already a column or relation (measured: a contact attribute named `id`) is
4
+ # skipped with a warning, so relations must be declared first. Dates arrive as epoch seconds.
5
+ module CustomAttributes
6
+ private
7
+
8
+ def register_attribute_columns
9
+ @attribute_columns = @attributes.reject { |attribute| collides?(attribute) }
10
+ @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) }
11
+ end
12
+
13
+ def attribute_columns = @attribute_columns || []
14
+
15
+ # What the translator needs of them: the names the schema published, so a
16
+ # filter reaching one is refused with the reason the table carries for
17
+ # the whole family rather than with the message for a column that is not
18
+ # in it.
19
+ def attribute_column_names = attribute_columns.map(&:column_name)
20
+
21
+ # What the log calls these, which is the workspace's own vocabulary: a
22
+ # ticket attribute is declared per ticket type, a contact attribute per
23
+ # model.
24
+ def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind")
25
+
26
+ def collides?(attribute)
27
+ return false unless fields.key?(attribute.column_name)
28
+
29
+ ForestAdminDatasourceIntercom.logger.warn(
30
+ "[forest_admin_datasource_intercom] #{name} skips the #{attribute_kind} attribute " \
31
+ "#{attribute.name.inspect}: a native column or relation already carries the name " \
32
+ "#{attribute.column_name.inspect}, and overwriting it would show the attribute where the operator " \
33
+ 'expects the record field. Rename it in Intercom to publish it.'
34
+ )
35
+ true
36
+ end
37
+
38
+ # Nil rather than absent when the record lacks the attribute: the column exists on every row.
39
+ def attribute_values(values)
40
+ held = values.is_a?(Hash) ? values : {}
41
+
42
+ attribute_columns.to_h do |attribute|
43
+ [attribute.column_name, coerce_attribute(held[attribute.name], attribute)]
44
+ end
45
+ end
46
+
47
+ def coerce_attribute(value, attribute)
48
+ return nil if value.nil?
49
+ return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric)
50
+
51
+ value
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,196 @@
1
+ module ForestAdminDatasourceIntercom
2
+ module Collections
3
+ # Collections Intercom returns whole in one unpaginated response, so filtering, sorting and
4
+ # counting in memory is exact. The response is cached for `reference_cache_ttl` (0 disables).
5
+ class FetchAllCollection < BaseCollection # rubocop:disable Metrics/ClassLength
6
+ # The filters a column may advertise, per column type. Restricted to what
7
+ # the toolkit can evaluate in memory, since the in-memory pass is the only
8
+ # pass there is here: an operator with no equivalence makes `match` answer
9
+ # nil, which `apply` reads as "no match" and would empty the page instead
10
+ # of filtering it.
11
+ OPERATOR_CANDIDATES = {
12
+ 'String' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN,
13
+ Operators::PRESENT, Operators::BLANK, Operators::CONTAINS, Operators::I_CONTAINS,
14
+ Operators::NOT_CONTAINS, Operators::STARTS_WITH, Operators::ENDS_WITH],
15
+ 'Boolean' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN,
16
+ Operators::PRESENT, Operators::BLANK]
17
+ }.freeze
18
+
19
+ # The operators `ConditionTreeLeaf#match` evaluates natively; anything else
20
+ # needs an equivalence for the column's type to be evaluable at all.
21
+ IN_MEMORY_OPERATORS = [Operators::IN, Operators::EQUAL, Operators::LESS_THAN, Operators::GREATER_THAN,
22
+ Operators::MATCH, Operators::STARTS_WITH, Operators::ENDS_WITH,
23
+ Operators::LONGER_THAN, Operators::SHORTER_THAN, Operators::INCLUDES_ALL,
24
+ Operators::NOT_IN, Operators::NOT_EQUAL, Operators::NOT_CONTAINS].freeze
25
+
26
+ # Candidates are re-checked against the toolkit rather than trusted, so an
27
+ # equivalence it stops providing takes the filter out of the schema instead
28
+ # of turning every page using it into an empty one.
29
+ def self.operators_for(column_type)
30
+ Array(OPERATOR_CANDIDATES[column_type]).select do |operator|
31
+ Equivalent.equivalent_tree?(operator, IN_MEMORY_OPERATORS, column_type)
32
+ end
33
+ end
34
+
35
+ # Countable, unlike the cursor-paginated collections: the count answered
36
+ # here is taken over every record Intercom holds rather than over the pages
37
+ # a walk happened to collect.
38
+ def initialize(datasource, name)
39
+ super
40
+ enable_count
41
+ end
42
+
43
+ def list(caller, filter, projection)
44
+ client.with_read_scope do
45
+ records = sort_in_memory(filtered_records(caller, filter), filter&.sort)
46
+ window = page_window(records, filter)
47
+ rows = window.map { |record| project(record, projection) }
48
+
49
+ enrich(window, rows, projection)
50
+ embed_relations(caller, window, rows, projection)
51
+ rows
52
+ end
53
+ end
54
+
55
+ # Relations are embedded first: a group through one (`team:name`) would otherwise read nil
56
+ # and fold every row into one group.
57
+ def aggregate(caller, filter, aggregation, limit = nil)
58
+ client.with_read_scope do
59
+ records = filtered_records(caller, filter)
60
+ rows = records.map(&:dup)
61
+ embed_relations(caller, records, rows, aggregation.projection)
62
+
63
+ aggregation.apply(rows, timezone_for(caller), limit)
64
+ .map { |row| { 'group' => row[:group], 'value' => row[:value] } }
65
+ end
66
+ end
67
+
68
+ protected
69
+
70
+ # Read-only: an editable column would offer a Save reaching an unimplemented `update`.
71
+ def add_column(name, type, is_primary_key: false)
72
+ add_field(name, ColumnSchema.new(column_type: type,
73
+ filter_operators: self.class.operators_for(type),
74
+ is_primary_key: is_primary_key,
75
+ is_read_only: true,
76
+ is_sortable: type != 'Json',
77
+ is_groupable: type != 'Json'))
78
+ end
79
+
80
+ # Hook for what a row needs beyond the endpoint this collection reads --
81
+ # a name held on the other side of a membership, say. Called with the rows
82
+ # of the page only, and with the projection, so a column nobody asked for
83
+ # costs no request.
84
+ def enrich(_records, _rows, _projection); end
85
+
86
+ # Every record of the collection, straight from its endpoint.
87
+ def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all")
88
+
89
+ def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize")
90
+
91
+ private
92
+
93
+ # The complete dataset, serialized and narrowed to the rows the filter
94
+ # keeps: what `list` pages and what `aggregate` counts are the same rows.
95
+ def filtered_records(caller, filter)
96
+ records = fetch_all.map { |entity| serialize(entity) }
97
+ tree = filter&.condition_tree
98
+ return records if tree.nil?
99
+
100
+ # A condition through a relation becomes a membership on the foreign key:
101
+ # this tier filters in memory, where a list of ids costs no more than one
102
+ # id and none of Intercom's group limits apply -- they bound its search
103
+ # DSL, which nothing here goes through.
104
+ tree = rewrite_relation_conditions(caller, tree) { |key, ids, _| Leaf.new(key, Operators::IN, ids) }
105
+ return [] if tree == NOTHING
106
+
107
+ refuse_unevaluable!(tree)
108
+ tree.apply(records, self, timezone_for(caller))
109
+ end
110
+
111
+ # A condition this collection cannot evaluate is refused rather than
112
+ # applied. `match` answers nil for an operator with no in-memory
113
+ # equivalence and `apply` reads that as "no match", so an unevaluable
114
+ # condition would hand back an empty page that looks like a filter
115
+ # matching nothing -- indistinguishable, to the operator, from a real
116
+ # answer. The schema advertises no such operator; a scope, a segment or a
117
+ # customizer can still send one.
118
+ def refuse_unevaluable!(tree)
119
+ offender = nil
120
+ tree.some_leaf do |leaf|
121
+ offender = leaf unless evaluable?(leaf)
122
+ !offender.nil?
123
+ end
124
+ return if offender.nil?
125
+
126
+ raise UnsupportedOperatorError,
127
+ "#{name} cannot filter '#{offender.field}' with '#{offender.operator}': it is read whole from " \
128
+ 'Intercom and filtered in memory, which supports only the operators its columns advertise. ' \
129
+ 'Change the condition, or the scope or segment carrying it.'
130
+ end
131
+
132
+ def evaluable?(leaf)
133
+ schema = fields[leaf.field]
134
+ schema.is_a?(ColumnSchema) && schema.filter_operators.include?(leaf.operator)
135
+ end
136
+
137
+ # Not `sort`/`Sort#apply`: `sort` is unstable and `<=>` answers nil on nulls, booleans and
138
+ # mixed types. Ties fall back to Intercom's order.
139
+ def sort_in_memory(records, sort)
140
+ clauses = sort_clauses(sort)
141
+ return records if clauses.empty?
142
+
143
+ records.each_with_index.sort do |(left, left_index), (right, right_index)|
144
+ compare_clauses(left, right, clauses).nonzero? || (left_index <=> right_index)
145
+ end.map(&:first)
146
+ end
147
+
148
+ # Unsortable clauses are dropped with a warning. They do arrive: `Filter#nest` prefixes the
149
+ # condition tree but not the sort, so a many-to-many membership gets the target's columns.
150
+ def sort_clauses(sort)
151
+ known, unknown = Array(sort).partition { |clause| sortable_column?(sort_field(clause)) }
152
+ warn_unsortable(unknown) unless unknown.empty?
153
+
154
+ known.map { |clause| [sort_field(clause), ascending?(clause) != false] }
155
+ end
156
+
157
+ def sortable_column?(field)
158
+ schema = fields[field]
159
+ schema.is_a?(ColumnSchema) && schema.is_sortable
160
+ end
161
+
162
+ def warn_unsortable(clauses)
163
+ ForestAdminDatasourceIntercom.logger.warn(
164
+ "[forest_admin_datasource_intercom] #{name} was asked to sort on " \
165
+ "#{clauses.map { |clause| sort_field(clause).inspect }.join(", ")}, which it does not carry as a " \
166
+ 'sortable column; the rows ' \
167
+ 'come back in the order Intercom returned them. A related list through a many-to-many is ordered by the ' \
168
+ 'columns of the collection it travels through, not by those of the collection it reaches.'
169
+ )
170
+ end
171
+
172
+ def compare_clauses(left, right, clauses)
173
+ clauses.each do |field, ascending|
174
+ comparison = compare_values(left[field], right[field])
175
+ next if comparison.zero?
176
+
177
+ return ascending ? comparison : -comparison
178
+ end
179
+
180
+ 0
181
+ end
182
+
183
+ # Nulls sort last ascending and first descending, the way a database orders
184
+ # them; values `<=>` cannot compare -- two booleans, for one -- are
185
+ # compared through their string form rather than left undefined, which puts
186
+ # `false` before `true`, again like a database.
187
+ def compare_values(left, right)
188
+ return 0 if left.nil? && right.nil?
189
+ return 1 if left.nil?
190
+ return -1 if right.nil?
191
+
192
+ (left <=> right) || (left.to_s <=> right.to_s)
193
+ end
194
+ end
195
+ end
196
+ end