forest_admin_datasource_intercom 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/LICENSE +202 -0
- data/README.md +811 -0
- data/Rakefile +6 -0
- data/exe/forest_admin_intercom_probe +287 -0
- data/forest_admin_datasource_intercom.gemspec +38 -0
- data/lib/forest_admin_datasource_intercom/cache.rb +84 -0
- data/lib/forest_admin_datasource_intercom/client.rb +546 -0
- data/lib/forest_admin_datasource_intercom/collections/admin.rb +83 -0
- data/lib/forest_admin_datasource_intercom/collections/base_collection.rb +161 -0
- data/lib/forest_admin_datasource_intercom/collections/company/serializer.rb +44 -0
- data/lib/forest_admin_datasource_intercom/collections/company.rb +69 -0
- data/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb +106 -0
- data/lib/forest_admin_datasource_intercom/collections/contact.rb +266 -0
- data/lib/forest_admin_datasource_intercom/collections/contact_identity.rb +77 -0
- data/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb +75 -0
- data/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb +40 -0
- data/lib/forest_admin_datasource_intercom/collections/conversation.rb +153 -0
- data/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +363 -0
- data/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb +55 -0
- data/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb +196 -0
- data/lib/forest_admin_datasource_intercom/collections/offset_collection.rb +311 -0
- data/lib/forest_admin_datasource_intercom/collections/records_by_id.rb +97 -0
- data/lib/forest_admin_datasource_intercom/collections/relations.rb +294 -0
- data/lib/forest_admin_datasource_intercom/collections/team.rb +70 -0
- data/lib/forest_admin_datasource_intercom/collections/team_membership.rb +51 -0
- data/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb +130 -0
- data/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb +70 -0
- data/lib/forest_admin_datasource_intercom/collections/ticket.rb +143 -0
- data/lib/forest_admin_datasource_intercom/collections/ticket_state.rb +40 -0
- data/lib/forest_admin_datasource_intercom/collections/ticket_type.rb +46 -0
- data/lib/forest_admin_datasource_intercom/collections/timeline.rb +40 -0
- data/lib/forest_admin_datasource_intercom/configuration.rb +226 -0
- data/lib/forest_admin_datasource_intercom/datasource.rb +78 -0
- data/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb +123 -0
- data/lib/forest_admin_datasource_intercom/query/caller_zone.rb +48 -0
- data/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb +169 -0
- data/lib/forest_admin_datasource_intercom/query/day_bounds.rb +75 -0
- data/lib/forest_admin_datasource_intercom/query/filter_value.rb +177 -0
- data/lib/forest_admin_datasource_intercom/query/operator_table.rb +54 -0
- data/lib/forest_admin_datasource_intercom/query/search_fields.rb +225 -0
- data/lib/forest_admin_datasource_intercom/query/search_fields.yml +663 -0
- data/lib/forest_admin_datasource_intercom/rate_limiter.rb +148 -0
- data/lib/forest_admin_datasource_intercom/retry_policy.rb +95 -0
- data/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb +52 -0
- data/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb +88 -0
- data/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb +133 -0
- data/lib/forest_admin_datasource_intercom/throttle.rb +19 -0
- data/lib/forest_admin_datasource_intercom/version.rb +3 -0
- data/lib/forest_admin_datasource_intercom.rb +57 -0
- metadata +180 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Pagination
|
|
3
|
+
# Intercom only paginates by cursor (no jump to page N), so an offset/limit window costs one sequential
|
|
4
|
+
# request per page. Capped for operator patience (R9), and every truncation is logged, never silent.
|
|
5
|
+
class CursorWalker
|
|
6
|
+
# 50 pages of 150 records. Intercom's quota lets these be generous: the
|
|
7
|
+
# walk is bounded by patience, and by the point past which a list view is
|
|
8
|
+
# not being read but scraped.
|
|
9
|
+
MAX_PAGES = 50
|
|
10
|
+
MAX_RECORDS = 7_500
|
|
11
|
+
|
|
12
|
+
def initialize(max_pages: MAX_PAGES, max_records: MAX_RECORDS)
|
|
13
|
+
@max_pages = max_pages
|
|
14
|
+
@max_records = max_records
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Yields `(per_page, cursor)` and expects a Client::Page back. A nil limit means every record past
|
|
18
|
+
# the offset, so a cap stopping the walk is known to be a truncation and is logged.
|
|
19
|
+
def walk(offset:, limit:, &page_source)
|
|
20
|
+
offset = offset.to_i.clamp(0, nil)
|
|
21
|
+
limit = limit&.to_i
|
|
22
|
+
return [] if limit && !limit.positive?
|
|
23
|
+
|
|
24
|
+
records = collect(offset, limit, &page_source)
|
|
25
|
+
|
|
26
|
+
limit ? (records[offset, limit] || []) : records.drop(offset)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# The walk itself: pages are collected until the window is covered, the
|
|
32
|
+
# source says there is nothing left, or a cap stops it. The slicing is
|
|
33
|
+
# `walk`'s; this only decides how far to go.
|
|
34
|
+
def collect(offset, limit)
|
|
35
|
+
needed = limit && (offset + limit)
|
|
36
|
+
records = []
|
|
37
|
+
cursor = nil
|
|
38
|
+
seen_ids = Set.new
|
|
39
|
+
seen_cursors = Set.new
|
|
40
|
+
pages = 0
|
|
41
|
+
|
|
42
|
+
loop do
|
|
43
|
+
page = yield(batch_size(needed, records.size), cursor)
|
|
44
|
+
records.concat(fresh(page.records, seen_ids))
|
|
45
|
+
pages += 1
|
|
46
|
+
|
|
47
|
+
# The only silent exits: Intercom said there is no page left, or the
|
|
48
|
+
# window asked for is covered. Everything below hands back less than
|
|
49
|
+
# was asked for, and says so.
|
|
50
|
+
break if page.next_cursor.nil?
|
|
51
|
+
break if needed && records.size >= needed
|
|
52
|
+
break if cut_short?(page, seen_cursors, pages, records: records.size, window: window_of(offset, limit))
|
|
53
|
+
|
|
54
|
+
cursor = page.next_cursor
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
records
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Whether the walk is handing back less than it was asked for, and which
|
|
61
|
+
# of the two reasons it is. Both are logged here rather than by the
|
|
62
|
+
# caller: a page that looks like the whole answer and is not is the
|
|
63
|
+
# failure this datasource exists to avoid, so there is no route out of
|
|
64
|
+
# this walk that is short and quiet.
|
|
65
|
+
def cut_short?(page, seen_cursors, pages, records:, window:)
|
|
66
|
+
if stalled?(page, seen_cursors)
|
|
67
|
+
log_stalled(window: window, pages: pages, collected: records)
|
|
68
|
+
elsif capped?(pages, records)
|
|
69
|
+
log_truncation(window: window, pages: pages, collected: records)
|
|
70
|
+
else
|
|
71
|
+
return false
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
true
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Intercom documents duplicate or missed records when items change mid-walk; duplicates are dropped
|
|
78
|
+
# here (missed ones cannot be). A record with no id is kept.
|
|
79
|
+
def fresh(records, seen_ids)
|
|
80
|
+
records.select { |record| record['id'].nil? || seen_ids.add?(record['id']) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# An empty page with a next cursor, or a cursor already followed: Intercom does neither today, but a
|
|
84
|
+
# walk driven by a remote value must stop on its own terms -- and the stop is logged as a truncation.
|
|
85
|
+
def stalled?(page, seen_cursors)
|
|
86
|
+
page.records.empty? || !seen_cursors.add?(page.next_cursor)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def capped?(pages, collected)
|
|
90
|
+
pages >= @max_pages || collected >= @max_records
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Bounded by the record budget left, and by the window still missing when
|
|
94
|
+
# there is one, so the walk never collects past @max_records. `Client`
|
|
95
|
+
# bounds it again to what Intercom accepts.
|
|
96
|
+
def batch_size(needed, collected)
|
|
97
|
+
budget = @max_records - collected
|
|
98
|
+
budget = [needed - collected, budget].min if needed
|
|
99
|
+
Client.bounded_per_page(budget)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def log_truncation(window:, pages:, collected:)
|
|
103
|
+
ForestAdminDatasourceIntercom.logger.warn(
|
|
104
|
+
"[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \
|
|
105
|
+
"#{collected} record(s) while fetching #{window}; results are truncated. " \
|
|
106
|
+
'Narrow the filter to reach records past this point.'
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def log_stalled(window:, pages:, collected:)
|
|
111
|
+
ForestAdminDatasourceIntercom.logger.warn(
|
|
112
|
+
"[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \
|
|
113
|
+
"#{collected} record(s) while fetching #{window}: Intercom advertised a next page this could not " \
|
|
114
|
+
'follow -- an empty page, or a cursor already read. Results are truncated.'
|
|
115
|
+
)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def window_of(offset, limit)
|
|
119
|
+
limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}"
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Query
|
|
3
|
+
# The timezone a filter was written in, and what reading a value in it
|
|
4
|
+
# costs when the caller names one nothing knows.
|
|
5
|
+
#
|
|
6
|
+
# Split from FilterValue for the reason DayBounds was: that class knows
|
|
7
|
+
# what shape a field expects on the wire, and whether a value carries the
|
|
8
|
+
# caller's wall clock or the server's is a different question.
|
|
9
|
+
class CallerZone
|
|
10
|
+
# Kept stripped rather than only checked stripped: as it came, a
|
|
11
|
+
# `" Europe/Paris "` passes the blank guard and then fails the zone
|
|
12
|
+
# lookup, and a day boundary lands an offset away from where the filter
|
|
13
|
+
# meant it.
|
|
14
|
+
attr_reader :name
|
|
15
|
+
|
|
16
|
+
def initialize(identifier)
|
|
17
|
+
stripped = identifier.to_s.strip
|
|
18
|
+
@name = stripped.empty? ? 'UTC' : stripped
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# A day with no time of day is the caller's day, not the server's: it is
|
|
22
|
+
# the timezone the filter was written in that says when that day starts.
|
|
23
|
+
def start_of_day(date)
|
|
24
|
+
read('the day boundary') { Time.zone.local(date.year, date.month, date.day).to_i }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# An offset-less timestamp is the caller's wall clock: FilterFactory writes previous-period bounds as
|
|
28
|
+
# `%Y-%m-%d %H:%M:%S`, and a midnight shifted by any offset truncates onto another UTC day.
|
|
29
|
+
def timestamp(value)
|
|
30
|
+
read('the timestamp') { Time.zone.parse(value).to_i }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
# Falling back to UTC silently would move a boundary by the offset, which
|
|
36
|
+
# is the failure a timezone is read for in the first place.
|
|
37
|
+
def read(what, &block)
|
|
38
|
+
Time.use_zone(@name, &block)
|
|
39
|
+
rescue ArgumentError
|
|
40
|
+
ForestAdminDatasourceIntercom.logger.warn(
|
|
41
|
+
"[forest_admin_datasource_intercom] unknown timezone #{@name.inspect}, reading #{what} of a date " \
|
|
42
|
+
'filter in UTC instead.'
|
|
43
|
+
)
|
|
44
|
+
Time.use_zone('UTC', &block)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Query
|
|
3
|
+
# Whatever it cannot translate it refuses, naming the field or operator: a dropped condition would serve
|
|
4
|
+
# an unfiltered page that looks filtered. What each endpoint filters comes from `search_fields.yml`.
|
|
5
|
+
class ConditionTreeTranslator
|
|
6
|
+
Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch
|
|
7
|
+
Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
|
|
8
|
+
|
|
9
|
+
AGGREGATORS = { 'and' => 'AND', 'or' => 'OR' }.freeze
|
|
10
|
+
|
|
11
|
+
# Intercom nests a search two levels deep and takes fifteen conditions per
|
|
12
|
+
# group. Both are checked here rather than left to the API: over either,
|
|
13
|
+
# Intercom answers a 400 whose body names neither the limit nor the part of
|
|
14
|
+
# the filter that reached it, and an operator reading it has no way of
|
|
15
|
+
# knowing that their segment plus their scope plus their own filter is what
|
|
16
|
+
# went over.
|
|
17
|
+
MAX_DEPTH = 2
|
|
18
|
+
MAX_GROUP_SIZE = 15
|
|
19
|
+
|
|
20
|
+
def self.call(condition_tree, endpoint:, collection:, timezone: nil, attribute_columns: [])
|
|
21
|
+
return nil if condition_tree.nil?
|
|
22
|
+
|
|
23
|
+
new(endpoint: endpoint, collection: collection, timezone: timezone,
|
|
24
|
+
attribute_columns: attribute_columns).translate(condition_tree)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# `attribute_columns` are the columns a workspace's own attributes became,
|
|
28
|
+
# which no row of the table can name: they are discovered at boot. They
|
|
29
|
+
# share one refusal, carried by the endpoint, and passing them here is
|
|
30
|
+
# what lets it be read instead of the generic message.
|
|
31
|
+
def initialize(endpoint:, collection:, timezone: nil, attribute_columns: [])
|
|
32
|
+
@endpoint = endpoint
|
|
33
|
+
@collection = collection
|
|
34
|
+
@attribute_columns = Array(attribute_columns).map(&:to_s)
|
|
35
|
+
@value = FilterValue.new(collection: collection, timezone: timezone)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def translate(node, depth = 1)
|
|
39
|
+
case node
|
|
40
|
+
when Branch then translate_branch(node, depth)
|
|
41
|
+
when Leaf then translate_leaf(node)
|
|
42
|
+
else raise UnsupportedOperatorError, "#{@collection} cannot read #{node.class} as a condition."
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def translate_branch(branch, depth)
|
|
49
|
+
conditions = Array(branch.conditions)
|
|
50
|
+
refuse_empty_branch!(branch) if conditions.empty?
|
|
51
|
+
|
|
52
|
+
# Read before the unwrap below, so a branch is refused on the aggregator
|
|
53
|
+
# it carries rather than on how many conditions it holds.
|
|
54
|
+
operator = aggregator(branch)
|
|
55
|
+
|
|
56
|
+
# A branch holding one condition needs no group of its own. The agent
|
|
57
|
+
# builds a tree one branch at a time -- a scope, then a segment, then the
|
|
58
|
+
# operator's own filter -- and the nesting Intercom allows is shallow
|
|
59
|
+
# enough that a wrapper around nothing is a level worth not spending.
|
|
60
|
+
return translate(conditions.first, depth) if conditions.size == 1
|
|
61
|
+
|
|
62
|
+
refuse_too_deep!(branch, depth) if depth > MAX_DEPTH
|
|
63
|
+
refuse_too_wide!(branch, conditions.size) if conditions.size > MAX_GROUP_SIZE
|
|
64
|
+
|
|
65
|
+
{ 'operator' => operator, 'value' => conditions.map { |condition| translate(condition, depth + 1) } }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# What reaches this depth is a group inside a group inside a group. The
|
|
69
|
+
# message names the shape rather than a number, since the tree an operator
|
|
70
|
+
# can act on is the segment and the scope they wrote, not the one the agent
|
|
71
|
+
# assembled out of them.
|
|
72
|
+
def refuse_too_deep!(branch, depth)
|
|
73
|
+
raise UnsupportedOperatorError,
|
|
74
|
+
"#{@collection} cannot answer this filter: Intercom nests a search #{MAX_DEPTH} levels deep and this " \
|
|
75
|
+
"one reaches #{depth}. #{deepening_cause(branch)}"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# A group the operator wrote is theirs to flatten. A group a relation
|
|
79
|
+
# expanded into is not: it is the several records the relation matched,
|
|
80
|
+
# written one equality each because Intercom takes no membership operator
|
|
81
|
+
# on a foreign key -- and telling them to flatten a nesting they never
|
|
82
|
+
# wrote is a refusal they cannot act on.
|
|
83
|
+
def deepening_cause(branch)
|
|
84
|
+
field = branch.respond_to?(:relation_field) ? branch.relation_field : nil
|
|
85
|
+
|
|
86
|
+
if field.nil?
|
|
87
|
+
'A group inside a group inside a group is one level too many -- flatten the segment, the scope or the ' \
|
|
88
|
+
'filter carrying the innermost one.'
|
|
89
|
+
else
|
|
90
|
+
"The innermost group is what #{field.inspect} expanded into, one equality per record it matched: narrow " \
|
|
91
|
+
'that condition until it names a single record, or lift it out of the groups nesting it.'
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Fifteen is reached without trying: a scope, a segment and a filter add up,
|
|
96
|
+
# and a condition naming several values is expanded into one condition per
|
|
97
|
+
# value on the way here, Intercom taking no membership operator on these
|
|
98
|
+
# fields.
|
|
99
|
+
def refuse_too_wide!(branch, size)
|
|
100
|
+
raise UnsupportedOperatorError,
|
|
101
|
+
"#{@collection} cannot answer this filter: Intercom takes #{MAX_GROUP_SIZE} conditions per group and " \
|
|
102
|
+
"this #{branch.aggregator} carries #{size}. A filter naming several values counts one condition per " \
|
|
103
|
+
'value here, so narrowing the list, the segment or the scope is what brings it back under the limit.'
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def aggregator(branch)
|
|
107
|
+
AGGREGATORS[branch.aggregator.to_s.downcase] ||
|
|
108
|
+
raise(UnsupportedOperatorError,
|
|
109
|
+
"#{@collection} cannot read #{branch.aggregator.inspect} as a condition tree aggregator; " \
|
|
110
|
+
"expected 'And' or 'Or'.")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def translate_leaf(leaf)
|
|
114
|
+
field = @endpoint.field(leaf.field.to_s) || refuse_unfilterable!(leaf.field.to_s)
|
|
115
|
+
spelling = OperatorTable.intercom_operator(field, leaf.operator) || refuse_operator!(leaf, field)
|
|
116
|
+
|
|
117
|
+
{ 'field' => field.field, 'operator' => spelling, 'value' => @value.call(leaf, field, spelling) }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# A column the endpoint does not filter, and the reason it does not, taken
|
|
121
|
+
# from the table when it carries one: those reasons are the difference
|
|
122
|
+
# between "no" and a message an operator can do something with.
|
|
123
|
+
def refuse_unfilterable!(column)
|
|
124
|
+
raise UnsupportedOperatorError, "#{@collection} cannot filter #{column.inspect}: #{unfilterable_reason(column)}"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def unfilterable_reason(column)
|
|
128
|
+
return relation_reason(column) if column.include?(':')
|
|
129
|
+
|
|
130
|
+
refusal = @endpoint.refusal(column) || attribute_refusal(column)
|
|
131
|
+
return refusal.reason if refusal
|
|
132
|
+
|
|
133
|
+
"#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}."
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# A relation reaches the translator as `relation:field`, and it should not:
|
|
137
|
+
# `Relations#rewrite_relation_conditions` trades every one of them for a
|
|
138
|
+
# condition on the foreign key before a tree gets this far, or refuses it
|
|
139
|
+
# by name -- with the relations the collection does declare. So this is
|
|
140
|
+
# the message for a caller that skipped that pass, and it names what this
|
|
141
|
+
# endpoint filters rather than a relation nobody resolved.
|
|
142
|
+
def relation_reason(_column)
|
|
143
|
+
"#{@endpoint.path} filters columns, not paths through a relation -- those are resolved against the " \
|
|
144
|
+
'collection they point at before a condition reaches here. Filter on one of: ' \
|
|
145
|
+
"#{@endpoint.filterable_columns.join(", ")}."
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# A column the table cannot carry a row for, the workspace having named it:
|
|
149
|
+
# the endpoint's one refusal for the whole family answers for it.
|
|
150
|
+
def attribute_refusal(column)
|
|
151
|
+
@attribute_columns.include?(column) ? @endpoint.attribute_refusal : nil
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def refuse_operator!(leaf, field)
|
|
155
|
+
supported = OperatorTable.forest_operators(field)
|
|
156
|
+
|
|
157
|
+
raise UnsupportedOperatorError,
|
|
158
|
+
"#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator.inspect}: " \
|
|
159
|
+
"#{@endpoint.path} answers #{supported.join(", ")} on #{field.field.inspect} and nothing else."
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def refuse_empty_branch!(branch)
|
|
163
|
+
raise UnsupportedOperatorError,
|
|
164
|
+
"#{@collection} was given a #{branch.aggregator} branch carrying no condition, which names no record " \
|
|
165
|
+
'and no filter.'
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Query
|
|
3
|
+
# Intercom truncates a date search to the UTC day (measured; the docs promise the workspace timezone):
|
|
4
|
+
# `> V` starts at the day after V's, `< V` stops before V's day. Each bound is moved one day outward (a `<`
|
|
5
|
+
# already on a day boundary stays) so Intercom answers the day the filter named: the window is day-granular.
|
|
6
|
+
# A bound inside a day would be answered for the whole of it, rows outside the window included, so it is
|
|
7
|
+
# refused -- `future`, `after_x_hours_ago`, a time of day. Except a `<` at the present (`past`, the
|
|
8
|
+
# `*_to_date` family): widened to midnight it only adds rows dated after now, and a column holding past
|
|
9
|
+
# instants has none.
|
|
10
|
+
class DayBounds
|
|
11
|
+
UTC_DAY = 86_400
|
|
12
|
+
|
|
13
|
+
# The one filterable date that can lie ahead of the present.
|
|
14
|
+
FUTURE_DATED = %w[snoozed_until].freeze
|
|
15
|
+
|
|
16
|
+
# The toolkit reads the clock before this does and writes the bound to the second.
|
|
17
|
+
CLOCK_SLACK = 5
|
|
18
|
+
|
|
19
|
+
def initialize(collection:, timezone:, now: -> { Time.now.to_i })
|
|
20
|
+
@collection = collection
|
|
21
|
+
@timezone = timezone
|
|
22
|
+
@now = now
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def call(seconds, spelling, leaf, field)
|
|
26
|
+
open_ended = open_ended?(seconds, spelling, field)
|
|
27
|
+
refuse_inside_a_day!(seconds, leaf) unless midnight?(seconds) || open_ended
|
|
28
|
+
|
|
29
|
+
day = seconds - (seconds % UTC_DAY)
|
|
30
|
+
report_utc_day unless seconds == day || open_ended
|
|
31
|
+
|
|
32
|
+
return day - UTC_DAY if spelling == '>'
|
|
33
|
+
|
|
34
|
+
seconds == day ? seconds : day + UTC_DAY
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# The caller's midnight, which is where the toolkit puts the bounds of `today` and the previous periods,
|
|
40
|
+
# or the UTC one, which is the day Intercom compares by.
|
|
41
|
+
def midnight?(seconds)
|
|
42
|
+
return true if (seconds % UTC_DAY).zero?
|
|
43
|
+
|
|
44
|
+
Time.use_zone(@timezone) { Time.zone.at(seconds) == Time.zone.at(seconds).beginning_of_day }
|
|
45
|
+
rescue ArgumentError
|
|
46
|
+
false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def open_ended?(seconds, spelling, field)
|
|
50
|
+
spelling == '<' && !FUTURE_DATED.include?(field.field) && seconds >= @now.call - CLOCK_SLACK
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def refuse_inside_a_day!(seconds, leaf)
|
|
54
|
+
raise UnsupportedOperatorError,
|
|
55
|
+
"#{@collection} cannot filter #{leaf.field.inspect} on #{Time.at(seconds).utc.iso8601}, a time " \
|
|
56
|
+
'inside a day: Intercom compares a date by the UTC day, so it would answer the whole day and ' \
|
|
57
|
+
'return rows outside the window. Filter by day instead -- before or after a date, today, ' \
|
|
58
|
+
'yesterday, a previous period.'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A bound on the caller's midnight but not the UTC one is widened to the UTC day, whichever side of UTC
|
|
62
|
+
# the caller sits. Reported once per filter rather than per bound.
|
|
63
|
+
def report_utc_day
|
|
64
|
+
return if @reported
|
|
65
|
+
|
|
66
|
+
@reported = true
|
|
67
|
+
ForestAdminDatasourceIntercom.logger.warn(
|
|
68
|
+
"[forest_admin_datasource_intercom] #{@collection} was filtered on a date written in #{@timezone}, and " \
|
|
69
|
+
'Intercom truncates a date search to the UTC day whatever the workspace timezone says. The rows come ' \
|
|
70
|
+
'back for the UTC days the window overlaps, which is up to a day wider at each end.'
|
|
71
|
+
)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Query
|
|
3
|
+
# Intercom types its filter values, and a wrongly shaped one fails with `data_invalid`, the same code as an
|
|
4
|
+
# unsupported operator, so values are converted here rather than by reading error codes.
|
|
5
|
+
# Long by line count only: most of it is the refusals, one per shape.
|
|
6
|
+
class FilterValue # rubocop:disable Metrics/ClassLength
|
|
7
|
+
# What the frontend sends for a Dateonly column, and what a segment written
|
|
8
|
+
# in Ruby carries: a day with no time of day, which is midnight in the
|
|
9
|
+
# timezone of whoever wrote the filter rather than in the server's.
|
|
10
|
+
DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/
|
|
11
|
+
|
|
12
|
+
# What `Date._parse` has to have found for a string to name a day at all.
|
|
13
|
+
DATE_PARTS = %i[year mon mday].freeze
|
|
14
|
+
|
|
15
|
+
INTEGER = /\A-?\d+\z/
|
|
16
|
+
|
|
17
|
+
# Measured on every list, whatever group it stands in: Intercom counts
|
|
18
|
+
# each value as an element of a composite query and answers a 400 naming
|
|
19
|
+
# neither the field nor the condition past 15.
|
|
20
|
+
MAX_LIST_SIZE = 15
|
|
21
|
+
|
|
22
|
+
def initialize(collection:, timezone: nil, now: -> { Time.now.to_i })
|
|
23
|
+
@collection = collection
|
|
24
|
+
@zone = CallerZone.new(timezone)
|
|
25
|
+
@day_bounds = DayBounds.new(collection: collection, timezone: @zone.name, now: now)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def call(leaf, field, spelling)
|
|
29
|
+
return list(leaf, field) if OperatorTable.list_operator?(spelling)
|
|
30
|
+
|
|
31
|
+
refuse_absence!(leaf, field) if blank?(leaf.value)
|
|
32
|
+
|
|
33
|
+
value = scalar(leaf.value, leaf, field)
|
|
34
|
+
field.type == 'date' ? @day_bounds.call(value, spelling, leaf, field) : value
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Dropping the blanks would answer a different question: `not_in [nil,
|
|
40
|
+
# 'open']` was asked to exclude the records carrying neither and would come
|
|
41
|
+
# back including them. An empty list is as bad the other way round, being a
|
|
42
|
+
# filter that matches everything.
|
|
43
|
+
def list(leaf, field)
|
|
44
|
+
values = Array(leaf.value)
|
|
45
|
+
refuse_empty_list!(leaf) if values.empty?
|
|
46
|
+
refuse_long_list!(leaf, values.size) if values.size > MAX_LIST_SIZE
|
|
47
|
+
refuse_absence!(leaf, field) if values.any? { |value| blank?(value) }
|
|
48
|
+
|
|
49
|
+
values.map { |value| scalar(value, leaf, field) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def scalar(value, leaf, field)
|
|
53
|
+
case field.type
|
|
54
|
+
when 'date' then epoch(value, leaf)
|
|
55
|
+
when 'number' then number(value, leaf)
|
|
56
|
+
when 'boolean' then boolean(value, leaf)
|
|
57
|
+
else field.value_map ? mapped(value.to_s, leaf, field) : value.to_s
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A value outside the map is refused rather than sent as typed: Intercom
|
|
62
|
+
# would compare it against a vocabulary the column never shows.
|
|
63
|
+
def mapped(value, leaf, field)
|
|
64
|
+
field.value_map.fetch(value) { refuse_value!(leaf, value, "one of #{field.value_map.keys.join(", ")}") }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Intercom stores and compares its dates as epoch seconds. What arrives
|
|
68
|
+
# here is an ISO8601 string most of the time -- the frontend sends one, and
|
|
69
|
+
# so does every interval operator the toolkit rewrites into a pair of
|
|
70
|
+
# bounds -- but a scope or a segment written in Ruby carries a Time or a
|
|
71
|
+
# Date, and neither has a timezone of its own.
|
|
72
|
+
def epoch(value, leaf)
|
|
73
|
+
case value
|
|
74
|
+
when DateTime then value.to_time.to_i
|
|
75
|
+
when Date then @zone.start_of_day(value)
|
|
76
|
+
when Time then value.to_i
|
|
77
|
+
when Numeric then seconds(value, leaf)
|
|
78
|
+
when String then parse(value, leaf)
|
|
79
|
+
else refuse_value!(leaf, value, 'a date')
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# An Infinity or a NaN -- a cast that overflowed above this datasource --
|
|
84
|
+
# makes `Integer()` raise a FloatDomainError naming a float where the
|
|
85
|
+
# operator asked for a date, and a Complex a RangeError, which is the
|
|
86
|
+
# same class one level up. Refused like every other value the field
|
|
87
|
+
# cannot take, for the reason `number` refuses them too.
|
|
88
|
+
def seconds(value, leaf)
|
|
89
|
+
Integer(value)
|
|
90
|
+
rescue RangeError, TypeError
|
|
91
|
+
refuse_value!(leaf, value, 'a date')
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Refuses a string lacking a year, month or day instead of letting a parser complete it: `Time.zone.parse`
|
|
95
|
+
# answers today for `last tuesday`, and `Time.parse` fills in the server's date for `12:00`.
|
|
96
|
+
def parse(value, leaf)
|
|
97
|
+
return @zone.start_of_day(Date.parse(value)) if DATE_ONLY.match?(value)
|
|
98
|
+
|
|
99
|
+
refuse_value!(leaf, value, 'a date') unless dated?(value)
|
|
100
|
+
|
|
101
|
+
@zone.timestamp(value)
|
|
102
|
+
rescue ArgumentError, TypeError
|
|
103
|
+
refuse_value!(leaf, value, 'a date')
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def dated?(value)
|
|
107
|
+
parts = Date._parse(value)
|
|
108
|
+
|
|
109
|
+
DATE_PARTS.all? { |part| parts.key?(part) }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The agent casts every Number column with `to_f`, so an integer field
|
|
113
|
+
# would be filtered with `42.0` -- a form none of its values carry. A float
|
|
114
|
+
# with nothing after the point travels as the integer it is.
|
|
115
|
+
#
|
|
116
|
+
# A cast that overflowed to Infinity, or a NaN, is refused rather than
|
|
117
|
+
# passed on: the JSON encoder raises on both, a step later, as a 500 naming
|
|
118
|
+
# nothing the operator can act on.
|
|
119
|
+
def number(value, leaf)
|
|
120
|
+
case value
|
|
121
|
+
when Integer then value
|
|
122
|
+
when Float then finite(value, leaf)
|
|
123
|
+
when String then value.match?(INTEGER) ? value.to_i : finite(Float(value, exception: false), leaf)
|
|
124
|
+
else refuse_value!(leaf, value, 'a number')
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def finite(value, leaf)
|
|
129
|
+
refuse_value!(leaf, value, 'a number') unless value.is_a?(Float) && value.finite?
|
|
130
|
+
|
|
131
|
+
value == value.to_i ? value.to_i : value
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def boolean(value, leaf)
|
|
135
|
+
case value
|
|
136
|
+
when true, false then value
|
|
137
|
+
when 'true' then true
|
|
138
|
+
when 'false' then false
|
|
139
|
+
else refuse_value!(leaf, value, 'a true or a false')
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def blank?(value) = value.nil? || value.to_s.empty?
|
|
144
|
+
|
|
145
|
+
# `present`, `blank` and `missing` are derived by the agent from an
|
|
146
|
+
# equality, above this datasource, and rewritten into a comparison with an
|
|
147
|
+
# empty value. Intercom's search matches values and has no spelling for the
|
|
148
|
+
# absence of one, so the rewritten condition is refused here rather than
|
|
149
|
+
# sent as a comparison against the empty string -- which Intercom would
|
|
150
|
+
# answer as if it were a value of its own.
|
|
151
|
+
def refuse_absence!(leaf, field)
|
|
152
|
+
raise UnsupportedOperatorError,
|
|
153
|
+
"#{@collection} cannot filter #{leaf.field.inspect} for absence: Intercom's search matches values " \
|
|
154
|
+
"and has no operator for the lack of one, so a #{leaf.operator} condition on " \
|
|
155
|
+
"#{field.field.inspect} cannot be translated. Filter on a value instead."
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def refuse_empty_list!(leaf)
|
|
159
|
+
raise UnsupportedOperatorError,
|
|
160
|
+
"#{@collection} was asked to filter #{leaf.field.inspect} with #{leaf.operator} and an empty list, " \
|
|
161
|
+
'which names no record and no filter. Pass at least one value.'
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def refuse_long_list!(leaf, size)
|
|
165
|
+
raise UnsupportedOperatorError,
|
|
166
|
+
"#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator} and #{size} values: " \
|
|
167
|
+
"Intercom takes #{MAX_LIST_SIZE} at most in one list. Narrow the list, or split the filter."
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def refuse_value!(leaf, value, expected)
|
|
171
|
+
raise UnsupportedOperatorError,
|
|
172
|
+
"#{@collection} cannot filter #{leaf.field.inspect} with #{value.inspect}: Intercom expects " \
|
|
173
|
+
"#{expected} on this field."
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
module ForestAdminDatasourceIntercom
|
|
2
|
+
module Query
|
|
3
|
+
# Read by both the schema and the translator, so a column never advertises an operator the translator
|
|
4
|
+
# refuses. Which operators an endpoint accepts per field lives in `search_fields.yml`.
|
|
5
|
+
module OperatorTable
|
|
6
|
+
Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
|
|
7
|
+
|
|
8
|
+
EQUALITY = { Operators::EQUAL => '=', Operators::NOT_EQUAL => '!=',
|
|
9
|
+
Operators::IN => 'IN', Operators::NOT_IN => 'NIN' }.freeze
|
|
10
|
+
|
|
11
|
+
# `contains` and `i_contains` both map to `~`, Intercom's only substring operator. `not_i_contains` is
|
|
12
|
+
# absent: the toolkit's `Rules` rejects it on a String column (PRD-989).
|
|
13
|
+
SUBSTRING = { Operators::CONTAINS => '~', Operators::I_CONTAINS => '~',
|
|
14
|
+
Operators::NOT_CONTAINS => '!~',
|
|
15
|
+
Operators::STARTS_WITH => '^', Operators::ENDS_WITH => '$' }.freeze
|
|
16
|
+
|
|
17
|
+
BOUNDS = { Operators::GREATER_THAN => '>', Operators::LESS_THAN => '<' }.freeze
|
|
18
|
+
|
|
19
|
+
# Dates publish only the two bounds although both endpoints accept `=`, `!=`, `>=`, `<=` (measured):
|
|
20
|
+
# declaring `equal` makes the toolkit republish `in`, which its validator refuses (PRD-989). The toolkit
|
|
21
|
+
# still derives `today`, `past`, `previous_*` and the rest from the two bounds.
|
|
22
|
+
MAPS = {
|
|
23
|
+
'string' => EQUALITY,
|
|
24
|
+
'boolean' => EQUALITY,
|
|
25
|
+
'number' => EQUALITY.merge(BOUNDS),
|
|
26
|
+
'text' => EQUALITY.merge(SUBSTRING),
|
|
27
|
+
'date' => BOUNDS
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
# The operators that take a list rather than a value, on the wire.
|
|
31
|
+
LIST_OPERATORS = %w[IN NIN].freeze
|
|
32
|
+
|
|
33
|
+
class << self
|
|
34
|
+
def types = MAPS.keys
|
|
35
|
+
|
|
36
|
+
# What the column publishes: the Forest operators whose Intercom spelling
|
|
37
|
+
# this endpoint accepts on this field, and no other.
|
|
38
|
+
def forest_operators(field)
|
|
39
|
+
MAPS.fetch(field.type).select { |_, spelling| field.operators.include?(spelling) }.keys
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# nil when the endpoint does not accept that operator on that field,
|
|
43
|
+
# which the translator turns into a refusal naming what it does accept.
|
|
44
|
+
def intercom_operator(field, forest_operator)
|
|
45
|
+
spelling = MAPS.fetch(field.type)[forest_operator]
|
|
46
|
+
|
|
47
|
+
spelling if spelling && field.operators.include?(spelling)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def list_operator?(spelling) = LIST_OPERATORS.include?(spelling)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|