@kairos-es/feed-algolia 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +66 -2
- package/dist/cjs/classify.js +157 -0
- package/dist/cjs/classify.js.map +1 -0
- package/dist/cjs/identifiers.js +38 -0
- package/dist/cjs/identifiers.js.map +1 -0
- package/dist/cjs/index.js +39 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/records.js +105 -0
- package/dist/cjs/records.js.map +1 -0
- package/dist/cjs/rest.js +293 -0
- package/dist/cjs/rest.js.map +1 -0
- package/dist/cjs/sink.js +463 -0
- package/dist/cjs/sink.js.map +1 -0
- package/dist/dts-cjs/classify.d.ts +101 -0
- package/dist/dts-cjs/classify.d.ts.map +1 -0
- package/dist/dts-cjs/identifiers.d.ts +34 -0
- package/dist/dts-cjs/identifiers.d.ts.map +1 -0
- package/dist/dts-cjs/index.d.ts +39 -0
- package/dist/dts-cjs/index.d.ts.map +1 -0
- package/dist/dts-cjs/package.json +3 -0
- package/dist/dts-cjs/records.d.ts +56 -0
- package/dist/dts-cjs/records.d.ts.map +1 -0
- package/dist/dts-cjs/rest.d.ts +215 -0
- package/dist/dts-cjs/rest.d.ts.map +1 -0
- package/dist/dts-cjs/sink.d.ts +220 -0
- package/dist/dts-cjs/sink.d.ts.map +1 -0
- package/dist/dts-esm/classify.d.ts +101 -0
- package/dist/dts-esm/classify.d.ts.map +1 -0
- package/dist/dts-esm/identifiers.d.ts +34 -0
- package/dist/dts-esm/identifiers.d.ts.map +1 -0
- package/dist/dts-esm/index.d.ts +39 -0
- package/dist/dts-esm/index.d.ts.map +1 -0
- package/dist/dts-esm/package.json +3 -0
- package/dist/dts-esm/records.d.ts +56 -0
- package/dist/dts-esm/records.d.ts.map +1 -0
- package/dist/dts-esm/rest.d.ts +215 -0
- package/dist/dts-esm/rest.d.ts.map +1 -0
- package/dist/dts-esm/sink.d.ts +220 -0
- package/dist/dts-esm/sink.d.ts.map +1 -0
- package/dist/esm/classify.js +145 -0
- package/dist/esm/classify.js.map +1 -0
- package/dist/esm/identifiers.js +31 -0
- package/dist/esm/identifiers.js.map +1 -0
- package/dist/esm/index.js +38 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/records.js +96 -0
- package/dist/esm/records.js.map +1 -0
- package/dist/esm/rest.js +285 -0
- package/dist/esm/rest.js.map +1 -0
- package/dist/esm/sink.js +454 -0
- package/dist/esm/sink.js.map +1 -0
- package/package.json +33 -3
- package/src/classify.ts +201 -0
- package/src/identifiers.ts +45 -0
- package/src/index.ts +48 -0
- package/src/records.ts +109 -0
- package/src/rest.ts +773 -0
- package/src/sink.ts +733 -0
package/src/rest.ts
ADDED
|
@@ -0,0 +1,773 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slice of Algolia's REST API this sink speaks, over `effect`'s own
|
|
3
|
+
* `HttpClient`.
|
|
4
|
+
*
|
|
5
|
+
* ## Why no SDK
|
|
6
|
+
*
|
|
7
|
+
* The surface the sink needs is eight calls — `batch`, `getTask`, `operationIndex`
|
|
8
|
+
* (copy and move), `deleteIndex`, `browse`, `listIndices`, and a settings read and
|
|
9
|
+
* write, which is how a resync target is made to exist and marked as its attempt's
|
|
10
|
+
* own — and each is one JSON request. Peering `algoliasearch` would bring a large
|
|
11
|
+
* non-Effect dependency for that, and its retries would sit underneath the feed
|
|
12
|
+
* driver's own supervision, which already owns retrying. Over `HttpClient` the
|
|
13
|
+
* sink is Effect-native, adds no peer beyond `effect`, and is faked in tests by a
|
|
14
|
+
* stub client with no network at all.
|
|
15
|
+
*
|
|
16
|
+
* ## Host failover
|
|
17
|
+
*
|
|
18
|
+
* The one piece of the SDK's transport kept here is the part the driver cannot do
|
|
19
|
+
* for itself: trying ANOTHER HOST. Algolia serves an application from
|
|
20
|
+
* `<appId>.algolia.net` and documents three fallbacks on separate infrastructure,
|
|
21
|
+
* `<appId>-1.algolianet.com` to `<appId>-3.algolianet.com`, to be tried when the
|
|
22
|
+
* first does not answer. A request that gets no answer (a transport failure, or no
|
|
23
|
+
* answer within its timeout), a 5xx, or a 1xx or 3xx — a status no call here
|
|
24
|
+
* expects, on which the official JavaScript client tries the next host too — is
|
|
25
|
+
* sent again at once to the next host, and only when every host it tried has
|
|
26
|
+
* failed is the call `SinkRetryable`, carrying the last host's failure. Without
|
|
27
|
+
* it, one unreachable host would hold the feed in the driver's back-off for as
|
|
28
|
+
* long as the host stayed down, with three healthy hosts unasked.
|
|
29
|
+
*
|
|
30
|
+
* - Hosts are STATEFUL, one state per client and so per sink, in the shape the
|
|
31
|
+
* official JavaScript client gives them. A host that failed in one of those ways
|
|
32
|
+
* is marked DOWN for two minutes, by `Clock`, and skipped by every call while it
|
|
33
|
+
* is down; a host that answers is marked up again. Without that memory a primary
|
|
34
|
+
* that drops packets would cost a whole timeout on EVERY call, each `getTask`
|
|
35
|
+
* poll included, so a single wait for a task could spend its whole
|
|
36
|
+
* `taskPolling.timeout` on the dead host. When every host is down, every host is
|
|
37
|
+
* tried again, in order, rather than none.
|
|
38
|
+
* - The fallbacks are SHUFFLED, once, when the client is built, from Effect's
|
|
39
|
+
* `Random`, so many workers do not all fail over onto the same host; the primary
|
|
40
|
+
* goes first whenever it is up.
|
|
41
|
+
* - Each hand-off to the next host is logged at DEBUG level, with the call, the
|
|
42
|
+
* host that failed, the host tried next and why. A call that lands after a
|
|
43
|
+
* failover looks like any other, so that line is the only trace one leaves.
|
|
44
|
+
* Hosts carry the application id, which is not a secret; the key travels in a
|
|
45
|
+
* header and reaches no log.
|
|
46
|
+
* - Three timeouts, as the official client has: `connect`, `read` and `write`,
|
|
47
|
+
* defaulting to its 2, 5 and 30 seconds. A read is a `GET` or a `browse`, and
|
|
48
|
+
* every other call is a write. `HttpClient` does not show this module the moment
|
|
49
|
+
* a connection is made, so one request's budget on one host is `connect` plus
|
|
50
|
+
* `read`, or `connect` plus `write`, over the whole exchange. A transport that
|
|
51
|
+
* enforces a connect timeout of its own fails sooner, and that failure fails
|
|
52
|
+
* over the same way.
|
|
53
|
+
* - Every call uses the same list, reads included. The SDK sends reads to
|
|
54
|
+
* `<appId>-dsn.algolia.net` first, which is the search-serving network; every
|
|
55
|
+
* read here is an admin read (a task's status, a listing, a settings read, a
|
|
56
|
+
* browse for reconcile) that wants the cluster the writes went to.
|
|
57
|
+
* - A 429 is NOT retried on another host: it is the application's rate limit or
|
|
58
|
+
* quota, not a host's health (`classify.ts` owns which). Nor is any other 4xx,
|
|
59
|
+
* which another host would answer the same.
|
|
60
|
+
* - A request can be sent more than once this way, and that is safe for the
|
|
61
|
+
* reason every resend here is (`classify.ts`): every write is a full-record
|
|
62
|
+
* replace, a delete, an index operation or a settings write, each of which
|
|
63
|
+
* leaves the application the same however often it lands.
|
|
64
|
+
* - The worst case for one call is therefore its budget times the number of hosts
|
|
65
|
+
* that are up when it starts.
|
|
66
|
+
* - An explicit `host` turns failover OFF: the caller has named the one place to
|
|
67
|
+
* talk to (a proxy, a test double), and there are no fallbacks of it to guess.
|
|
68
|
+
* That one host is never skipped, since every host being down tries it again.
|
|
69
|
+
*
|
|
70
|
+
* ## The admin key
|
|
71
|
+
*
|
|
72
|
+
* It is held as `Redacted` and unwrapped only into the request header, at the
|
|
73
|
+
* moment the request is built. Two things keep it out of anything printed: no
|
|
74
|
+
* failure this module produces carries the request or its headers — only the
|
|
75
|
+
* method, the path and Algolia's own message (`classify.ts`) — and every request
|
|
76
|
+
* runs with Algolia's key header added to the names `effect`'s `Headers` redact,
|
|
77
|
+
* so a request that is logged or traced by something further down shows the
|
|
78
|
+
* header as redacted.
|
|
79
|
+
*
|
|
80
|
+
* ## Not found
|
|
81
|
+
*
|
|
82
|
+
* Three of the calls meet a missing index as an ordinary outcome rather than a
|
|
83
|
+
* failure — dropping a leftover resync index that is not there, browsing an index
|
|
84
|
+
* that does not exist yet, reading the settings of an index that does not exist —
|
|
85
|
+
* so they answer a 404 as `Option.none()` instead of a `SinkHalted`. Every other
|
|
86
|
+
* call halts on a 404, with Algolia's own message in the reason. An index
|
|
87
|
+
* operation is one of those: Algolia documents a copy of a missing source as
|
|
88
|
+
* creating an empty destination, and a move of one as ignored without an error,
|
|
89
|
+
* so neither is a 404 (the sink's `publish` owns what that costs). A 404 is never
|
|
90
|
+
* failed over: another host would answer the same.
|
|
91
|
+
*/
|
|
92
|
+
import type { FeedKey } from '@kairos-es/feed'
|
|
93
|
+
import {
|
|
94
|
+
Array as Arr,
|
|
95
|
+
Clock,
|
|
96
|
+
Duration,
|
|
97
|
+
Effect,
|
|
98
|
+
Option,
|
|
99
|
+
Random,
|
|
100
|
+
Redacted,
|
|
101
|
+
Ref,
|
|
102
|
+
Schema,
|
|
103
|
+
} from 'effect'
|
|
104
|
+
import type { NonEmptyReadonlyArray } from 'effect/Array'
|
|
105
|
+
import {
|
|
106
|
+
Headers,
|
|
107
|
+
HttpBody,
|
|
108
|
+
HttpClient,
|
|
109
|
+
type HttpClientError,
|
|
110
|
+
HttpClientRequest,
|
|
111
|
+
type HttpMethod,
|
|
112
|
+
} from 'effect/http'
|
|
113
|
+
import {
|
|
114
|
+
type AlgoliaCallLabel,
|
|
115
|
+
type AlgoliaFailure,
|
|
116
|
+
classifyAnswer,
|
|
117
|
+
transportFailure,
|
|
118
|
+
unreadableAnswer,
|
|
119
|
+
} from './classify'
|
|
120
|
+
import {
|
|
121
|
+
type AlgoliaAppId,
|
|
122
|
+
type AlgoliaIndexName,
|
|
123
|
+
AlgoliaTaskId,
|
|
124
|
+
} from './identifiers'
|
|
125
|
+
|
|
126
|
+
/** Where and as whom the sink talks to Algolia. */
|
|
127
|
+
export interface AlgoliaConnection {
|
|
128
|
+
readonly appId: AlgoliaAppId
|
|
129
|
+
/** The admin API key. Never logged; see the module doc. */
|
|
130
|
+
readonly adminKey: Redacted.Redacted<string>
|
|
131
|
+
/**
|
|
132
|
+
* ONE base URL to send every call to, with no failover. Leave it unset to talk
|
|
133
|
+
* to Algolia itself: `https://<appId>.algolia.net`, failing over to the three
|
|
134
|
+
* documented fallback hosts (see the module doc). Set it for a proxy or a test
|
|
135
|
+
* double, whose fallbacks there is no way to know.
|
|
136
|
+
*/
|
|
137
|
+
readonly host?: string
|
|
138
|
+
/**
|
|
139
|
+
* How long one request to one host may take before it counts as unanswered,
|
|
140
|
+
* that host is marked down, and the next is tried; see the module doc.
|
|
141
|
+
*/
|
|
142
|
+
readonly timeouts?: AlgoliaTimeouts
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The three per-host timeouts, with the official JavaScript client's defaults. A
|
|
147
|
+
* request's budget on one host is `connect` plus `read` for a read, or `connect`
|
|
148
|
+
* plus `write` for a write; see the module doc for why they are added.
|
|
149
|
+
*/
|
|
150
|
+
export interface AlgoliaTimeouts {
|
|
151
|
+
/** Default 2 seconds. */
|
|
152
|
+
readonly connect?: Duration.Input
|
|
153
|
+
/** For a `GET` or a `browse`. Default 5 seconds. */
|
|
154
|
+
readonly read?: Duration.Input
|
|
155
|
+
/** For every other call. Default 30 seconds. */
|
|
156
|
+
readonly write?: Duration.Input
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** The header Algolia reads the admin key from, lower-cased as `Headers` keeps it. */
|
|
160
|
+
const KEY_HEADER = 'x-algolia-api-key'
|
|
161
|
+
const APP_HEADER = 'x-algolia-application-id'
|
|
162
|
+
|
|
163
|
+
const DEFAULT_CONNECT_TIMEOUT = Duration.seconds(2)
|
|
164
|
+
const DEFAULT_READ_TIMEOUT = Duration.seconds(5)
|
|
165
|
+
const DEFAULT_WRITE_TIMEOUT = Duration.seconds(30)
|
|
166
|
+
|
|
167
|
+
/** How long a host that failed is skipped for: the official client's two minutes. */
|
|
168
|
+
const HOST_DOWN_MILLIS = Duration.toMillis(Duration.minutes(2))
|
|
169
|
+
|
|
170
|
+
/** One batch operation, as Algolia's `batch` takes it. Full-record writes only. */
|
|
171
|
+
export type BatchOperation =
|
|
172
|
+
| {
|
|
173
|
+
readonly action: 'addObject'
|
|
174
|
+
readonly body: Readonly<Record<string, unknown>>
|
|
175
|
+
}
|
|
176
|
+
| {
|
|
177
|
+
readonly action: 'deleteObject'
|
|
178
|
+
readonly body: { readonly objectID: string }
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** What an index-level `operationIndex` call asks for. */
|
|
182
|
+
export interface IndexOperation {
|
|
183
|
+
readonly operation: 'copy' | 'move'
|
|
184
|
+
readonly destination: AlgoliaIndexName
|
|
185
|
+
readonly scope?: ReadonlyArray<'settings' | 'synonyms' | 'rules'>
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** One page of a `browse`: the ids on it, and the cursor to the next, if any. */
|
|
189
|
+
export interface BrowsePage {
|
|
190
|
+
readonly objectIDs: ReadonlyArray<string>
|
|
191
|
+
readonly cursor: Option.Option<string>
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** One page of `listIndices`: the index names on it, and whether more follow. */
|
|
195
|
+
export interface IndexListPage {
|
|
196
|
+
readonly names: ReadonlyArray<string>
|
|
197
|
+
readonly more: boolean
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** One raw call: everything the typed calls below are built from. */
|
|
201
|
+
export interface AlgoliaCall {
|
|
202
|
+
readonly method: HttpMethod.HttpMethod
|
|
203
|
+
/** The path, already encoded: `/1/indexes/<name>/...`. */
|
|
204
|
+
readonly path: string
|
|
205
|
+
readonly body?: unknown
|
|
206
|
+
/** The keys the request carries, so a 400 naming one can be a rejection. */
|
|
207
|
+
readonly batchKeys?: ReadonlySet<FeedKey>
|
|
208
|
+
/**
|
|
209
|
+
* A read whose method is not `GET` (a `browse`), so it gets the read timeout.
|
|
210
|
+
* Every `GET` is a read already.
|
|
211
|
+
*/
|
|
212
|
+
readonly read?: true
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Where a task stands, as a caller that tolerates a missing index sees it: still
|
|
217
|
+
* running, published, or unanswerable because its index does not exist.
|
|
218
|
+
*/
|
|
219
|
+
export type AlgoliaTaskState = 'pending' | 'published' | 'indexMissing'
|
|
220
|
+
|
|
221
|
+
/** The Algolia calls the sink makes. Every failure is already classified. */
|
|
222
|
+
export interface AlgoliaRest {
|
|
223
|
+
/** A raw call; `Option.none()` on a 404. Exposed for the package's own tests. */
|
|
224
|
+
readonly call: (
|
|
225
|
+
call: AlgoliaCall,
|
|
226
|
+
) => Effect.Effect<Option.Option<unknown>, AlgoliaFailure>
|
|
227
|
+
readonly batch: (
|
|
228
|
+
index: AlgoliaIndexName,
|
|
229
|
+
operations: ReadonlyArray<BatchOperation>,
|
|
230
|
+
batchKeys: ReadonlySet<FeedKey>,
|
|
231
|
+
) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
|
|
232
|
+
readonly taskPublished: (
|
|
233
|
+
index: AlgoliaIndexName,
|
|
234
|
+
taskID: AlgoliaTaskId,
|
|
235
|
+
) => Effect.Effect<boolean, AlgoliaFailure>
|
|
236
|
+
/**
|
|
237
|
+
* `taskPublished`, as a state, with a 404 — Algolia's answer to `getTask` on an
|
|
238
|
+
* index that does not exist — as `'indexMissing'` rather than a halt, for a
|
|
239
|
+
* caller that has another way to learn what the task did.
|
|
240
|
+
*/
|
|
241
|
+
readonly taskStateUnlessIndexMissing: (
|
|
242
|
+
index: AlgoliaIndexName,
|
|
243
|
+
taskID: AlgoliaTaskId,
|
|
244
|
+
) => Effect.Effect<AlgoliaTaskState, AlgoliaFailure>
|
|
245
|
+
/**
|
|
246
|
+
* A copy or a move. Algolia answers a missing source with a task too: a copy
|
|
247
|
+
* creates an empty destination, and a move does nothing (see the module doc).
|
|
248
|
+
*/
|
|
249
|
+
readonly operation: (
|
|
250
|
+
source: AlgoliaIndexName,
|
|
251
|
+
operation: IndexOperation,
|
|
252
|
+
) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
|
|
253
|
+
/** `Option.none()` when the index does not exist. */
|
|
254
|
+
readonly deleteIndex: (
|
|
255
|
+
index: AlgoliaIndexName,
|
|
256
|
+
) => Effect.Effect<Option.Option<AlgoliaTaskId>, AlgoliaFailure>
|
|
257
|
+
/** An index's settings; `Option.none()` when the index does not exist. */
|
|
258
|
+
readonly readSettings: (
|
|
259
|
+
index: AlgoliaIndexName,
|
|
260
|
+
) => Effect.Effect<
|
|
261
|
+
Option.Option<Readonly<Record<string, unknown>>>,
|
|
262
|
+
AlgoliaFailure
|
|
263
|
+
>
|
|
264
|
+
/**
|
|
265
|
+
* A settings write of the settings named, and only those; a `null` resets one
|
|
266
|
+
* to its default. Algolia creates an index on a settings write, so this is also
|
|
267
|
+
* how an index is made to exist without a record.
|
|
268
|
+
*/
|
|
269
|
+
readonly writeSettings: (
|
|
270
|
+
index: AlgoliaIndexName,
|
|
271
|
+
settings: Readonly<Record<string, unknown>>,
|
|
272
|
+
) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
|
|
273
|
+
/**
|
|
274
|
+
* One page of the application's index names, `page` counting from 0. Every
|
|
275
|
+
* index the key may see is listed, so a caller filters by name.
|
|
276
|
+
*/
|
|
277
|
+
readonly listIndexNames: (
|
|
278
|
+
page: number,
|
|
279
|
+
hitsPerPage: number,
|
|
280
|
+
) => Effect.Effect<IndexListPage, AlgoliaFailure>
|
|
281
|
+
/** A missing index browses as one empty last page. */
|
|
282
|
+
readonly browseObjectIds: (
|
|
283
|
+
index: AlgoliaIndexName,
|
|
284
|
+
cursor: Option.Option<string>,
|
|
285
|
+
hitsPerPage: number,
|
|
286
|
+
) => Effect.Effect<BrowsePage, AlgoliaFailure>
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** The path of an index, or of something under it. */
|
|
290
|
+
export const indexPath = (index: AlgoliaIndexName, rest = ''): string =>
|
|
291
|
+
`/1/indexes/${encodeURIComponent(index)}${rest}`
|
|
292
|
+
|
|
293
|
+
const TaskAnswer = Schema.Struct({ taskID: AlgoliaTaskId })
|
|
294
|
+
const TaskStatus = Schema.Struct({
|
|
295
|
+
status: Schema.Literals(['published', 'notPublished']),
|
|
296
|
+
})
|
|
297
|
+
const IndexListAnswer = Schema.Struct({
|
|
298
|
+
items: Schema.Array(Schema.Struct({ name: Schema.String })),
|
|
299
|
+
nbPages: Schema.optionalKey(Schema.Number),
|
|
300
|
+
})
|
|
301
|
+
const SettingsAnswer = Schema.Record(Schema.String, Schema.Unknown)
|
|
302
|
+
const BrowseAnswer = Schema.Struct({
|
|
303
|
+
hits: Schema.Array(Schema.Struct({ objectID: Schema.String })),
|
|
304
|
+
cursor: Schema.optionalKey(Schema.String),
|
|
305
|
+
})
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* The most of a raw, non-JSON error body a reason carries. A proxy's or a load
|
|
309
|
+
* balancer's error page can be any size, and a reason is logged whole.
|
|
310
|
+
*/
|
|
311
|
+
const RAW_BODY_LIMIT = 500
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Algolia's error body is `{ message, status }`; anything else is shown raw, cut
|
|
315
|
+
* to {@link RAW_BODY_LIMIT} characters.
|
|
316
|
+
*/
|
|
317
|
+
const errorMessage = (text: string): string => {
|
|
318
|
+
try {
|
|
319
|
+
const parsed: unknown = JSON.parse(text)
|
|
320
|
+
if (
|
|
321
|
+
typeof parsed === 'object' &&
|
|
322
|
+
parsed !== null &&
|
|
323
|
+
'message' in parsed &&
|
|
324
|
+
typeof parsed.message === 'string'
|
|
325
|
+
) {
|
|
326
|
+
return parsed.message
|
|
327
|
+
}
|
|
328
|
+
} catch {
|
|
329
|
+
// Not JSON: the raw body is the best message there is.
|
|
330
|
+
}
|
|
331
|
+
if (text.length === 0) {
|
|
332
|
+
return '(empty body)'
|
|
333
|
+
}
|
|
334
|
+
return text.length <= RAW_BODY_LIMIT
|
|
335
|
+
? text
|
|
336
|
+
: `${text.slice(0, RAW_BODY_LIMIT)}… (${text.length - RAW_BODY_LIMIT} more characters)`
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* A status that says nothing about the request and something about the host: a
|
|
341
|
+
* 5xx, or a 1xx or 3xx, which no call here expects. Tried on the next host.
|
|
342
|
+
*/
|
|
343
|
+
const hostFailedWith = (status: number): boolean =>
|
|
344
|
+
status < 200 || (status >= 300 && status < 400) || status >= 500
|
|
345
|
+
|
|
346
|
+
const label = (call: AlgoliaCall): AlgoliaCallLabel => ({
|
|
347
|
+
method: call.method,
|
|
348
|
+
path: call.path,
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
/** An answer as it came back from one host, before it is classified. */
|
|
352
|
+
interface RawAnswer {
|
|
353
|
+
readonly status: number
|
|
354
|
+
readonly text: string
|
|
355
|
+
readonly retryAfter: Option.Option<string>
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** A classified answer: its JSON, or the 404 it was, with the failure it makes. */
|
|
359
|
+
type Sent =
|
|
360
|
+
| { readonly found: true; readonly json: unknown }
|
|
361
|
+
| { readonly found: false; readonly failure: AlgoliaFailure }
|
|
362
|
+
|
|
363
|
+
/** One `getTask` poll: published or not, or a 404 and the failure it makes. */
|
|
364
|
+
type TaskPoll = boolean | { readonly indexMissing: AlgoliaFailure }
|
|
365
|
+
|
|
366
|
+
/** The fallback hosts' numbers: `<appId>-<n>.algolianet.com`. */
|
|
367
|
+
const FALLBACKS = [1, 2, 3] as const
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* The base URLs a call is tried on, in order; see the module doc. Shuffled from
|
|
371
|
+
* `Random`, so a test can seed it.
|
|
372
|
+
*/
|
|
373
|
+
const hostsFor = (
|
|
374
|
+
connection: AlgoliaConnection,
|
|
375
|
+
): Effect.Effect<NonEmptyReadonlyArray<string>> =>
|
|
376
|
+
connection.host === undefined
|
|
377
|
+
? Effect.map(
|
|
378
|
+
Random.shuffle(
|
|
379
|
+
FALLBACKS.map(
|
|
380
|
+
(n) => `https://${connection.appId}-${n}.algolianet.com`,
|
|
381
|
+
),
|
|
382
|
+
),
|
|
383
|
+
(fallbacks) =>
|
|
384
|
+
[`https://${connection.appId}.algolia.net`, ...fallbacks] as const,
|
|
385
|
+
)
|
|
386
|
+
: Effect.succeed([connection.host.replace(/\/+$/, '')] as const)
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Build the REST client over the `HttpClient` in context. The client is captured
|
|
390
|
+
* here, so every call the sink makes later needs nothing from its caller.
|
|
391
|
+
*/
|
|
392
|
+
export const makeAlgoliaRest = (
|
|
393
|
+
connection: AlgoliaConnection,
|
|
394
|
+
): Effect.Effect<AlgoliaRest, never, HttpClient.HttpClient> =>
|
|
395
|
+
Effect.gen(function* () {
|
|
396
|
+
const http = yield* HttpClient.HttpClient
|
|
397
|
+
const hosts = yield* hostsFor(connection)
|
|
398
|
+
const timeoutOf = (
|
|
399
|
+
input: Duration.Input | undefined,
|
|
400
|
+
fallback: Duration.Duration,
|
|
401
|
+
) => (input === undefined ? fallback : Duration.fromInputUnsafe(input))
|
|
402
|
+
const connect = timeoutOf(
|
|
403
|
+
connection.timeouts?.connect,
|
|
404
|
+
DEFAULT_CONNECT_TIMEOUT,
|
|
405
|
+
)
|
|
406
|
+
const readBudget = Duration.sum(
|
|
407
|
+
connect,
|
|
408
|
+
timeoutOf(connection.timeouts?.read, DEFAULT_READ_TIMEOUT),
|
|
409
|
+
)
|
|
410
|
+
const writeBudget = Duration.sum(
|
|
411
|
+
connect,
|
|
412
|
+
timeoutOf(connection.timeouts?.write, DEFAULT_WRITE_TIMEOUT),
|
|
413
|
+
)
|
|
414
|
+
/** When each host that failed comes back up, in `Clock` millis. */
|
|
415
|
+
const downUntil = yield* Ref.make<ReadonlyMap<string, number>>(new Map())
|
|
416
|
+
const redactedNames = [
|
|
417
|
+
...Headers.CurrentRedactedNames.defaultValue(),
|
|
418
|
+
KEY_HEADER,
|
|
419
|
+
]
|
|
420
|
+
|
|
421
|
+
const fromClientError = (
|
|
422
|
+
call: AlgoliaCall,
|
|
423
|
+
error: HttpClientError.HttpClientError,
|
|
424
|
+
): AlgoliaFailure =>
|
|
425
|
+
error.reason._tag === 'TransportError' ||
|
|
426
|
+
error.reason._tag === 'DecodeError' ||
|
|
427
|
+
error.reason._tag === 'EmptyBodyError'
|
|
428
|
+
? transportFailure(label(call), error.reason._tag)
|
|
429
|
+
: unreadableAnswer(label(call), error.reason._tag)
|
|
430
|
+
|
|
431
|
+
/** One request to one host, unclassified. Fails only `SinkRetryable` or halted. */
|
|
432
|
+
const exchange = (
|
|
433
|
+
host: string,
|
|
434
|
+
spec: AlgoliaCall,
|
|
435
|
+
): Effect.Effect<RawAnswer, AlgoliaFailure> => {
|
|
436
|
+
const budget =
|
|
437
|
+
spec.method === 'GET' || spec.read === true ? readBudget : writeBudget
|
|
438
|
+
const request = HttpClientRequest.make(spec.method)(
|
|
439
|
+
`${host}${spec.path}`,
|
|
440
|
+
{
|
|
441
|
+
headers: {
|
|
442
|
+
[APP_HEADER]: connection.appId,
|
|
443
|
+
[KEY_HEADER]: Redacted.value(connection.adminKey),
|
|
444
|
+
},
|
|
445
|
+
acceptJson: true,
|
|
446
|
+
...(spec.body === undefined
|
|
447
|
+
? {}
|
|
448
|
+
: { body: HttpBody.jsonUnsafe(spec.body) }),
|
|
449
|
+
},
|
|
450
|
+
)
|
|
451
|
+
return Effect.flatMap(http.execute(request), (response) =>
|
|
452
|
+
Effect.map(
|
|
453
|
+
response.text,
|
|
454
|
+
(text): RawAnswer => ({
|
|
455
|
+
status: response.status,
|
|
456
|
+
text,
|
|
457
|
+
retryAfter: Headers.get(response.headers, 'retry-after'),
|
|
458
|
+
}),
|
|
459
|
+
),
|
|
460
|
+
).pipe(
|
|
461
|
+
Effect.mapError((error) => fromClientError(spec, error)),
|
|
462
|
+
Effect.timeoutOrElse({
|
|
463
|
+
duration: budget,
|
|
464
|
+
orElse: () =>
|
|
465
|
+
Effect.fail(
|
|
466
|
+
transportFailure(
|
|
467
|
+
label(spec),
|
|
468
|
+
`no answer within ${Duration.format(budget)}`,
|
|
469
|
+
),
|
|
470
|
+
),
|
|
471
|
+
}),
|
|
472
|
+
)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const markDown = (host: string) =>
|
|
476
|
+
Effect.flatMap(Clock.currentTimeMillis, (now) =>
|
|
477
|
+
Ref.update(downUntil, (down) =>
|
|
478
|
+
new Map(down).set(host, now + HOST_DOWN_MILLIS),
|
|
479
|
+
),
|
|
480
|
+
)
|
|
481
|
+
const markUp = (host: string) =>
|
|
482
|
+
Ref.update(downUntil, (down) => {
|
|
483
|
+
if (!down.has(host)) {
|
|
484
|
+
return down
|
|
485
|
+
}
|
|
486
|
+
const next = new Map(down)
|
|
487
|
+
next.delete(host)
|
|
488
|
+
return next
|
|
489
|
+
})
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* Say, at debug level, that a call is leaving `host` for `next` and why: the
|
|
493
|
+
* only trace a failover leaves, since the call that lands looks like any
|
|
494
|
+
* other. Hosts and the call's label only; the key is in a header, which
|
|
495
|
+
* nothing here reads.
|
|
496
|
+
*/
|
|
497
|
+
const logFailover = (
|
|
498
|
+
spec: AlgoliaCall,
|
|
499
|
+
host: string,
|
|
500
|
+
next: string,
|
|
501
|
+
why: string,
|
|
502
|
+
) =>
|
|
503
|
+
Effect.logDebug('feed-algolia: failing over to the next host', {
|
|
504
|
+
call: `${spec.method} ${spec.path}`,
|
|
505
|
+
failed: host,
|
|
506
|
+
next,
|
|
507
|
+
why,
|
|
508
|
+
})
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* The request on each host in turn until one answers with a status that is
|
|
512
|
+
* the request's own rather than the host's; see the module doc. The last
|
|
513
|
+
* host's outcome is returned whatever it is, so a call every host failed ends
|
|
514
|
+
* as the last one did.
|
|
515
|
+
*/
|
|
516
|
+
const tryHosts = (
|
|
517
|
+
spec: AlgoliaCall,
|
|
518
|
+
[host, ...rest]: NonEmptyReadonlyArray<string>,
|
|
519
|
+
): Effect.Effect<RawAnswer, AlgoliaFailure> =>
|
|
520
|
+
// `matchEffect`, so a failure of the NEXT host's attempt is not caught here
|
|
521
|
+
// again: each host is asked once per call.
|
|
522
|
+
Effect.matchEffect(exchange(host, spec), {
|
|
523
|
+
onFailure: (failure) =>
|
|
524
|
+
failure._tag !== 'SinkRetryable'
|
|
525
|
+
? Effect.fail(failure)
|
|
526
|
+
: Effect.andThen(
|
|
527
|
+
markDown(host),
|
|
528
|
+
Arr.isReadonlyArrayNonEmpty(rest)
|
|
529
|
+
? Effect.andThen(
|
|
530
|
+
logFailover(spec, host, rest[0], failure.reason),
|
|
531
|
+
tryHosts(spec, rest),
|
|
532
|
+
)
|
|
533
|
+
: Effect.fail(failure),
|
|
534
|
+
),
|
|
535
|
+
onSuccess: (answer) =>
|
|
536
|
+
!hostFailedWith(answer.status)
|
|
537
|
+
? Effect.as(markUp(host), answer)
|
|
538
|
+
: Effect.andThen(
|
|
539
|
+
markDown(host),
|
|
540
|
+
Arr.isReadonlyArrayNonEmpty(rest)
|
|
541
|
+
? Effect.andThen(
|
|
542
|
+
logFailover(
|
|
543
|
+
spec,
|
|
544
|
+
host,
|
|
545
|
+
rest[0],
|
|
546
|
+
`answered ${answer.status}`,
|
|
547
|
+
),
|
|
548
|
+
tryHosts(spec, rest),
|
|
549
|
+
)
|
|
550
|
+
: Effect.succeed(answer),
|
|
551
|
+
),
|
|
552
|
+
})
|
|
553
|
+
|
|
554
|
+
/** The hosts that are up, in order; every host when none is. */
|
|
555
|
+
const acrossHosts = (
|
|
556
|
+
spec: AlgoliaCall,
|
|
557
|
+
): Effect.Effect<RawAnswer, AlgoliaFailure> =>
|
|
558
|
+
Effect.gen(function* () {
|
|
559
|
+
const now = yield* Clock.currentTimeMillis
|
|
560
|
+
const down = yield* Ref.get(downUntil)
|
|
561
|
+
const up = hosts.filter((host) => (down.get(host) ?? 0) <= now)
|
|
562
|
+
return yield* tryHosts(
|
|
563
|
+
spec,
|
|
564
|
+
Arr.isReadonlyArrayNonEmpty(up) ? up : hosts,
|
|
565
|
+
)
|
|
566
|
+
})
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* One call, classified. A 404 is answered rather than failed, beside the
|
|
570
|
+
* failure it would be — Algolia's message kept — so each caller decides
|
|
571
|
+
* whether a missing index is an outcome or a halt.
|
|
572
|
+
*/
|
|
573
|
+
const send = (spec: AlgoliaCall): Effect.Effect<Sent, AlgoliaFailure> =>
|
|
574
|
+
Effect.gen(function* () {
|
|
575
|
+
const answer = yield* acrossHosts(spec)
|
|
576
|
+
if (answer.status >= 200 && answer.status < 300) {
|
|
577
|
+
try {
|
|
578
|
+
return {
|
|
579
|
+
found: true,
|
|
580
|
+
json:
|
|
581
|
+
answer.text.length === 0
|
|
582
|
+
? {}
|
|
583
|
+
: (JSON.parse(answer.text) as unknown),
|
|
584
|
+
} satisfies Sent
|
|
585
|
+
} catch {
|
|
586
|
+
return yield* Effect.fail(
|
|
587
|
+
unreadableAnswer(label(spec), 'the body is not JSON'),
|
|
588
|
+
)
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
const now = yield* Clock.currentTimeMillis
|
|
592
|
+
const failure = classifyAnswer(
|
|
593
|
+
label(spec),
|
|
594
|
+
{
|
|
595
|
+
status: answer.status,
|
|
596
|
+
message: errorMessage(answer.text),
|
|
597
|
+
retryAfter: answer.retryAfter,
|
|
598
|
+
},
|
|
599
|
+
spec.batchKeys ?? new Set(),
|
|
600
|
+
now,
|
|
601
|
+
)
|
|
602
|
+
return answer.status === 404
|
|
603
|
+
? ({ found: false, failure } satisfies Sent)
|
|
604
|
+
: yield* Effect.fail(failure)
|
|
605
|
+
}).pipe(
|
|
606
|
+
Effect.provideService(Headers.CurrentRedactedNames, redactedNames),
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
const call = (spec: AlgoliaCall) =>
|
|
610
|
+
Effect.map(send(spec), (sent) =>
|
|
611
|
+
sent.found ? Option.some(sent.json) : Option.none(),
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
/** A call whose 404 is a failure like any other 4xx, Algolia's message kept. */
|
|
615
|
+
const required = (spec: AlgoliaCall) =>
|
|
616
|
+
Effect.flatMap(send(spec), (sent) =>
|
|
617
|
+
sent.found ? Effect.succeed(sent.json) : Effect.fail(sent.failure),
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
const decode =
|
|
621
|
+
<S extends Schema.Top & { readonly DecodingServices: never }>(
|
|
622
|
+
schema: S,
|
|
623
|
+
spec: AlgoliaCall,
|
|
624
|
+
) =>
|
|
625
|
+
(json: unknown): Effect.Effect<S['Type'], AlgoliaFailure> =>
|
|
626
|
+
Option.match(Schema.decodeUnknownOption(schema)(json), {
|
|
627
|
+
onNone: () =>
|
|
628
|
+
Effect.fail(
|
|
629
|
+
unreadableAnswer(
|
|
630
|
+
label(spec),
|
|
631
|
+
'the body is not the documented shape',
|
|
632
|
+
),
|
|
633
|
+
),
|
|
634
|
+
onSome: Effect.succeed,
|
|
635
|
+
})
|
|
636
|
+
|
|
637
|
+
/** One `getTask`; a 404 is answered beside its failure, as `send` does. */
|
|
638
|
+
const taskPoll = (index: AlgoliaIndexName, taskID: AlgoliaTaskId) => {
|
|
639
|
+
const spec: AlgoliaCall = {
|
|
640
|
+
method: 'GET',
|
|
641
|
+
path: indexPath(index, `/task/${taskID}`),
|
|
642
|
+
}
|
|
643
|
+
return Effect.flatMap(
|
|
644
|
+
send(spec),
|
|
645
|
+
(sent): Effect.Effect<TaskPoll, AlgoliaFailure> =>
|
|
646
|
+
sent.found
|
|
647
|
+
? Effect.map(
|
|
648
|
+
decode(TaskStatus, spec)(sent.json),
|
|
649
|
+
(answer) => answer.status === 'published',
|
|
650
|
+
)
|
|
651
|
+
: Effect.succeed({ indexMissing: sent.failure }),
|
|
652
|
+
)
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
const taskOf = (spec: AlgoliaCall) => (json: unknown) =>
|
|
656
|
+
Effect.map(decode(TaskAnswer, spec)(json), (answer) => answer.taskID)
|
|
657
|
+
|
|
658
|
+
/** A call whose 404 means "no such index", answered with a task otherwise. */
|
|
659
|
+
const optionalTask = (spec: AlgoliaCall) =>
|
|
660
|
+
Effect.flatMap(call(spec), (answer) =>
|
|
661
|
+
Option.match(answer, {
|
|
662
|
+
onNone: () => Effect.succeed(Option.none<AlgoliaTaskId>()),
|
|
663
|
+
onSome: (json) => Effect.map(taskOf(spec)(json), Option.some),
|
|
664
|
+
}),
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
return {
|
|
668
|
+
call,
|
|
669
|
+
batch: (index, operations, batchKeys) => {
|
|
670
|
+
const spec: AlgoliaCall = {
|
|
671
|
+
method: 'POST',
|
|
672
|
+
path: indexPath(index, '/batch'),
|
|
673
|
+
body: { requests: operations },
|
|
674
|
+
batchKeys,
|
|
675
|
+
}
|
|
676
|
+
return Effect.flatMap(required(spec), taskOf(spec))
|
|
677
|
+
},
|
|
678
|
+
taskPublished: (index, taskID) =>
|
|
679
|
+
Effect.flatMap(taskPoll(index, taskID), (poll) =>
|
|
680
|
+
typeof poll === 'boolean'
|
|
681
|
+
? Effect.succeed(poll)
|
|
682
|
+
: Effect.fail(poll.indexMissing),
|
|
683
|
+
),
|
|
684
|
+
taskStateUnlessIndexMissing: (index, taskID) =>
|
|
685
|
+
Effect.map(
|
|
686
|
+
taskPoll(index, taskID),
|
|
687
|
+
(poll): AlgoliaTaskState =>
|
|
688
|
+
typeof poll !== 'boolean'
|
|
689
|
+
? 'indexMissing'
|
|
690
|
+
: poll
|
|
691
|
+
? 'published'
|
|
692
|
+
: 'pending',
|
|
693
|
+
),
|
|
694
|
+
operation: (source, operation) => {
|
|
695
|
+
const spec: AlgoliaCall = {
|
|
696
|
+
method: 'POST',
|
|
697
|
+
path: indexPath(source, '/operation'),
|
|
698
|
+
body: operation,
|
|
699
|
+
}
|
|
700
|
+
return Effect.flatMap(required(spec), taskOf(spec))
|
|
701
|
+
},
|
|
702
|
+
deleteIndex: (index) =>
|
|
703
|
+
optionalTask({ method: 'DELETE', path: indexPath(index) }),
|
|
704
|
+
readSettings: (index) => {
|
|
705
|
+
const spec: AlgoliaCall = {
|
|
706
|
+
method: 'GET',
|
|
707
|
+
path: indexPath(index, '/settings'),
|
|
708
|
+
}
|
|
709
|
+
return Effect.flatMap(call(spec), (answer) =>
|
|
710
|
+
Option.match(answer, {
|
|
711
|
+
onNone: () => Effect.succeed(Option.none()),
|
|
712
|
+
onSome: (json) =>
|
|
713
|
+
Effect.map(decode(SettingsAnswer, spec)(json), Option.some),
|
|
714
|
+
}),
|
|
715
|
+
)
|
|
716
|
+
},
|
|
717
|
+
writeSettings: (index, settings) => {
|
|
718
|
+
const spec: AlgoliaCall = {
|
|
719
|
+
method: 'PUT',
|
|
720
|
+
path: indexPath(index, '/settings'),
|
|
721
|
+
body: settings,
|
|
722
|
+
}
|
|
723
|
+
return Effect.flatMap(required(spec), taskOf(spec))
|
|
724
|
+
},
|
|
725
|
+
listIndexNames: (page, hitsPerPage) => {
|
|
726
|
+
const spec: AlgoliaCall = {
|
|
727
|
+
method: 'GET',
|
|
728
|
+
path: `/1/indexes?page=${page}&hitsPerPage=${hitsPerPage}`,
|
|
729
|
+
}
|
|
730
|
+
// An answer without `nbPages` is unpaginated, so it is the last page; so
|
|
731
|
+
// is an empty one, which keeps a miscounted `nbPages` from looping.
|
|
732
|
+
return Effect.map(
|
|
733
|
+
Effect.flatMap(required(spec), decode(IndexListAnswer, spec)),
|
|
734
|
+
(answer): IndexListPage => ({
|
|
735
|
+
names: answer.items.map((item) => item.name),
|
|
736
|
+
more:
|
|
737
|
+
answer.items.length > 0 &&
|
|
738
|
+
answer.nbPages !== undefined &&
|
|
739
|
+
page + 1 < answer.nbPages,
|
|
740
|
+
}),
|
|
741
|
+
)
|
|
742
|
+
},
|
|
743
|
+
browseObjectIds: (index, cursor, hitsPerPage) => {
|
|
744
|
+
const spec: AlgoliaCall = {
|
|
745
|
+
method: 'POST',
|
|
746
|
+
path: indexPath(index, '/browse'),
|
|
747
|
+
read: true,
|
|
748
|
+
body: {
|
|
749
|
+
attributesToRetrieve: ['objectID'],
|
|
750
|
+
hitsPerPage,
|
|
751
|
+
...(Option.isSome(cursor) ? { cursor: cursor.value } : {}),
|
|
752
|
+
},
|
|
753
|
+
}
|
|
754
|
+
return Effect.flatMap(call(spec), (answer) =>
|
|
755
|
+
Option.match(answer, {
|
|
756
|
+
onNone: () =>
|
|
757
|
+
Effect.succeed<BrowsePage>({
|
|
758
|
+
objectIDs: [],
|
|
759
|
+
cursor: Option.none(),
|
|
760
|
+
}),
|
|
761
|
+
onSome: (json) =>
|
|
762
|
+
Effect.map(
|
|
763
|
+
decode(BrowseAnswer, spec)(json),
|
|
764
|
+
(page): BrowsePage => ({
|
|
765
|
+
objectIDs: page.hits.map((hit) => hit.objectID),
|
|
766
|
+
cursor: Option.fromUndefinedOr(page.cursor),
|
|
767
|
+
}),
|
|
768
|
+
),
|
|
769
|
+
}),
|
|
770
|
+
)
|
|
771
|
+
},
|
|
772
|
+
} satisfies AlgoliaRest
|
|
773
|
+
})
|