@kairos-es/feed-algolia 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/LICENSE +28 -0
  2. package/README.md +66 -2
  3. package/dist/cjs/classify.js +157 -0
  4. package/dist/cjs/classify.js.map +1 -0
  5. package/dist/cjs/identifiers.js +38 -0
  6. package/dist/cjs/identifiers.js.map +1 -0
  7. package/dist/cjs/index.js +39 -0
  8. package/dist/cjs/index.js.map +1 -0
  9. package/dist/cjs/records.js +105 -0
  10. package/dist/cjs/records.js.map +1 -0
  11. package/dist/cjs/rest.js +293 -0
  12. package/dist/cjs/rest.js.map +1 -0
  13. package/dist/cjs/sink.js +463 -0
  14. package/dist/cjs/sink.js.map +1 -0
  15. package/dist/dts-cjs/classify.d.ts +101 -0
  16. package/dist/dts-cjs/classify.d.ts.map +1 -0
  17. package/dist/dts-cjs/identifiers.d.ts +34 -0
  18. package/dist/dts-cjs/identifiers.d.ts.map +1 -0
  19. package/dist/dts-cjs/index.d.ts +39 -0
  20. package/dist/dts-cjs/index.d.ts.map +1 -0
  21. package/dist/dts-cjs/package.json +3 -0
  22. package/dist/dts-cjs/records.d.ts +56 -0
  23. package/dist/dts-cjs/records.d.ts.map +1 -0
  24. package/dist/dts-cjs/rest.d.ts +215 -0
  25. package/dist/dts-cjs/rest.d.ts.map +1 -0
  26. package/dist/dts-cjs/sink.d.ts +220 -0
  27. package/dist/dts-cjs/sink.d.ts.map +1 -0
  28. package/dist/dts-esm/classify.d.ts +101 -0
  29. package/dist/dts-esm/classify.d.ts.map +1 -0
  30. package/dist/dts-esm/identifiers.d.ts +34 -0
  31. package/dist/dts-esm/identifiers.d.ts.map +1 -0
  32. package/dist/dts-esm/index.d.ts +39 -0
  33. package/dist/dts-esm/index.d.ts.map +1 -0
  34. package/dist/dts-esm/package.json +3 -0
  35. package/dist/dts-esm/records.d.ts +56 -0
  36. package/dist/dts-esm/records.d.ts.map +1 -0
  37. package/dist/dts-esm/rest.d.ts +215 -0
  38. package/dist/dts-esm/rest.d.ts.map +1 -0
  39. package/dist/dts-esm/sink.d.ts +220 -0
  40. package/dist/dts-esm/sink.d.ts.map +1 -0
  41. package/dist/esm/classify.js +145 -0
  42. package/dist/esm/classify.js.map +1 -0
  43. package/dist/esm/identifiers.js +31 -0
  44. package/dist/esm/identifiers.js.map +1 -0
  45. package/dist/esm/index.js +38 -0
  46. package/dist/esm/index.js.map +1 -0
  47. package/dist/esm/package.json +4 -0
  48. package/dist/esm/records.js +96 -0
  49. package/dist/esm/records.js.map +1 -0
  50. package/dist/esm/rest.js +285 -0
  51. package/dist/esm/rest.js.map +1 -0
  52. package/dist/esm/sink.js +454 -0
  53. package/dist/esm/sink.js.map +1 -0
  54. package/package.json +33 -3
  55. package/src/classify.ts +201 -0
  56. package/src/identifiers.ts +45 -0
  57. package/src/index.ts +48 -0
  58. package/src/records.ts +109 -0
  59. package/src/rest.ts +773 -0
  60. package/src/sink.ts +733 -0
package/src/rest.ts ADDED
@@ -0,0 +1,773 @@
1
+ /**
2
+ * The slice of Algolia's REST API this sink speaks, over `effect`'s own
3
+ * `HttpClient`.
4
+ *
5
+ * ## Why no SDK
6
+ *
7
+ * The surface the sink needs is eight calls — `batch`, `getTask`, `operationIndex`
8
+ * (copy and move), `deleteIndex`, `browse`, `listIndices`, and a settings read and
9
+ * write, which is how a resync target is made to exist and marked as its attempt's
10
+ * own — and each is one JSON request. Peering `algoliasearch` would bring a large
11
+ * non-Effect dependency for that, and its retries would sit underneath the feed
12
+ * driver's own supervision, which already owns retrying. Over `HttpClient` the
13
+ * sink is Effect-native, adds no peer beyond `effect`, and is faked in tests by a
14
+ * stub client with no network at all.
15
+ *
16
+ * ## Host failover
17
+ *
18
+ * The one piece of the SDK's transport kept here is the part the driver cannot do
19
+ * for itself: trying ANOTHER HOST. Algolia serves an application from
20
+ * `<appId>.algolia.net` and documents three fallbacks on separate infrastructure,
21
+ * `<appId>-1.algolianet.com` to `<appId>-3.algolianet.com`, to be tried when the
22
+ * first does not answer. A request that gets no answer (a transport failure, or no
23
+ * answer within its timeout), a 5xx, or a 1xx or 3xx — a status no call here
24
+ * expects, on which the official JavaScript client tries the next host too — is
25
+ * sent again at once to the next host, and only when every host it tried has
26
+ * failed is the call `SinkRetryable`, carrying the last host's failure. Without
27
+ * it, one unreachable host would hold the feed in the driver's back-off for as
28
+ * long as the host stayed down, with three healthy hosts unasked.
29
+ *
30
+ * - Hosts are STATEFUL, one state per client and so per sink, in the shape the
31
+ * official JavaScript client gives them. A host that failed in one of those ways
32
+ * is marked DOWN for two minutes, by `Clock`, and skipped by every call while it
33
+ * is down; a host that answers is marked up again. Without that memory a primary
34
+ * that drops packets would cost a whole timeout on EVERY call, each `getTask`
35
+ * poll included, so a single wait for a task could spend its whole
36
+ * `taskPolling.timeout` on the dead host. When every host is down, every host is
37
+ * tried again, in order, rather than none.
38
+ * - The fallbacks are SHUFFLED, once, when the client is built, from Effect's
39
+ * `Random`, so many workers do not all fail over onto the same host; the primary
40
+ * goes first whenever it is up.
41
+ * - Each hand-off to the next host is logged at DEBUG level, with the call, the
42
+ * host that failed, the host tried next and why. A call that lands after a
43
+ * failover looks like any other, so that line is the only trace one leaves.
44
+ * Hosts carry the application id, which is not a secret; the key travels in a
45
+ * header and reaches no log.
46
+ * - Three timeouts, as the official client has: `connect`, `read` and `write`,
47
+ * defaulting to its 2, 5 and 30 seconds. A read is a `GET` or a `browse`, and
48
+ * every other call is a write. `HttpClient` does not show this module the moment
49
+ * a connection is made, so one request's budget on one host is `connect` plus
50
+ * `read`, or `connect` plus `write`, over the whole exchange. A transport that
51
+ * enforces a connect timeout of its own fails sooner, and that failure fails
52
+ * over the same way.
53
+ * - Every call uses the same list, reads included. The SDK sends reads to
54
+ * `<appId>-dsn.algolia.net` first, which is the search-serving network; every
55
+ * read here is an admin read (a task's status, a listing, a settings read, a
56
+ * browse for reconcile) that wants the cluster the writes went to.
57
+ * - A 429 is NOT retried on another host: it is the application's rate limit or
58
+ * quota, not a host's health (`classify.ts` owns which). Nor is any other 4xx,
59
+ * which another host would answer the same.
60
+ * - A request can be sent more than once this way, and that is safe for the
61
+ * reason every resend here is (`classify.ts`): every write is a full-record
62
+ * replace, a delete, an index operation or a settings write, each of which
63
+ * leaves the application the same however often it lands.
64
+ * - The worst case for one call is therefore its budget times the number of hosts
65
+ * that are up when it starts.
66
+ * - An explicit `host` turns failover OFF: the caller has named the one place to
67
+ * talk to (a proxy, a test double), and there are no fallbacks of it to guess.
68
+ * That one host is never skipped, since every host being down tries it again.
69
+ *
70
+ * ## The admin key
71
+ *
72
+ * It is held as `Redacted` and unwrapped only into the request header, at the
73
+ * moment the request is built. Two things keep it out of anything printed: no
74
+ * failure this module produces carries the request or its headers — only the
75
+ * method, the path and Algolia's own message (`classify.ts`) — and every request
76
+ * runs with Algolia's key header added to the names `effect`'s `Headers` redact,
77
+ * so a request that is logged or traced by something further down shows the
78
+ * header as redacted.
79
+ *
80
+ * ## Not found
81
+ *
82
+ * Three of the calls meet a missing index as an ordinary outcome rather than a
83
+ * failure — dropping a leftover resync index that is not there, browsing an index
84
+ * that does not exist yet, reading the settings of an index that does not exist —
85
+ * so they answer a 404 as `Option.none()` instead of a `SinkHalted`. Every other
86
+ * call halts on a 404, with Algolia's own message in the reason. An index
87
+ * operation is one of those: Algolia documents a copy of a missing source as
88
+ * creating an empty destination, and a move of one as ignored without an error,
89
+ * so neither is a 404 (the sink's `publish` owns what that costs). A 404 is never
90
+ * failed over: another host would answer the same.
91
+ */
92
+ import type { FeedKey } from '@kairos-es/feed'
93
+ import {
94
+ Array as Arr,
95
+ Clock,
96
+ Duration,
97
+ Effect,
98
+ Option,
99
+ Random,
100
+ Redacted,
101
+ Ref,
102
+ Schema,
103
+ } from 'effect'
104
+ import type { NonEmptyReadonlyArray } from 'effect/Array'
105
+ import {
106
+ Headers,
107
+ HttpBody,
108
+ HttpClient,
109
+ type HttpClientError,
110
+ HttpClientRequest,
111
+ type HttpMethod,
112
+ } from 'effect/http'
113
+ import {
114
+ type AlgoliaCallLabel,
115
+ type AlgoliaFailure,
116
+ classifyAnswer,
117
+ transportFailure,
118
+ unreadableAnswer,
119
+ } from './classify'
120
+ import {
121
+ type AlgoliaAppId,
122
+ type AlgoliaIndexName,
123
+ AlgoliaTaskId,
124
+ } from './identifiers'
125
+
126
+ /** Where and as whom the sink talks to Algolia. */
127
+ export interface AlgoliaConnection {
128
+ readonly appId: AlgoliaAppId
129
+ /** The admin API key. Never logged; see the module doc. */
130
+ readonly adminKey: Redacted.Redacted<string>
131
+ /**
132
+ * ONE base URL to send every call to, with no failover. Leave it unset to talk
133
+ * to Algolia itself: `https://<appId>.algolia.net`, failing over to the three
134
+ * documented fallback hosts (see the module doc). Set it for a proxy or a test
135
+ * double, whose fallbacks there is no way to know.
136
+ */
137
+ readonly host?: string
138
+ /**
139
+ * How long one request to one host may take before it counts as unanswered,
140
+ * that host is marked down, and the next is tried; see the module doc.
141
+ */
142
+ readonly timeouts?: AlgoliaTimeouts
143
+ }
144
+
145
+ /**
146
+ * The three per-host timeouts, with the official JavaScript client's defaults. A
147
+ * request's budget on one host is `connect` plus `read` for a read, or `connect`
148
+ * plus `write` for a write; see the module doc for why they are added.
149
+ */
150
+ export interface AlgoliaTimeouts {
151
+ /** Default 2 seconds. */
152
+ readonly connect?: Duration.Input
153
+ /** For a `GET` or a `browse`. Default 5 seconds. */
154
+ readonly read?: Duration.Input
155
+ /** For every other call. Default 30 seconds. */
156
+ readonly write?: Duration.Input
157
+ }
158
+
159
+ /** The header Algolia reads the admin key from, lower-cased as `Headers` keeps it. */
160
+ const KEY_HEADER = 'x-algolia-api-key'
161
+ const APP_HEADER = 'x-algolia-application-id'
162
+
163
+ const DEFAULT_CONNECT_TIMEOUT = Duration.seconds(2)
164
+ const DEFAULT_READ_TIMEOUT = Duration.seconds(5)
165
+ const DEFAULT_WRITE_TIMEOUT = Duration.seconds(30)
166
+
167
+ /** How long a host that failed is skipped for: the official client's two minutes. */
168
+ const HOST_DOWN_MILLIS = Duration.toMillis(Duration.minutes(2))
169
+
170
+ /** One batch operation, as Algolia's `batch` takes it. Full-record writes only. */
171
+ export type BatchOperation =
172
+ | {
173
+ readonly action: 'addObject'
174
+ readonly body: Readonly<Record<string, unknown>>
175
+ }
176
+ | {
177
+ readonly action: 'deleteObject'
178
+ readonly body: { readonly objectID: string }
179
+ }
180
+
181
+ /** What an index-level `operationIndex` call asks for. */
182
+ export interface IndexOperation {
183
+ readonly operation: 'copy' | 'move'
184
+ readonly destination: AlgoliaIndexName
185
+ readonly scope?: ReadonlyArray<'settings' | 'synonyms' | 'rules'>
186
+ }
187
+
188
+ /** One page of a `browse`: the ids on it, and the cursor to the next, if any. */
189
+ export interface BrowsePage {
190
+ readonly objectIDs: ReadonlyArray<string>
191
+ readonly cursor: Option.Option<string>
192
+ }
193
+
194
+ /** One page of `listIndices`: the index names on it, and whether more follow. */
195
+ export interface IndexListPage {
196
+ readonly names: ReadonlyArray<string>
197
+ readonly more: boolean
198
+ }
199
+
200
+ /** One raw call: everything the typed calls below are built from. */
201
+ export interface AlgoliaCall {
202
+ readonly method: HttpMethod.HttpMethod
203
+ /** The path, already encoded: `/1/indexes/<name>/...`. */
204
+ readonly path: string
205
+ readonly body?: unknown
206
+ /** The keys the request carries, so a 400 naming one can be a rejection. */
207
+ readonly batchKeys?: ReadonlySet<FeedKey>
208
+ /**
209
+ * A read whose method is not `GET` (a `browse`), so it gets the read timeout.
210
+ * Every `GET` is a read already.
211
+ */
212
+ readonly read?: true
213
+ }
214
+
215
+ /**
216
+ * Where a task stands, as a caller that tolerates a missing index sees it: still
217
+ * running, published, or unanswerable because its index does not exist.
218
+ */
219
+ export type AlgoliaTaskState = 'pending' | 'published' | 'indexMissing'
220
+
221
+ /** The Algolia calls the sink makes. Every failure is already classified. */
222
+ export interface AlgoliaRest {
223
+ /** A raw call; `Option.none()` on a 404. Exposed for the package's own tests. */
224
+ readonly call: (
225
+ call: AlgoliaCall,
226
+ ) => Effect.Effect<Option.Option<unknown>, AlgoliaFailure>
227
+ readonly batch: (
228
+ index: AlgoliaIndexName,
229
+ operations: ReadonlyArray<BatchOperation>,
230
+ batchKeys: ReadonlySet<FeedKey>,
231
+ ) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
232
+ readonly taskPublished: (
233
+ index: AlgoliaIndexName,
234
+ taskID: AlgoliaTaskId,
235
+ ) => Effect.Effect<boolean, AlgoliaFailure>
236
+ /**
237
+ * `taskPublished`, as a state, with a 404 — Algolia's answer to `getTask` on an
238
+ * index that does not exist — as `'indexMissing'` rather than a halt, for a
239
+ * caller that has another way to learn what the task did.
240
+ */
241
+ readonly taskStateUnlessIndexMissing: (
242
+ index: AlgoliaIndexName,
243
+ taskID: AlgoliaTaskId,
244
+ ) => Effect.Effect<AlgoliaTaskState, AlgoliaFailure>
245
+ /**
246
+ * A copy or a move. Algolia answers a missing source with a task too: a copy
247
+ * creates an empty destination, and a move does nothing (see the module doc).
248
+ */
249
+ readonly operation: (
250
+ source: AlgoliaIndexName,
251
+ operation: IndexOperation,
252
+ ) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
253
+ /** `Option.none()` when the index does not exist. */
254
+ readonly deleteIndex: (
255
+ index: AlgoliaIndexName,
256
+ ) => Effect.Effect<Option.Option<AlgoliaTaskId>, AlgoliaFailure>
257
+ /** An index's settings; `Option.none()` when the index does not exist. */
258
+ readonly readSettings: (
259
+ index: AlgoliaIndexName,
260
+ ) => Effect.Effect<
261
+ Option.Option<Readonly<Record<string, unknown>>>,
262
+ AlgoliaFailure
263
+ >
264
+ /**
265
+ * A settings write of the settings named, and only those; a `null` resets one
266
+ * to its default. Algolia creates an index on a settings write, so this is also
267
+ * how an index is made to exist without a record.
268
+ */
269
+ readonly writeSettings: (
270
+ index: AlgoliaIndexName,
271
+ settings: Readonly<Record<string, unknown>>,
272
+ ) => Effect.Effect<AlgoliaTaskId, AlgoliaFailure>
273
+ /**
274
+ * One page of the application's index names, `page` counting from 0. Every
275
+ * index the key may see is listed, so a caller filters by name.
276
+ */
277
+ readonly listIndexNames: (
278
+ page: number,
279
+ hitsPerPage: number,
280
+ ) => Effect.Effect<IndexListPage, AlgoliaFailure>
281
+ /** A missing index browses as one empty last page. */
282
+ readonly browseObjectIds: (
283
+ index: AlgoliaIndexName,
284
+ cursor: Option.Option<string>,
285
+ hitsPerPage: number,
286
+ ) => Effect.Effect<BrowsePage, AlgoliaFailure>
287
+ }
288
+
289
+ /** The path of an index, or of something under it. */
290
+ export const indexPath = (index: AlgoliaIndexName, rest = ''): string =>
291
+ `/1/indexes/${encodeURIComponent(index)}${rest}`
292
+
293
+ const TaskAnswer = Schema.Struct({ taskID: AlgoliaTaskId })
294
+ const TaskStatus = Schema.Struct({
295
+ status: Schema.Literals(['published', 'notPublished']),
296
+ })
297
+ const IndexListAnswer = Schema.Struct({
298
+ items: Schema.Array(Schema.Struct({ name: Schema.String })),
299
+ nbPages: Schema.optionalKey(Schema.Number),
300
+ })
301
+ const SettingsAnswer = Schema.Record(Schema.String, Schema.Unknown)
302
+ const BrowseAnswer = Schema.Struct({
303
+ hits: Schema.Array(Schema.Struct({ objectID: Schema.String })),
304
+ cursor: Schema.optionalKey(Schema.String),
305
+ })
306
+
307
+ /**
308
+ * The most of a raw, non-JSON error body a reason carries. A proxy's or a load
309
+ * balancer's error page can be any size, and a reason is logged whole.
310
+ */
311
+ const RAW_BODY_LIMIT = 500
312
+
313
+ /**
314
+ * Algolia's error body is `{ message, status }`; anything else is shown raw, cut
315
+ * to {@link RAW_BODY_LIMIT} characters.
316
+ */
317
+ const errorMessage = (text: string): string => {
318
+ try {
319
+ const parsed: unknown = JSON.parse(text)
320
+ if (
321
+ typeof parsed === 'object' &&
322
+ parsed !== null &&
323
+ 'message' in parsed &&
324
+ typeof parsed.message === 'string'
325
+ ) {
326
+ return parsed.message
327
+ }
328
+ } catch {
329
+ // Not JSON: the raw body is the best message there is.
330
+ }
331
+ if (text.length === 0) {
332
+ return '(empty body)'
333
+ }
334
+ return text.length <= RAW_BODY_LIMIT
335
+ ? text
336
+ : `${text.slice(0, RAW_BODY_LIMIT)}… (${text.length - RAW_BODY_LIMIT} more characters)`
337
+ }
338
+
339
+ /**
340
+ * A status that says nothing about the request and something about the host: a
341
+ * 5xx, or a 1xx or 3xx, which no call here expects. Tried on the next host.
342
+ */
343
+ const hostFailedWith = (status: number): boolean =>
344
+ status < 200 || (status >= 300 && status < 400) || status >= 500
345
+
346
+ const label = (call: AlgoliaCall): AlgoliaCallLabel => ({
347
+ method: call.method,
348
+ path: call.path,
349
+ })
350
+
351
+ /** An answer as it came back from one host, before it is classified. */
352
+ interface RawAnswer {
353
+ readonly status: number
354
+ readonly text: string
355
+ readonly retryAfter: Option.Option<string>
356
+ }
357
+
358
+ /** A classified answer: its JSON, or the 404 it was, with the failure it makes. */
359
+ type Sent =
360
+ | { readonly found: true; readonly json: unknown }
361
+ | { readonly found: false; readonly failure: AlgoliaFailure }
362
+
363
+ /** One `getTask` poll: published or not, or a 404 and the failure it makes. */
364
+ type TaskPoll = boolean | { readonly indexMissing: AlgoliaFailure }
365
+
366
+ /** The fallback hosts' numbers: `<appId>-<n>.algolianet.com`. */
367
+ const FALLBACKS = [1, 2, 3] as const
368
+
369
+ /**
370
+ * The base URLs a call is tried on, in order; see the module doc. Shuffled from
371
+ * `Random`, so a test can seed it.
372
+ */
373
+ const hostsFor = (
374
+ connection: AlgoliaConnection,
375
+ ): Effect.Effect<NonEmptyReadonlyArray<string>> =>
376
+ connection.host === undefined
377
+ ? Effect.map(
378
+ Random.shuffle(
379
+ FALLBACKS.map(
380
+ (n) => `https://${connection.appId}-${n}.algolianet.com`,
381
+ ),
382
+ ),
383
+ (fallbacks) =>
384
+ [`https://${connection.appId}.algolia.net`, ...fallbacks] as const,
385
+ )
386
+ : Effect.succeed([connection.host.replace(/\/+$/, '')] as const)
387
+
388
+ /**
389
+ * Build the REST client over the `HttpClient` in context. The client is captured
390
+ * here, so every call the sink makes later needs nothing from its caller.
391
+ */
392
+ export const makeAlgoliaRest = (
393
+ connection: AlgoliaConnection,
394
+ ): Effect.Effect<AlgoliaRest, never, HttpClient.HttpClient> =>
395
+ Effect.gen(function* () {
396
+ const http = yield* HttpClient.HttpClient
397
+ const hosts = yield* hostsFor(connection)
398
+ const timeoutOf = (
399
+ input: Duration.Input | undefined,
400
+ fallback: Duration.Duration,
401
+ ) => (input === undefined ? fallback : Duration.fromInputUnsafe(input))
402
+ const connect = timeoutOf(
403
+ connection.timeouts?.connect,
404
+ DEFAULT_CONNECT_TIMEOUT,
405
+ )
406
+ const readBudget = Duration.sum(
407
+ connect,
408
+ timeoutOf(connection.timeouts?.read, DEFAULT_READ_TIMEOUT),
409
+ )
410
+ const writeBudget = Duration.sum(
411
+ connect,
412
+ timeoutOf(connection.timeouts?.write, DEFAULT_WRITE_TIMEOUT),
413
+ )
414
+ /** When each host that failed comes back up, in `Clock` millis. */
415
+ const downUntil = yield* Ref.make<ReadonlyMap<string, number>>(new Map())
416
+ const redactedNames = [
417
+ ...Headers.CurrentRedactedNames.defaultValue(),
418
+ KEY_HEADER,
419
+ ]
420
+
421
+ const fromClientError = (
422
+ call: AlgoliaCall,
423
+ error: HttpClientError.HttpClientError,
424
+ ): AlgoliaFailure =>
425
+ error.reason._tag === 'TransportError' ||
426
+ error.reason._tag === 'DecodeError' ||
427
+ error.reason._tag === 'EmptyBodyError'
428
+ ? transportFailure(label(call), error.reason._tag)
429
+ : unreadableAnswer(label(call), error.reason._tag)
430
+
431
+ /** One request to one host, unclassified. Fails only `SinkRetryable` or halted. */
432
+ const exchange = (
433
+ host: string,
434
+ spec: AlgoliaCall,
435
+ ): Effect.Effect<RawAnswer, AlgoliaFailure> => {
436
+ const budget =
437
+ spec.method === 'GET' || spec.read === true ? readBudget : writeBudget
438
+ const request = HttpClientRequest.make(spec.method)(
439
+ `${host}${spec.path}`,
440
+ {
441
+ headers: {
442
+ [APP_HEADER]: connection.appId,
443
+ [KEY_HEADER]: Redacted.value(connection.adminKey),
444
+ },
445
+ acceptJson: true,
446
+ ...(spec.body === undefined
447
+ ? {}
448
+ : { body: HttpBody.jsonUnsafe(spec.body) }),
449
+ },
450
+ )
451
+ return Effect.flatMap(http.execute(request), (response) =>
452
+ Effect.map(
453
+ response.text,
454
+ (text): RawAnswer => ({
455
+ status: response.status,
456
+ text,
457
+ retryAfter: Headers.get(response.headers, 'retry-after'),
458
+ }),
459
+ ),
460
+ ).pipe(
461
+ Effect.mapError((error) => fromClientError(spec, error)),
462
+ Effect.timeoutOrElse({
463
+ duration: budget,
464
+ orElse: () =>
465
+ Effect.fail(
466
+ transportFailure(
467
+ label(spec),
468
+ `no answer within ${Duration.format(budget)}`,
469
+ ),
470
+ ),
471
+ }),
472
+ )
473
+ }
474
+
475
+ const markDown = (host: string) =>
476
+ Effect.flatMap(Clock.currentTimeMillis, (now) =>
477
+ Ref.update(downUntil, (down) =>
478
+ new Map(down).set(host, now + HOST_DOWN_MILLIS),
479
+ ),
480
+ )
481
+ const markUp = (host: string) =>
482
+ Ref.update(downUntil, (down) => {
483
+ if (!down.has(host)) {
484
+ return down
485
+ }
486
+ const next = new Map(down)
487
+ next.delete(host)
488
+ return next
489
+ })
490
+
491
+ /**
492
+ * Say, at debug level, that a call is leaving `host` for `next` and why: the
493
+ * only trace a failover leaves, since the call that lands looks like any
494
+ * other. Hosts and the call's label only; the key is in a header, which
495
+ * nothing here reads.
496
+ */
497
+ const logFailover = (
498
+ spec: AlgoliaCall,
499
+ host: string,
500
+ next: string,
501
+ why: string,
502
+ ) =>
503
+ Effect.logDebug('feed-algolia: failing over to the next host', {
504
+ call: `${spec.method} ${spec.path}`,
505
+ failed: host,
506
+ next,
507
+ why,
508
+ })
509
+
510
+ /**
511
+ * The request on each host in turn until one answers with a status that is
512
+ * the request's own rather than the host's; see the module doc. The last
513
+ * host's outcome is returned whatever it is, so a call every host failed ends
514
+ * as the last one did.
515
+ */
516
+ const tryHosts = (
517
+ spec: AlgoliaCall,
518
+ [host, ...rest]: NonEmptyReadonlyArray<string>,
519
+ ): Effect.Effect<RawAnswer, AlgoliaFailure> =>
520
+ // `matchEffect`, so a failure of the NEXT host's attempt is not caught here
521
+ // again: each host is asked once per call.
522
+ Effect.matchEffect(exchange(host, spec), {
523
+ onFailure: (failure) =>
524
+ failure._tag !== 'SinkRetryable'
525
+ ? Effect.fail(failure)
526
+ : Effect.andThen(
527
+ markDown(host),
528
+ Arr.isReadonlyArrayNonEmpty(rest)
529
+ ? Effect.andThen(
530
+ logFailover(spec, host, rest[0], failure.reason),
531
+ tryHosts(spec, rest),
532
+ )
533
+ : Effect.fail(failure),
534
+ ),
535
+ onSuccess: (answer) =>
536
+ !hostFailedWith(answer.status)
537
+ ? Effect.as(markUp(host), answer)
538
+ : Effect.andThen(
539
+ markDown(host),
540
+ Arr.isReadonlyArrayNonEmpty(rest)
541
+ ? Effect.andThen(
542
+ logFailover(
543
+ spec,
544
+ host,
545
+ rest[0],
546
+ `answered ${answer.status}`,
547
+ ),
548
+ tryHosts(spec, rest),
549
+ )
550
+ : Effect.succeed(answer),
551
+ ),
552
+ })
553
+
554
+ /** The hosts that are up, in order; every host when none is. */
555
+ const acrossHosts = (
556
+ spec: AlgoliaCall,
557
+ ): Effect.Effect<RawAnswer, AlgoliaFailure> =>
558
+ Effect.gen(function* () {
559
+ const now = yield* Clock.currentTimeMillis
560
+ const down = yield* Ref.get(downUntil)
561
+ const up = hosts.filter((host) => (down.get(host) ?? 0) <= now)
562
+ return yield* tryHosts(
563
+ spec,
564
+ Arr.isReadonlyArrayNonEmpty(up) ? up : hosts,
565
+ )
566
+ })
567
+
568
+ /**
569
+ * One call, classified. A 404 is answered rather than failed, beside the
570
+ * failure it would be — Algolia's message kept — so each caller decides
571
+ * whether a missing index is an outcome or a halt.
572
+ */
573
+ const send = (spec: AlgoliaCall): Effect.Effect<Sent, AlgoliaFailure> =>
574
+ Effect.gen(function* () {
575
+ const answer = yield* acrossHosts(spec)
576
+ if (answer.status >= 200 && answer.status < 300) {
577
+ try {
578
+ return {
579
+ found: true,
580
+ json:
581
+ answer.text.length === 0
582
+ ? {}
583
+ : (JSON.parse(answer.text) as unknown),
584
+ } satisfies Sent
585
+ } catch {
586
+ return yield* Effect.fail(
587
+ unreadableAnswer(label(spec), 'the body is not JSON'),
588
+ )
589
+ }
590
+ }
591
+ const now = yield* Clock.currentTimeMillis
592
+ const failure = classifyAnswer(
593
+ label(spec),
594
+ {
595
+ status: answer.status,
596
+ message: errorMessage(answer.text),
597
+ retryAfter: answer.retryAfter,
598
+ },
599
+ spec.batchKeys ?? new Set(),
600
+ now,
601
+ )
602
+ return answer.status === 404
603
+ ? ({ found: false, failure } satisfies Sent)
604
+ : yield* Effect.fail(failure)
605
+ }).pipe(
606
+ Effect.provideService(Headers.CurrentRedactedNames, redactedNames),
607
+ )
608
+
609
+ const call = (spec: AlgoliaCall) =>
610
+ Effect.map(send(spec), (sent) =>
611
+ sent.found ? Option.some(sent.json) : Option.none(),
612
+ )
613
+
614
+ /** A call whose 404 is a failure like any other 4xx, Algolia's message kept. */
615
+ const required = (spec: AlgoliaCall) =>
616
+ Effect.flatMap(send(spec), (sent) =>
617
+ sent.found ? Effect.succeed(sent.json) : Effect.fail(sent.failure),
618
+ )
619
+
620
+ const decode =
621
+ <S extends Schema.Top & { readonly DecodingServices: never }>(
622
+ schema: S,
623
+ spec: AlgoliaCall,
624
+ ) =>
625
+ (json: unknown): Effect.Effect<S['Type'], AlgoliaFailure> =>
626
+ Option.match(Schema.decodeUnknownOption(schema)(json), {
627
+ onNone: () =>
628
+ Effect.fail(
629
+ unreadableAnswer(
630
+ label(spec),
631
+ 'the body is not the documented shape',
632
+ ),
633
+ ),
634
+ onSome: Effect.succeed,
635
+ })
636
+
637
+ /** One `getTask`; a 404 is answered beside its failure, as `send` does. */
638
+ const taskPoll = (index: AlgoliaIndexName, taskID: AlgoliaTaskId) => {
639
+ const spec: AlgoliaCall = {
640
+ method: 'GET',
641
+ path: indexPath(index, `/task/${taskID}`),
642
+ }
643
+ return Effect.flatMap(
644
+ send(spec),
645
+ (sent): Effect.Effect<TaskPoll, AlgoliaFailure> =>
646
+ sent.found
647
+ ? Effect.map(
648
+ decode(TaskStatus, spec)(sent.json),
649
+ (answer) => answer.status === 'published',
650
+ )
651
+ : Effect.succeed({ indexMissing: sent.failure }),
652
+ )
653
+ }
654
+
655
+ const taskOf = (spec: AlgoliaCall) => (json: unknown) =>
656
+ Effect.map(decode(TaskAnswer, spec)(json), (answer) => answer.taskID)
657
+
658
+ /** A call whose 404 means "no such index", answered with a task otherwise. */
659
+ const optionalTask = (spec: AlgoliaCall) =>
660
+ Effect.flatMap(call(spec), (answer) =>
661
+ Option.match(answer, {
662
+ onNone: () => Effect.succeed(Option.none<AlgoliaTaskId>()),
663
+ onSome: (json) => Effect.map(taskOf(spec)(json), Option.some),
664
+ }),
665
+ )
666
+
667
+ return {
668
+ call,
669
+ batch: (index, operations, batchKeys) => {
670
+ const spec: AlgoliaCall = {
671
+ method: 'POST',
672
+ path: indexPath(index, '/batch'),
673
+ body: { requests: operations },
674
+ batchKeys,
675
+ }
676
+ return Effect.flatMap(required(spec), taskOf(spec))
677
+ },
678
+ taskPublished: (index, taskID) =>
679
+ Effect.flatMap(taskPoll(index, taskID), (poll) =>
680
+ typeof poll === 'boolean'
681
+ ? Effect.succeed(poll)
682
+ : Effect.fail(poll.indexMissing),
683
+ ),
684
+ taskStateUnlessIndexMissing: (index, taskID) =>
685
+ Effect.map(
686
+ taskPoll(index, taskID),
687
+ (poll): AlgoliaTaskState =>
688
+ typeof poll !== 'boolean'
689
+ ? 'indexMissing'
690
+ : poll
691
+ ? 'published'
692
+ : 'pending',
693
+ ),
694
+ operation: (source, operation) => {
695
+ const spec: AlgoliaCall = {
696
+ method: 'POST',
697
+ path: indexPath(source, '/operation'),
698
+ body: operation,
699
+ }
700
+ return Effect.flatMap(required(spec), taskOf(spec))
701
+ },
702
+ deleteIndex: (index) =>
703
+ optionalTask({ method: 'DELETE', path: indexPath(index) }),
704
+ readSettings: (index) => {
705
+ const spec: AlgoliaCall = {
706
+ method: 'GET',
707
+ path: indexPath(index, '/settings'),
708
+ }
709
+ return Effect.flatMap(call(spec), (answer) =>
710
+ Option.match(answer, {
711
+ onNone: () => Effect.succeed(Option.none()),
712
+ onSome: (json) =>
713
+ Effect.map(decode(SettingsAnswer, spec)(json), Option.some),
714
+ }),
715
+ )
716
+ },
717
+ writeSettings: (index, settings) => {
718
+ const spec: AlgoliaCall = {
719
+ method: 'PUT',
720
+ path: indexPath(index, '/settings'),
721
+ body: settings,
722
+ }
723
+ return Effect.flatMap(required(spec), taskOf(spec))
724
+ },
725
+ listIndexNames: (page, hitsPerPage) => {
726
+ const spec: AlgoliaCall = {
727
+ method: 'GET',
728
+ path: `/1/indexes?page=${page}&hitsPerPage=${hitsPerPage}`,
729
+ }
730
+ // An answer without `nbPages` is unpaginated, so it is the last page; so
731
+ // is an empty one, which keeps a miscounted `nbPages` from looping.
732
+ return Effect.map(
733
+ Effect.flatMap(required(spec), decode(IndexListAnswer, spec)),
734
+ (answer): IndexListPage => ({
735
+ names: answer.items.map((item) => item.name),
736
+ more:
737
+ answer.items.length > 0 &&
738
+ answer.nbPages !== undefined &&
739
+ page + 1 < answer.nbPages,
740
+ }),
741
+ )
742
+ },
743
+ browseObjectIds: (index, cursor, hitsPerPage) => {
744
+ const spec: AlgoliaCall = {
745
+ method: 'POST',
746
+ path: indexPath(index, '/browse'),
747
+ read: true,
748
+ body: {
749
+ attributesToRetrieve: ['objectID'],
750
+ hitsPerPage,
751
+ ...(Option.isSome(cursor) ? { cursor: cursor.value } : {}),
752
+ },
753
+ }
754
+ return Effect.flatMap(call(spec), (answer) =>
755
+ Option.match(answer, {
756
+ onNone: () =>
757
+ Effect.succeed<BrowsePage>({
758
+ objectIDs: [],
759
+ cursor: Option.none(),
760
+ }),
761
+ onSome: (json) =>
762
+ Effect.map(
763
+ decode(BrowseAnswer, spec)(json),
764
+ (page): BrowsePage => ({
765
+ objectIDs: page.hits.map((hit) => hit.objectID),
766
+ cursor: Option.fromUndefinedOr(page.cursor),
767
+ }),
768
+ ),
769
+ }),
770
+ )
771
+ },
772
+ } satisfies AlgoliaRest
773
+ })