@kairos-es/feed-algolia 0.0.0-stage → 0.0.0

package/src/classify.ts

@@ -0,0 +1,201 @@
+/**
+ * How an Algolia answer becomes one of the feed's three sink failures.
+ *
+ * The driver's whole behaviour on failure follows from which of the three a sink
+ * names (`@kairos-es/feed`'s `errors.ts` owns that), so this mapping is the most
+ * consequential thing in the package and it is kept pure, in one place, and tested
+ * on its own. The rules:
+ *
+ * - **A 429 is `SinkRetryable`**, carrying `Retry-After` when Algolia sends one,
+ *   so the driver backs off no less than Algolia asked — UNLESS its message names
+ *   a QUOTA rather than a rate: Algolia answers 429 for an application over its
+ *   record quota or its indexing file-size allowance too, and neither lifts by
+ *   waiting, so retrying would only hold the feed until supervision reported it
+ *   stalled, with Algolia's reason lost. Such a 429 is `SinkHalted`, carrying
+ *   Algolia's message. The wordings recognised are {@link QUOTA_429}; a 429 whose
+ *   wording is not among them stays retryable, which is the safe side to err on.
+ * - **Every 5xx is `SinkRetryable`, and so is a 1xx or 3xx**, a status no call
+ *   here expects and one the transport has already tried on every host, as the
+ *   official client does. A transport failure — no answer at all — is retryable
+ *   too: the request may never have arrived, and resending is safe because every
+ *   write this sink makes is a full-record replace or a delete.
+ * - **A 400 that names an `objectID` in the batch is `SinkRejected` for that key.**
+ *   Algolia refuses a whole batch for one bad record, and names the record in its
+ *   message (`… objectID=<id> …`). Parking just that key lets the driver send the
+ *   rest again without it, which is the difference between one record missing from
+ *   search and the whole feed stopping. The key has to be one the batch carried: an
+ *   id the batch did not send is not a record the driver can park, so such an
+ *   answer falls through to the next rule. The message does not delimit the id, and
+ *   a feed key may hold a space, a comma or anything else, so the id is not parsed
+ *   out of the message: the batch's own keys are matched against what follows
+ *   `objectID=` (`objectIdNamedIn`).
+ * - **Every other 4xx is `SinkHalted`.** Bad credentials, a key without the right
+ *   ACL, an index name Algolia will not accept, a request this package built wrong:
+ *   none of them is mended by sending again, so the worker stops and supervision
+ *   reports it.
+ *
+ * Nothing here reads the request, and nothing here sees a header but `Retry-After`,
+ * so no classification can carry the admin key into a reason string.
+ */
+
+import type { FeedKey } from '@kairos-es/feed'
+import { SinkHalted, SinkRejected, SinkRetryable } from '@kairos-es/feed'
+import { DateTime, Duration, Option } from 'effect'
+
+/** Every failure an Algolia call can end in, already in the feed's vocabulary. */
+export type AlgoliaFailure = SinkRetryable | SinkRejected | SinkHalted
+
+/** An Algolia answer that was not a success, reduced to what classifying needs. */
+export interface AlgoliaErrorAnswer {
+  readonly status: number
+  /** Algolia's `message`, or the raw body when it sent none. */
+  readonly message: string
+  /** The `Retry-After` header, verbatim, when there was one. */
+  readonly retryAfter: Option.Option<string>
+}
+
+/** What the call was, for the reason string: never its headers. */
+export interface AlgoliaCallLabel {
+  readonly method: string
+  readonly path: string
+}
+
+const OBJECT_ID_MARKER = 'objectID='
+
+/** What may follow an id in Algolia's message: the end, a space or punctuation. */
+const ID_ENDS = /^(?:$|[\s,;.:)])/
+
+/**
+ * The key of `batchKeys` an Algolia error message names, when it names one.
+ *
+ * Algolia's record-level refusals all carry `objectID=<id>` (the record-too-big
+ * message is the common one), with nothing marking where the id ends. So each
+ * batch key is tried against the text after each `objectID=`, and it matches when
+ * that text starts with the key and the key is followed by the end of the message,
+ * a space or punctuation; the longest key that matches wins, so of `big` and
+ * `big one` the message `objectID=big one is too big` names the second.
+ */
+export const objectIdNamedIn = (
+  message: string,
+  batchKeys: ReadonlySet<FeedKey>,
+): Option.Option<FeedKey> => {
+  let named: FeedKey | undefined
+  let from = message.indexOf(OBJECT_ID_MARKER)
+  while (from >= 0) {
+    const after = message.slice(from + OBJECT_ID_MARKER.length)
+    for (const key of batchKeys) {
+      if (
+        after.startsWith(key) &&
+        ID_ENDS.test(after.slice(key.length)) &&
+        (named === undefined || key.length > named.length)
+      ) {
+        named = key
+      }
+    }
+    from = message.indexOf(OBJECT_ID_MARKER, from + 1)
+  }
+  return Option.fromUndefinedOr(named)
+}
+
+/**
+ * The 429 wordings that name a quota, not a rate, matched case-insensitively
+ * anywhere in Algolia's message: an application over its record quota, or over
+ * its allowance of indexing by file size. See the module doc for why each halts.
+ */
+export const QUOTA_429: ReadonlyArray<RegExp> = [/quota/i, /file size/i]
+
+/**
+ * `Retry-After` as a duration, given the current time.
+ *
+ * The header is either a whole number of seconds or an HTTP date. A date is turned
+ * into a wait against `now`, which the caller reads from `Clock` so a test's
+ * `TestClock` governs it; a date already past is a wait of zero. Anything else is
+ * ignored rather than guessed at, and the driver's own back-off applies.
+ */
+export const parseRetryAfter = (
+  header: string,
+  nowMillis: number,
+): Option.Option<Duration.Duration> => {
+  const trimmed = header.trim()
+  if (/^\d+$/.test(trimmed)) {
+    return Option.some(Duration.seconds(Number(trimmed)))
+  }
+  // A bare number with a fraction or a sign is neither form; do not let the date
+  // parser read it as a year.
+  if (/^[-+\d.]+$/.test(trimmed)) {
+    return Option.none()
+  }
+  return Option.map(DateTime.make(trimmed), (at) =>
+    Duration.millis(Math.max(0, DateTime.toEpochMillis(at) - nowMillis)),
+  )
+}
+
+/**
+ * Classify an Algolia error answer; see the module doc for the rules.
+ *
+ * `batchKeys` is the set of keys the request carried, empty for a call that sends
+ * no records, so only a write can ever produce a `SinkRejected`.
+ */
+export const classifyAnswer = (
+  call: AlgoliaCallLabel,
+  answer: AlgoliaErrorAnswer,
+  batchKeys: ReadonlySet<FeedKey>,
+  nowMillis: number,
+): AlgoliaFailure => {
+  const reason = `Algolia answered ${answer.status} to ${call.method} ${call.path}: ${answer.message}`
+  if (
+    answer.status === 429 &&
+    QUOTA_429.some((wording) => wording.test(answer.message))
+  ) {
+    return new SinkHalted({ reason })
+  }
+  if (
+    answer.status === 429 ||
+    answer.status >= 500 ||
+    answer.status < 200 ||
+    (answer.status >= 300 && answer.status < 400)
+  ) {
+    const retryAfter = Option.flatMap(answer.retryAfter, (header) =>
+      parseRetryAfter(header, nowMillis),
+    )
+    return new SinkRetryable({
+      reason,
+      ...(Option.isSome(retryAfter) ? { retryAfter: retryAfter.value } : {}),
+    })
+  }
+  if (answer.status === 400) {
+    const named = objectIdNamedIn(answer.message, batchKeys)
+    if (Option.isSome(named)) {
+      const key = named.value
+      return new SinkRejected({
+        keys: [key],
+        reasons: new Map([[key, answer.message]]),
+      })
+    }
+  }
+  return new SinkHalted({ reason })
+}
+
+/**
+ * A request that got no answer at all. Retryable: see the module doc for why a
+ * resend is safe.
+ */
+export const transportFailure = (
+  call: AlgoliaCallLabel,
+  detail: string,
+): AlgoliaFailure =>
+  new SinkRetryable({
+    reason: `no answer from Algolia to ${call.method} ${call.path}: ${detail}`,
+  })
+
+/**
+ * An answer this package cannot read: a success whose body is not the shape
+ * Algolia documents. Halted, because sending again will get the same body back.
+ */
+export const unreadableAnswer = (
+  call: AlgoliaCallLabel,
+  detail: string,
+): AlgoliaFailure =>
+  new SinkHalted({
+    reason: `unreadable answer from Algolia to ${call.method} ${call.path}: ${detail}`,
+  })

package/src/identifiers.ts

@@ -0,0 +1,45 @@
+/**
+ * The Algolia sink's branded identifiers.
+ *
+ * Three values cross this package's boundary that would otherwise be bare
+ * strings and numbers, and each is easy to pass in the place of another: an
+ * application id where an index name belongs, a feed key where an index name
+ * belongs, a task id from one index polled against another. Branding them is the
+ * repository's standing rule for a domain identifier, and here it is also what
+ * keeps a `Receipt` — an index and a task id — from being assembled by hand out of
+ * values that never belonged together.
+ */
+import { Schema } from 'effect'
+
+/**
+ * The Algolia application id: the first half of the credentials, and the name of
+ * the application's hosts. Not a secret — it is sent beside every search a browser
+ * makes — so it is a plain branded string, unlike the admin key.
+ */
+export const AlgoliaAppId = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('AlgoliaAppId'),
+)
+export type AlgoliaAppId = typeof AlgoliaAppId.Type
+
+/**
+ * The name of an Algolia index. The sink's SERVED index is one; a resync builds into
+ * another, named after the served one, the generation it rebuilds and the attempt.
+ */
+export const AlgoliaIndexName = Schema.String.pipe(
+  Schema.check(Schema.isNonEmpty()),
+  Schema.brand('AlgoliaIndexName'),
+)
+export type AlgoliaIndexName = typeof AlgoliaIndexName.Type
+
+/**
+ * An Algolia task id: what every write returns, and what `getTask` is polled with
+ * until the write is published. A task belongs to the index it was issued on, and
+ * is polled there and nowhere else — which is why a receipt carries the index
+ * beside it.
+ */
+export const AlgoliaTaskId = Schema.Number.pipe(
+  Schema.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)),
+  Schema.brand('AlgoliaTaskId'),
+)
+export type AlgoliaTaskId = typeof AlgoliaTaskId.Type

package/src/index.ts

@@ -0,0 +1,48 @@
+/**
+ * @kairos-es/feed-algolia — a `FeedSink` onto an Algolia index, with both of the
+ * port's optional parts: a `FeedResync` that rebuilds into a fresh index and moves
+ * it over the served one, and a `FeedReconcile` that lists the served index's ids.
+ *
+ * It talks to Algolia's REST API through `effect`'s own `HttpClient`, so it peers
+ * `effect` and `@kairos-es/feed` and nothing else, and the application chooses the
+ * HTTP transport by providing the client — `FetchHttpClient.layer` from
+ * `effect/http`, say.
+ *
+ * Where each part's argument lives:
+ *
+ * - `makeAlgoliaSink` (`sink.ts`): the steady state, resync and reconcile, and how
+ *   `confirm` waits.
+ * - Encoding and the size pre-check (`records.ts`): why only full-record writes,
+ *   and why oversized records are refused before they are sent.
+ * - Error classification (`classify.ts`): which Algolia answers are retryable,
+ *   which refuse one record, and which stop the worker.
+ * - The REST calls (`rest.ts`): why no SDK, the host failover kept from it, and
+ *   how the admin key is kept out of anything printed.
+ *
+ * Wiring:
+ *
+ * ```ts
+ * const sink = yield* makeAlgoliaSink({
+ *   name: SinkName.make('search'),
+ *   appId: AlgoliaAppId.make(config.appId),
+ *   adminKey: config.adminKey, // a Redacted<string>, from the application's config
+ *   index: AlgoliaIndexName.make('courses'),
+ *   doc: CourseDoc,
+ * }).pipe(Effect.provide(FetchHttpClient.layer))
+ * yield* Effect.forkScoped(runFeedSync({ source, sink, store }))
+ * ```
+ */
+
+export {
+  AlgoliaAppId,
+  AlgoliaIndexName,
+  AlgoliaTaskId,
+} from './identifiers'
+export { DEFAULT_MAX_RECORD_BYTES } from './records'
+export type { AlgoliaConnection, AlgoliaTimeouts } from './rest'
+export {
+  type AlgoliaReceipt,
+  type AlgoliaSinkOptions,
+  type AlgoliaTaskPolling,
+  makeAlgoliaSink,
+} from './sink'

package/src/records.ts

@@ -0,0 +1,109 @@
+/**
+ * Turning a batch of feed changes into Algolia batch operations, and refusing the
+ * records Algolia would refuse BEFORE they are sent.
+ *
+ * ## Full-record replace, and nothing partial
+ *
+ * An upsert is `addObject` with the record's `objectID` set to the feed key, which
+ * replaces whatever Algolia held under that id; a delete is `deleteObject`. Both
+ * leave the index the same however many times they are sent, which is the
+ * idempotence the driver relies on when it resends a batch it could not confirm.
+ * Algolia's partial operations (`partialUpdateObject`, the built-in `Increment`
+ * and its kin) do not: applied twice they apply twice. They are never used.
+ *
+ * The key is written LAST, over the encoded document, so a document that carries an
+ * `objectID` field of its own cannot point a record at another key.
+ *
+ * ## Why oversized records are refused here
+ *
+ * Algolia refuses a whole `batch` when one record in it is over the plan's record
+ * size limit. Its answer does name the record, and `classify.ts` turns that into a
+ * rejection of that one key — but only after a round trip that wrote nothing, and
+ * only for the first oversized record the batch held. Checking every record's size
+ * up front rejects all of them at once, so the driver's resend of the rest goes
+ * through first time. The 400 path stays as the backstop for a limit configured
+ * higher than the plan's.
+ *
+ * The size is the record's JSON in UTF-8 bytes, `objectID` included, which is how
+ * Algolia measures it. The default limit is the smallest Algolia plan's (10 KB);
+ * an application on a larger plan raises it.
+ *
+ * ## A document that does not encode
+ *
+ * The document schema is the read model's, and a document it cannot encode — or one
+ * that encodes to something other than a JSON object, which is all an Algolia
+ * record can be — is a defect in the read model, not a record the sink refuses. It
+ * dies, loudly, rather than being parked where nobody looks.
+ */
+import { type FeedChange, type FeedKey, SinkRejected } from '@kairos-es/feed'
+import { Array as Arr, Effect, Schema } from 'effect'
+import type { NonEmptyReadonlyArray } from 'effect/Array'
+import type { BatchOperation } from './rest'
+
+/** The smallest Algolia plan's record size limit, in bytes. */
+export const DEFAULT_MAX_RECORD_BYTES = 10_000
+
+const utf8 = new TextEncoder()
+
+/** A record's size as Algolia measures it: its JSON, in UTF-8 bytes. */
+export const recordBytes = (
+  record: Readonly<Record<string, unknown>>,
+): number => utf8.encode(JSON.stringify(record)).length
+
+/** A batch ready to send: its operations, and the keys it carries. */
+export interface EncodedBatch {
+  readonly operations: ReadonlyArray<BatchOperation>
+  readonly keys: ReadonlySet<FeedKey>
+}
+
+const isJsonObject = (u: unknown): u is Readonly<Record<string, unknown>> =>
+  typeof u === 'object' && u !== null && !Array.isArray(u)
+
+/**
+ * Encode a batch, failing `SinkRejected` with EVERY oversized key when there is
+ * one; see the module doc.
+ */
+export const encodeBatch = <D, I>(
+  doc: Schema.Codec<D, I>,
+  maxRecordBytes: number,
+  batch: NonEmptyReadonlyArray<FeedChange<D>>,
+): Effect.Effect<EncodedBatch, SinkRejected> =>
+  Effect.gen(function* () {
+    const encode = Schema.encodeEffect(doc)
+    const operations: Array<BatchOperation> = []
+    const keys = new Set<FeedKey>()
+    const reasons = new Map<FeedKey, string>()
+    for (const change of batch) {
+      keys.add(change.key)
+      if (change._tag === 'Delete') {
+        operations.push({
+          action: 'deleteObject',
+          body: { objectID: change.key },
+        })
+        continue
+      }
+      const encoded: unknown = yield* Effect.orDie(encode(change.doc))
+      if (!isJsonObject(encoded)) {
+        return yield* Effect.die(
+          new Error(
+            `feed-algolia: the document for key '${change.key}' encodes to ${encoded === null ? 'null' : typeof encoded}, and an Algolia record must be a JSON object`,
+          ),
+        )
+      }
+      const record = { ...encoded, objectID: change.key }
+      const size = recordBytes(record)
+      if (size > maxRecordBytes) {
+        reasons.set(
+          change.key,
+          `the record is ${size} bytes, over the configured maxRecordBytes of ${maxRecordBytes}`,
+        )
+        continue
+      }
+      operations.push({ action: 'addObject', body: record })
+    }
+    const rejected = Array.from(reasons.keys())
+    if (Arr.isReadonlyArrayNonEmpty(rejected)) {
+      return yield* Effect.fail(new SinkRejected({ keys: rejected, reasons }))
+    }
+    return { operations, keys }
+  })