better-auth-elasticsearch 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,912 @@
1
+ import { createHash } from "node:crypto";
2
+
3
+ import type { Client, estypes } from "@elastic/elasticsearch";
4
+ import type { BetterAuthDBSchema } from "better-auth/db";
5
+
6
+ import {
7
+ isNotFound,
8
+ isVersionConflict,
9
+ isWriteBlocked,
10
+ statusCodeOf,
11
+ } from "./elasticsearch-client.js";
12
+ import type { IndexSettings } from "./index-layout.js";
13
+
14
+ /**
15
+ * How long a reservation whose row does not hold its value counts as a write
16
+ * still in flight. Older, it is left over from a crash and may be taken over,
17
+ * so a crash blocks a value for at most this long.
18
+ */
19
+ export const GRACE_MS = 120_000;
20
+
21
+ /** How often a claim of one value goes again after losing a race for it. */
22
+ const MAX_CLAIM_ATTEMPTS = 10;
23
+
24
+ /** How many rows a backfill reads and reserves per request. */
25
+ const BACKFILL_PAGE_SIZE = 1000;
26
+
27
+ // Re-stamps a reservation the claiming row already owns. One another row owns
28
+ // is left untouched, and the claimant decides what to do from its contents.
29
+ const CLAIM_SCRIPT = `
30
+ if (ctx._source.owner == params.owner) {
31
+ ctx._source.claimedAt = params.now;
32
+ } else {
33
+ ctx.op = 'noop';
34
+ }`;
35
+
36
+ // Deletes a reservation only while the deleted row still owns it.
37
+ const RELEASE_SCRIPT = `
38
+ if (ctx._source.owner == params.owner) {
39
+ ctx.op = 'delete';
40
+ } else {
41
+ ctx.op = 'noop';
42
+ }`;
43
+
44
+ type Source = Record<string, unknown>;
45
+
46
+ /**
47
+ * What a reservation document stores: the constraint's database field names
48
+ * joined with "+", the owning row, and when it was last claimed. Never the
49
+ * value itself.
50
+ */
51
+ interface Reservation {
52
+ field: string;
53
+ owner: string;
54
+ claimedAt: number;
55
+ }
56
+
57
+ /**
58
+ * A set of fields whose values together must be unique: a field declared
59
+ * `unique`, or a unique index of the table, which may span several fields.
60
+ */
61
+ export interface UniqueConstraint {
62
+ /** The schema field names in index order, joined with "+". */
63
+ name: string;
64
+ /** The fields' schema and database names, in index order. */
65
+ fields: { name: string; dbName: string }[];
66
+ }
67
+
68
+ /** The values a row holds in a constraint's fields, in its field order. */
69
+ export interface HeldValues {
70
+ constraint: UniqueConstraint;
71
+ values: unknown[];
72
+ }
73
+
74
+ /** The unique values one row's write takes and gives up. */
75
+ export interface RowChange {
76
+ id: string;
77
+ claim: HeldValues[];
78
+ drop?: HeldValues[];
79
+ }
80
+
81
+ /** Rows that already hold the same value of a unique field or index. */
82
+ export interface DuplicateValues {
83
+ /** The model's name in the better-auth schema, such as "user". */
84
+ model: string;
85
+ /**
86
+ * The field's name in the better-auth schema, such as "email", or for an
87
+ * index over several fields their names joined with "+".
88
+ */
89
+ field: string;
90
+ /** The ids of the rows holding the value. */
91
+ ids: string[];
92
+ }
93
+
94
+ /** The sequence number and primary term a conditional write needs. */
95
+ interface Version {
96
+ if_seq_no: number;
97
+ if_primary_term: number;
98
+ }
99
+
100
+ interface Entry extends HeldValues {
101
+ row: string;
102
+ /** The reservation document's `_id`. */
103
+ id: string;
104
+ /** Whether this write created the reservation rather than inheriting it. */
105
+ created: boolean;
106
+ /** The reservation as this write last left it. */
107
+ version?: Version;
108
+ }
109
+
110
+ /** A row's values of one constraint, as migrate() reserves them. */
111
+ interface BackfillEntry extends HeldValues {
112
+ row: string;
113
+ /** The row's stored values of every enforced field. */
114
+ source: Source | undefined;
115
+ /** The reservation document's `_id`. */
116
+ id: string;
117
+ }
118
+
119
+ const isValue = (value: unknown) => value !== null && value !== undefined;
120
+
121
+ const dbFieldsOf = (constraint: UniqueConstraint) =>
122
+ constraint.fields.map((field) => field.dbName);
123
+
124
+ /**
125
+ * The values a stored row holds in a constraint's fields, or undefined if
126
+ * any is null or missing: such a row holds no reservation.
127
+ */
128
+ const valuesIn = (
129
+ constraint: UniqueConstraint,
130
+ source: Source | undefined,
131
+ ): unknown[] | undefined => {
132
+ const values = dbFieldsOf(constraint).map((dbName) => source?.[dbName]);
133
+ return values.every(isValue) ? values : undefined;
134
+ };
135
+
136
+ /** Whether a stored row holds `held`, compared as stored. */
137
+ const holds = (source: Source | undefined, held: HeldValues) =>
138
+ JSON.stringify(valuesIn(held.constraint, source)) ===
139
+ JSON.stringify(held.values);
140
+
141
+ /**
142
+ * A reservation's `_id`: a hash, so the value itself is never stored. One
143
+ * field hashes as `[field, value]` however its uniqueness is declared;
144
+ * several hash as `[fields, values]`, which no single field can collide with.
145
+ */
146
+ export const reservationId = (
147
+ dbFields: readonly string[],
148
+ values: readonly unknown[],
149
+ ) =>
150
+ createHash("sha256")
151
+ .update(
152
+ JSON.stringify(
153
+ dbFields.length === 1 ? [dbFields[0], values[0]] : [dbFields, values],
154
+ ),
155
+ )
156
+ .digest("base64url");
157
+
158
+ const idOf = (held: HeldValues) =>
159
+ reservationId(dbFieldsOf(held.constraint), held.values);
160
+
161
+ /** The reservation document `held` claimed for `owner`. */
162
+ const reservationFor = (
163
+ held: HeldValues,
164
+ owner: string,
165
+ claimedAt: number,
166
+ ): Reservation => ({
167
+ field: dbFieldsOf(held.constraint).join("+"),
168
+ owner,
169
+ claimedAt,
170
+ });
171
+
172
+ const versionOf = (item: {
173
+ _seq_no?: number;
174
+ _primary_term?: number;
175
+ }): Version | undefined =>
176
+ item._seq_no === undefined || item._primary_term === undefined
177
+ ? undefined
178
+ : { if_seq_no: item._seq_no, if_primary_term: item._primary_term };
179
+
180
+ /** A write that failed in a way that leaves open whether it landed. */
181
+ const mayHaveLanded = (error: unknown) => {
182
+ const status = statusCodeOf(error);
183
+ return (
184
+ status === undefined || status === 408 || status === 429 || status >= 500
185
+ );
186
+ };
187
+
188
+ const isAlreadyExists = (error: unknown) =>
189
+ (error as { meta?: { body?: { error?: { type?: string } } } }).meta?.body
190
+ ?.error?.type === "resource_already_exists_exception";
191
+
192
+ export const duplicateError = (model: string, field: string) =>
193
+ new Error(
194
+ `duplicate key: a "${model}" row with this ${field} already exists`,
195
+ );
196
+
197
+ export interface ReservationsOptions {
198
+ client: Client;
199
+ schema: BetterAuthDBSchema;
200
+ /** The schema's name for a database model name. */
201
+ getDefaultModelName: (model: string) => string;
202
+ /**
203
+ * Constraints left unenforced, as `model.field`, or `model.fieldA+fieldB`
204
+ * for an index over several fields, in schema names and index order.
205
+ */
206
+ skipUniqueFor: string[];
207
+ settings: IndexSettings;
208
+ withRetry: <T>(operation: () => Promise<T>) => Promise<T>;
209
+ }
210
+
211
+ /**
212
+ * Enforces the fields the better-auth schema declares unique and its tables'
213
+ * unique indexes. Every value of such a field, or combination of values of
214
+ * such an index, is backed by a reservation document in `<alias>_unique`
215
+ * whose `_id` derives from the fields and the values, so Elasticsearch refuses a
216
+ * second one the way it refuses a second row with the same id. Reservations
217
+ * are only read and written by `_id`, which is real-time, so none of their
218
+ * writes refresh.
219
+ */
220
+ export const createReservations = ({
221
+ client,
222
+ schema,
223
+ getDefaultModelName,
224
+ skipUniqueFor,
225
+ settings,
226
+ withRetry,
227
+ }: ReservationsOptions) => {
228
+ const skipped = new Set(skipUniqueFor);
229
+ const constraintsByModel = new Map<string, UniqueConstraint[]>();
230
+
231
+ /**
232
+ * The enforced constraints of a model, by its schema name. A field
233
+ * declared unique both ways is one constraint, with one reservation.
234
+ */
235
+ const constraintsOfDefault = (defaultModel: string) => {
236
+ let constraints = constraintsByModel.get(defaultModel);
237
+ if (!constraints) {
238
+ const table = schema[defaultModel];
239
+ const fields = table?.fields ?? {};
240
+ const declared = [
241
+ ...Object.entries(fields)
242
+ .filter(([, field]) => field.unique === true)
243
+ .map(([name]) => [name]),
244
+ ...(table?.indexes ?? [])
245
+ .filter((index) => index.unique === true)
246
+ .map((index) => [...index.fields]),
247
+ ];
248
+ const byName = new Map<string, UniqueConstraint>();
249
+ for (const names of declared) {
250
+ const name = names.join("+");
251
+ if (byName.has(name) || skipped.has(`${defaultModel}.${name}`)) {
252
+ continue;
253
+ }
254
+ byName.set(name, {
255
+ name,
256
+ fields: names.map((field) => ({
257
+ name: field,
258
+ dbName: fields[field]?.fieldName ?? field,
259
+ })),
260
+ });
261
+ }
262
+ constraints = [...byName.values()];
263
+ constraintsByModel.set(defaultModel, constraints);
264
+ }
265
+ return constraints;
266
+ };
267
+
268
+ /** The enforced constraints of a model, by its database name. */
269
+ const constraintsOf = (model: string) =>
270
+ constraintsOfDefault(getDefaultModelName(model));
271
+
272
+ /**
273
+ * The database names of the fields of a model's enforced constraints
274
+ * that `data` sets any field of, or of all of them without `data`. A
275
+ * write needs the row's current values of all of these.
276
+ */
277
+ const fieldsOf = (model: string, data?: Source) => [
278
+ ...new Set(
279
+ constraintsOf(model)
280
+ .filter(
281
+ (constraint) =>
282
+ !data || dbFieldsOf(constraint).some((dbName) => dbName in data),
283
+ )
284
+ .flatMap(dbFieldsOf),
285
+ ),
286
+ ];
287
+
288
+ /** The schema name of a field `data` sets that a constraint covers. */
289
+ const constrainedFieldIn = (model: string, data: Source) =>
290
+ constraintsOf(model)
291
+ .flatMap((constraint) => constraint.fields)
292
+ .find((field) => field.dbName in data)?.name;
293
+
294
+ const indexFor = (alias: string) => `${alias}_unique`;
295
+
296
+ // One creation check per reservation index and process. A failed check is
297
+ // forgotten so the next call tries again.
298
+ const ready = new Map<string, Promise<void>>();
299
+
300
+ /**
301
+ * Creates the reservation index on first use. Without it, the first
302
+ * reservation write would have Elasticsearch create it with a dynamic
303
+ * mapping.
304
+ */
305
+ const ensureIndex = async (alias: string) => {
306
+ const index = indexFor(alias);
307
+ let check = ready.get(index);
308
+ if (!check) {
309
+ check = withRetry(async () => {
310
+ try {
311
+ await client.indices.create({
312
+ index,
313
+ settings: {
314
+ number_of_shards: settings.numberOfShards,
315
+ number_of_replicas: settings.numberOfReplicas,
316
+ },
317
+ mappings: {
318
+ dynamic: "strict",
319
+ properties: {
320
+ field: { type: "keyword" },
321
+ owner: { type: "keyword" },
322
+ claimedAt: { type: "date", format: "epoch_millis" },
323
+ },
324
+ },
325
+ });
326
+ } catch (error) {
327
+ if (!isAlreadyExists(error)) throw error;
328
+ }
329
+ });
330
+ ready.set(index, check);
331
+ check.catch(() => ready.delete(index));
332
+ }
333
+ await check;
334
+ return index;
335
+ };
336
+
337
+ /** Rows of `alias` by id, real-time, with only `fields` of their source. */
338
+ const rowsById = async (alias: string, ids: string[], fields: string[]) => {
339
+ const rows = new Map<string, Source>();
340
+ if (ids.length === 0) return rows;
341
+ const response = await withRetry(() =>
342
+ client.mget<Source>({ index: alias, ids, _source: fields }),
343
+ );
344
+ for (const doc of response.docs) {
345
+ if ("found" in doc && doc.found) rows.set(doc._id, doc._source ?? {});
346
+ }
347
+ return rows;
348
+ };
349
+
350
+ /** A row of `alias` by id, real-time, with only `fields` of its source. */
351
+ const rowById = async (alias: string, id: string, ...fields: string[]) => {
352
+ try {
353
+ const doc = await withRetry(() =>
354
+ client.get<Source>({ index: alias, id, _source_includes: fields }),
355
+ );
356
+ return doc.found ? (doc._source ?? {}) : undefined;
357
+ } catch (error) {
358
+ if (isNotFound(error)) return undefined;
359
+ throw error;
360
+ }
361
+ };
362
+
363
+ /**
364
+ * Deletes reservations only if each is still as `version` says. Conditional
365
+ * deletes are safe to repeat. A failure only leaves orphans behind, which
366
+ * are taken over once the grace period has passed, so it is not reported.
367
+ */
368
+ const deleteUnchanged = async (index: string, entries: Entry[]) => {
369
+ const operations = entries.flatMap((entry) =>
370
+ entry.version ? [{ delete: { _id: entry.id, ...entry.version } }] : [],
371
+ );
372
+ if (operations.length === 0) return;
373
+ await withRetry(() => client.bulk({ index, operations })).catch(() => {});
374
+ };
375
+
376
+ /** The enforced values a stored row holds. */
377
+ const valuesOf = (model: string, source: Source | undefined) =>
378
+ constraintsOf(model).flatMap((constraint): HeldValues[] => {
379
+ const values = valuesIn(constraint, source);
380
+ return values ? [{ constraint, values }] : [];
381
+ });
382
+
383
+ /**
384
+ * What a write of `changes` to a row stored as `before` takes and gives up.
385
+ * Undefined when it changes no enforced value. `before` must hold every
386
+ * field of each constraint `changes` touches (see `fieldsOf`).
387
+ */
388
+ const changeOf = (
389
+ model: string,
390
+ id: string,
391
+ before: Source | undefined,
392
+ changes: Source,
393
+ ): RowChange | undefined => {
394
+ const after = { ...before, ...changes };
395
+ const claim: HeldValues[] = [];
396
+ const drop: HeldValues[] = [];
397
+ for (const constraint of constraintsOf(model)) {
398
+ if (!dbFieldsOf(constraint).some((dbName) => dbName in changes)) {
399
+ continue;
400
+ }
401
+ const next = valuesIn(constraint, after);
402
+ const previous = valuesIn(constraint, before);
403
+ if (JSON.stringify(next) === JSON.stringify(previous)) continue;
404
+ if (next) claim.push({ constraint, values: next });
405
+ if (previous) drop.push({ constraint, values: previous });
406
+ }
407
+ if (claim.length === 0 && drop.length === 0) return undefined;
408
+ return { id, claim, drop };
409
+ };
410
+
411
+ /**
412
+ * The claims of one write, of one row or a page of rows: taken before the
413
+ * rows are written, then either kept, with the dropped values released, or
414
+ * given back.
415
+ */
416
+ const claims = (model: string, alias: string) => {
417
+ let claimed: Entry[] = [];
418
+ let dropped: Entry[] = [];
419
+ let stampedAt = 0;
420
+
421
+ const entriesOf = (row: string, held: HeldValues[] = []) =>
422
+ held.map(
423
+ (values): Entry => ({
424
+ ...values,
425
+ row,
426
+ id: idOf(values),
427
+ created: false,
428
+ }),
429
+ );
430
+
431
+ /** The reservation document as stored now, or null. */
432
+ const current = async (index: string, id: string) => {
433
+ try {
434
+ const doc = await withRetry(() =>
435
+ client.get<Reservation>({ index, id }),
436
+ );
437
+ return doc._source
438
+ ? { source: doc._source, version: versionOf(doc) }
439
+ : null;
440
+ } catch (error) {
441
+ if (isNotFound(error)) return null;
442
+ throw error;
443
+ }
444
+ };
445
+
446
+ /**
447
+ * Claims one value the bulk claim could not: creates it if it is gone,
448
+ * re-stamps it if the row owns it, and takes it over if it is an orphan,
449
+ * each conditional on what was read. Returns false for a duplicate.
450
+ *
451
+ * These writes are not retried, like the adapter's other conditional
452
+ * writes: one that timed out may have landed and would then lose to
453
+ * itself.
454
+ */
455
+ const claimContested = async (
456
+ index: string,
457
+ entry: Entry,
458
+ seen?: { source: Reservation; version?: Version } | null,
459
+ ) => {
460
+ let reservation = seen;
461
+ for (let attempt = 0; attempt < MAX_CLAIM_ATTEMPTS; attempt++) {
462
+ reservation ??= await current(index, entry.id);
463
+ const now = Date.now();
464
+ const document = reservationFor(entry, entry.row, now);
465
+ try {
466
+ if (!reservation) {
467
+ const result = await client.index({
468
+ index,
469
+ id: entry.id,
470
+ op_type: "create",
471
+ document,
472
+ });
473
+ entry.created = true;
474
+ entry.version = versionOf(result);
475
+ return true;
476
+ }
477
+ const owned = reservation.source.owner === entry.row;
478
+ if (!owned) {
479
+ // A reservation whose row does not hold the value is a write in
480
+ // flight until the grace period has passed, an orphan after.
481
+ if (now - Number(reservation.source.claimedAt) < GRACE_MS) {
482
+ return false;
483
+ }
484
+ const owner = await rowById(
485
+ alias,
486
+ reservation.source.owner,
487
+ ...dbFieldsOf(entry.constraint),
488
+ );
489
+ if (holds(owner, entry)) return false;
490
+ }
491
+ if (!reservation.version) {
492
+ reservation = null;
493
+ continue;
494
+ }
495
+ const result = await client.index({
496
+ index,
497
+ id: entry.id,
498
+ document,
499
+ ...reservation.version,
500
+ });
501
+ // Taken over from an orphan, it is not this write's to give back.
502
+ if (!owned) entry.created = false;
503
+ entry.version = versionOf(result);
504
+ return true;
505
+ } catch (error) {
506
+ if (!isVersionConflict(error)) throw error;
507
+ reservation = null;
508
+ }
509
+ }
510
+ throw new Error(
511
+ `Claiming a unique value for a "${model}" row did not complete after ${MAX_CLAIM_ATTEMPTS} attempts because other requests kept changing it`,
512
+ );
513
+ };
514
+
515
+ return {
516
+ /**
517
+ * Claims the values `changes` take, in one bulk request, and re-stamps
518
+ * the values they give up so their release can tell whether a later
519
+ * write re-claimed them. Throws a duplicate key error, having given back
520
+ * what it created, if another row holds or is writing a value. Called
521
+ * again when a write reruns, it re-stamps its own claims.
522
+ */
523
+ async claim(changes: RowChange[]) {
524
+ const createdBefore = new Set(
525
+ claimed
526
+ .filter((entry) => entry.created)
527
+ .map((entry) => `${entry.row}\n${entry.id}`),
528
+ );
529
+ claimed = changes.flatMap((change) =>
530
+ entriesOf(change.id, change.claim),
531
+ );
532
+ dropped = changes.flatMap((change) =>
533
+ entriesOf(change.id, change.drop),
534
+ );
535
+ if (claimed.length === 0 && dropped.length === 0) return;
536
+
537
+ const index = await ensureIndex(alias);
538
+ stampedAt = Date.now();
539
+ const params = (entry: Entry) => ({
540
+ lang: "painless",
541
+ source: CLAIM_SCRIPT,
542
+ params: { owner: entry.row, now: stampedAt },
543
+ });
544
+ const response = await withRetry(() =>
545
+ client.bulk({
546
+ index,
547
+ operations: [
548
+ ...claimed.flatMap((entry) => [
549
+ { update: { _id: entry.id, retry_on_conflict: 3 } },
550
+ {
551
+ script: params(entry),
552
+ upsert: reservationFor(entry, entry.row, stampedAt),
553
+ _source: true,
554
+ },
555
+ ]),
556
+ ...dropped.flatMap((entry) => [
557
+ { update: { _id: entry.id } },
558
+ { script: params(entry) },
559
+ ]),
560
+ ],
561
+ }),
562
+ );
563
+
564
+ const contested: Promise<boolean>[] = [];
565
+ const duplicates: Entry[] = [];
566
+ let failure: estypes.ErrorCause | undefined;
567
+ response.items.forEach((item, position) => {
568
+ const outcome = item.update;
569
+ const entry =
570
+ position < claimed.length
571
+ ? claimed[position]
572
+ : dropped[position - claimed.length];
573
+ if (!outcome || !entry) return;
574
+ if (position >= claimed.length) {
575
+ // Only a value this write re-stamped is its own to release; one
576
+ // another row owns, or none at all, is left alone.
577
+ if (outcome.result === "updated")
578
+ entry.version = versionOf(outcome);
579
+ return;
580
+ }
581
+ if (outcome.result === "created") {
582
+ entry.created = true;
583
+ entry.version = versionOf(outcome);
584
+ } else if (outcome.result === "updated") {
585
+ // Created by an earlier run of this write, it still is its own.
586
+ entry.created = createdBefore.has(`${entry.row}\n${entry.id}`);
587
+ entry.version = versionOf(outcome);
588
+ } else if (outcome.result === "noop") {
589
+ const seen = outcome.get?._source
590
+ ? {
591
+ source: outcome.get._source as Reservation,
592
+ version: versionOf(outcome.get),
593
+ }
594
+ : null;
595
+ contested.push(
596
+ claimContested(index, entry, seen).then((claimedIt) => {
597
+ if (!claimedIt) duplicates.push(entry);
598
+ return claimedIt;
599
+ }),
600
+ );
601
+ } else if (outcome.status === 409 || outcome.status === 429) {
602
+ contested.push(
603
+ claimContested(index, entry).then((claimedIt) => {
604
+ if (!claimedIt) duplicates.push(entry);
605
+ return claimedIt;
606
+ }),
607
+ );
608
+ } else {
609
+ failure ??= outcome.error ?? { type: "unexpected bulk response" };
610
+ }
611
+ });
612
+ const settled = await Promise.allSettled(contested);
613
+ const rejected = settled.find(
614
+ (result): result is PromiseRejectedResult =>
615
+ result.status === "rejected",
616
+ );
617
+ if (failure || rejected || duplicates.length > 0) {
618
+ await deleteUnchanged(
619
+ index,
620
+ claimed.filter((entry) => entry.created),
621
+ );
622
+ if (rejected) throw rejected.reason;
623
+ if (failure) {
624
+ throw new Error(
625
+ `Claiming the unique values of a "${model}" row failed: ${failure.reason ?? failure.type}`,
626
+ );
627
+ }
628
+ const [first] = duplicates;
629
+ throw duplicateError(model, first?.constraint.name ?? "value");
630
+ }
631
+ },
632
+
633
+ /**
634
+ * After a write of the rows `ids`: releases the values they gave up,
635
+ * unless a later write re-claimed them since this one re-stamped them.
636
+ */
637
+ async written(ids: Iterable<string>) {
638
+ const rows = new Set(ids);
639
+ await deleteUnchanged(
640
+ indexFor(alias),
641
+ dropped.filter((entry) => rows.has(entry.row)),
642
+ );
643
+ },
644
+
645
+ /**
646
+ * After the rows `ids` were certainly not written: gives back the
647
+ * reservations this write created for them, unless re-claimed since.
648
+ */
649
+ async unwritten(ids: Iterable<string>) {
650
+ const rows = new Set(ids);
651
+ await deleteUnchanged(
652
+ indexFor(alias),
653
+ claimed.filter((entry) => entry.created && rows.has(entry.row)),
654
+ );
655
+ },
656
+
657
+ /**
658
+ * After the single row `id` failed to be written with `error`: gives
659
+ * back the reservations this write created, unless the write may have
660
+ * landed after all or the row holds the value. A write refused by a
661
+ * migration's write block keeps them for its rerun.
662
+ */
663
+ async failed(id: string, error: unknown) {
664
+ if (isWriteBlocked(error) || mayHaveLanded(error)) return;
665
+ const created = claimed.filter((entry) => entry.created);
666
+ if (created.length === 0) return;
667
+ let row: Source | undefined;
668
+ try {
669
+ row = await rowById(
670
+ alias,
671
+ id,
672
+ ...created.flatMap((entry) => dbFieldsOf(entry.constraint)),
673
+ );
674
+ } catch {
675
+ return;
676
+ }
677
+ await deleteUnchanged(
678
+ indexFor(alias),
679
+ created.filter((entry) => !holds(row, entry)),
680
+ );
681
+ },
682
+
683
+ /**
684
+ * The schema name of a value the row no longer owns, if any. A write
685
+ * landing later than half the grace period after its claim may have
686
+ * had one taken over meanwhile; one landing sooner cannot, so it is
687
+ * not asked.
688
+ */
689
+ async lost(): Promise<string | undefined> {
690
+ if (claimed.length === 0 || Date.now() - stampedAt <= GRACE_MS / 2) {
691
+ return undefined;
692
+ }
693
+ const response = await withRetry(() =>
694
+ client.mget<Reservation>({
695
+ index: indexFor(alias),
696
+ ids: claimed.map((entry) => entry.id),
697
+ }),
698
+ );
699
+ const entry = claimed.find((claim, position) => {
700
+ const doc = response.docs[position];
701
+ return !(
702
+ doc &&
703
+ "found" in doc &&
704
+ doc.found &&
705
+ doc._source?.owner === claim.row
706
+ );
707
+ });
708
+ return entry?.constraint.name;
709
+ },
710
+ };
711
+ };
712
+
713
+ /**
714
+ * After the rows were deleted: releases the values they held, unless
715
+ * another row has taken one over since.
716
+ */
717
+ const release = async (
718
+ model: string,
719
+ alias: string,
720
+ rows: { id: string; source: Source | undefined }[],
721
+ ) => {
722
+ const operations = rows.flatMap(({ id, source }) =>
723
+ valuesOf(model, source).flatMap((held) => [
724
+ { update: { _id: idOf(held) } },
725
+ {
726
+ script: {
727
+ lang: "painless",
728
+ source: RELEASE_SCRIPT,
729
+ params: { owner: id },
730
+ },
731
+ },
732
+ ]),
733
+ );
734
+ if (operations.length === 0) return;
735
+ const index = await ensureIndex(alias);
736
+ // Repeating it is safe, and a failure only leaves orphans behind.
737
+ await withRetry(() => client.bulk({ index, operations })).catch(() => {});
738
+ };
739
+
740
+ /**
741
+ * Reserves the values of rows that hold none, as rows written before
742
+ * reservations existed do, and reports values more than one row holds.
743
+ * Changes nothing for rows already reserved, so it can run every time.
744
+ */
745
+ const backfill = async (
746
+ defaultModel: string,
747
+ alias: string,
748
+ ): Promise<DuplicateValues[]> => {
749
+ const constraints = constraintsOfDefault(defaultModel);
750
+ if (constraints.length === 0) return [];
751
+ if (!(await withRetry(() => client.indices.exists({ index: alias })))) {
752
+ return [];
753
+ }
754
+ const index = await ensureIndex(alias);
755
+ const dbNames = [...new Set(constraints.flatMap(dbFieldsOf))];
756
+ const duplicates = new Map<string, { field: string; ids: Set<string> }>();
757
+
758
+ let searchAfter: estypes.SortResults | undefined;
759
+ while (true) {
760
+ const page = await withRetry(() =>
761
+ client.search<Source>({
762
+ index: alias,
763
+ size: BACKFILL_PAGE_SIZE,
764
+ sort: [{ id: "asc" }],
765
+ search_after: searchAfter,
766
+ _source: dbNames,
767
+ track_total_hits: false,
768
+ }),
769
+ );
770
+ const hits = page.hits.hits;
771
+ if (hits.length === 0) break;
772
+ searchAfter = hits[hits.length - 1]?.sort;
773
+
774
+ const entries = hits.flatMap((hit): BackfillEntry[] =>
775
+ constraints.flatMap((constraint) => {
776
+ const values = valuesIn(constraint, hit._source);
777
+ if (!hit._id || !values) return [];
778
+ const held = { constraint, values };
779
+ return [
780
+ { ...held, row: hit._id, source: hit._source, id: idOf(held) },
781
+ ];
782
+ }),
783
+ );
784
+ if (entries.length > 0) {
785
+ const claimedAt = Date.now();
786
+ const response = await withRetry(() =>
787
+ client.bulk({
788
+ index,
789
+ operations: entries.flatMap((entry) => [
790
+ { create: { _id: entry.id } },
791
+ reservationFor(entry, entry.row, claimedAt),
792
+ ]),
793
+ }),
794
+ );
795
+ const taken = entries.filter((entry, position) => {
796
+ const outcome = response.items[position]?.create;
797
+ if (outcome?.status === 409) return true;
798
+ if (!outcome || outcome.error) {
799
+ throw new Error(
800
+ `Reserving the unique values of "${alias}" failed: ${outcome?.error?.reason ?? outcome?.error?.type ?? "unexpected bulk response"}`,
801
+ );
802
+ }
803
+ return false;
804
+ });
805
+ if (taken.length > 0) {
806
+ await resolveTaken(index, alias, taken, dbNames, (entry, owner) => {
807
+ const group = duplicates.get(entry.id) ?? {
808
+ field: entry.constraint.name,
809
+ ids: new Set<string>(),
810
+ };
811
+ group.ids.add(owner).add(entry.row);
812
+ duplicates.set(entry.id, group);
813
+ });
814
+ }
815
+ }
816
+ if (hits.length < BACKFILL_PAGE_SIZE) break;
817
+ }
818
+
819
+ return [...duplicates.values()].map(({ field, ids }) => ({
820
+ model: defaultModel,
821
+ field,
822
+ ids: [...ids].sort(),
823
+ }));
824
+ };
825
+
826
+ /**
827
+ * For backfilled values that already had a reservation: reports those whose
828
+ * owner holds the value too, and takes over orphans past the grace period.
829
+ * A reservation still in flight is left to its write.
830
+ */
831
+ const resolveTaken = async (
832
+ index: string,
833
+ alias: string,
834
+ taken: BackfillEntry[],
835
+ dbNames: string[],
836
+ onDuplicate: (entry: BackfillEntry, owner: string) => void,
837
+ ) => {
838
+ const ids = [...new Set(taken.map((entry) => entry.id))];
839
+ const response = await withRetry(() =>
840
+ client.mget<Reservation>({ index, ids }),
841
+ );
842
+ const reservations = new Map<
843
+ string,
844
+ { source: Reservation; version?: Version }
845
+ >();
846
+ for (const doc of response.docs) {
847
+ if ("found" in doc && doc.found && doc._source) {
848
+ reservations.set(doc._id, {
849
+ source: doc._source,
850
+ version: versionOf(doc),
851
+ });
852
+ }
853
+ }
854
+ // Only rows other than the backfilled one can be duplicates.
855
+ const owners = await rowsById(
856
+ alias,
857
+ [
858
+ ...new Set(
859
+ taken.flatMap((entry) => {
860
+ const owner = reservations.get(entry.id)?.source.owner;
861
+ return owner && owner !== entry.row ? [owner] : [];
862
+ }),
863
+ ),
864
+ ],
865
+ dbNames,
866
+ );
867
+ for (const entry of taken) {
868
+ const reservation = reservations.get(entry.id);
869
+ // Released since the bulk ran; the next run reserves it.
870
+ if (!reservation || reservation.source.owner === entry.row) continue;
871
+ const owner = reservation.source.owner;
872
+ if (holds(owners.get(owner), entry)) {
873
+ onDuplicate(entry, owner);
874
+ continue;
875
+ }
876
+ const now = Date.now();
877
+ if (
878
+ !reservation.version ||
879
+ now - Number(reservation.source.claimedAt) < GRACE_MS
880
+ ) {
881
+ continue;
882
+ }
883
+ try {
884
+ const document = reservationFor(entry, entry.row, now);
885
+ const result = await client.index({
886
+ index,
887
+ id: entry.id,
888
+ document,
889
+ ...reservation.version,
890
+ });
891
+ reservation.source = document;
892
+ reservation.version = versionOf(result);
893
+ // A later row of this page with the same value is then a duplicate
894
+ // of this one.
895
+ owners.set(entry.row, entry.source ?? {});
896
+ } catch (error) {
897
+ if (!isVersionConflict(error)) throw error;
898
+ }
899
+ }
900
+ };
901
+
902
+ return {
903
+ constraintsOf,
904
+ fieldsOf,
905
+ constrainedFieldIn,
906
+ valuesOf,
907
+ changeOf,
908
+ claims,
909
+ release,
910
+ backfill,
911
+ };
912
+ };