@etohq/search 2.0.4 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/NOTICE +24 -0
  2. package/README.md +189 -0
  3. package/dist/index.d.ts +1 -1
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +3 -17
  6. package/dist/index.js.map +1 -1
  7. package/dist/joiner-config.d.ts +8 -0
  8. package/dist/joiner-config.d.ts.map +1 -0
  9. package/dist/joiner-config.js +12 -0
  10. package/dist/joiner-config.js.map +1 -0
  11. package/dist/loaders/providers.d.ts +2 -15
  12. package/dist/loaders/providers.d.ts.map +1 -1
  13. package/dist/loaders/providers.js +14 -37
  14. package/dist/loaders/providers.js.map +1 -1
  15. package/dist/migrations/Migration20260805085626.d.ts +6 -0
  16. package/dist/migrations/Migration20260805085626.d.ts.map +1 -0
  17. package/dist/migrations/Migration20260805085626.js +23 -0
  18. package/dist/migrations/Migration20260805085626.js.map +1 -0
  19. package/dist/migrations/Migration20260903085616.d.ts +6 -0
  20. package/dist/migrations/Migration20260903085616.d.ts.map +1 -0
  21. package/dist/migrations/Migration20260903085616.js +32 -0
  22. package/dist/migrations/Migration20260903085616.js.map +1 -0
  23. package/dist/migrations/Migration20260911115931.d.ts +6 -0
  24. package/dist/migrations/Migration20260911115931.d.ts.map +1 -0
  25. package/dist/migrations/Migration20260911115931.js +14 -0
  26. package/dist/migrations/Migration20260911115931.js.map +1 -0
  27. package/dist/models/index.d.ts +4 -0
  28. package/dist/models/index.d.ts.map +1 -0
  29. package/dist/models/index.js +10 -0
  30. package/dist/models/index.js.map +1 -0
  31. package/dist/models/search-index-sync.d.ts +34 -0
  32. package/dist/models/search-index-sync.d.ts.map +1 -0
  33. package/dist/models/search-index-sync.js +47 -0
  34. package/dist/models/search-index-sync.js.map +1 -0
  35. package/dist/models/search-index-version.d.ts +34 -0
  36. package/dist/models/search-index-version.d.ts.map +1 -0
  37. package/dist/models/search-index-version.js +37 -0
  38. package/dist/models/search-index-version.js.map +1 -0
  39. package/dist/models/search-index.d.ts +28 -0
  40. package/dist/models/search-index.d.ts.map +1 -0
  41. package/dist/models/search-index.js +28 -0
  42. package/dist/models/search-index.js.map +1 -0
  43. package/dist/services/index.d.ts +1 -0
  44. package/dist/services/index.d.ts.map +1 -1
  45. package/dist/services/index.js +3 -1
  46. package/dist/services/index.js.map +1 -1
  47. package/dist/services/search-module-service.d.ts +67 -15
  48. package/dist/services/search-module-service.d.ts.map +1 -1
  49. package/dist/services/search-module-service.js +328 -12
  50. package/dist/services/search-module-service.js.map +1 -1
  51. package/dist/services/search-provider.d.ts +23 -0
  52. package/dist/services/search-provider.d.ts.map +1 -0
  53. package/dist/services/search-provider.js +64 -0
  54. package/dist/services/search-provider.js.map +1 -0
  55. package/dist/tsconfig.tsbuildinfo +1 -1
  56. package/dist/types/index.d.ts +129 -6
  57. package/dist/types/index.d.ts.map +1 -1
  58. package/dist/types/index.js +4 -0
  59. package/dist/types/index.js.map +1 -1
  60. package/dist/utils/active-version-cache.d.ts +36 -0
  61. package/dist/utils/active-version-cache.d.ts.map +1 -0
  62. package/dist/utils/active-version-cache.js +73 -0
  63. package/dist/utils/active-version-cache.js.map +1 -0
  64. package/dist/utils/deletion.d.ts +11 -0
  65. package/dist/utils/deletion.d.ts.map +1 -0
  66. package/dist/utils/deletion.js +50 -0
  67. package/dist/utils/deletion.js.map +1 -0
  68. package/dist/utils/index.d.ts +88 -0
  69. package/dist/utils/index.d.ts.map +1 -0
  70. package/dist/utils/index.js +524 -0
  71. package/dist/utils/index.js.map +1 -0
  72. package/dist/utils/ingestion.d.ts +12 -0
  73. package/dist/utils/ingestion.d.ts.map +1 -0
  74. package/dist/utils/ingestion.js +78 -0
  75. package/dist/utils/ingestion.js.map +1 -0
  76. package/dist/utils/migrations.d.ts +5 -0
  77. package/dist/utils/migrations.d.ts.map +1 -0
  78. package/dist/utils/migrations.js +143 -0
  79. package/dist/utils/migrations.js.map +1 -0
  80. package/dist/utils/rate-limit.d.ts +9 -0
  81. package/dist/utils/rate-limit.d.ts.map +1 -0
  82. package/dist/utils/rate-limit.js +47 -0
  83. package/dist/utils/rate-limit.js.map +1 -0
  84. package/dist/utils/seeding.d.ts +29 -0
  85. package/dist/utils/seeding.d.ts.map +1 -0
  86. package/dist/utils/seeding.js +639 -0
  87. package/dist/utils/seeding.js.map +1 -0
  88. package/dist/utils/versions.d.ts +50 -0
  89. package/dist/utils/versions.d.ts.map +1 -0
  90. package/dist/utils/versions.js +113 -0
  91. package/dist/utils/versions.js.map +1 -0
  92. package/package.json +9 -15
@@ -0,0 +1,143 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createIndexMigrationPlan = createIndexMigrationPlan;
4
+ exports.executeIndexMigrationPlan = executeIndexMigrationPlan;
5
+ const index_1 = require("./index");
6
+ const versions_1 = require("./versions");
7
+ async function createIndexMigrationPlan(context) {
8
+ const definitions = [...context.indexes.values()];
9
+ // Every record, not only the ones a definition names: an index nothing
10
+ // declares any more is exactly what a `drop` is planned from.
11
+ const records = (await context.indexService.list({}, { take: null }));
12
+ if (!definitions.length && !records.length) {
13
+ return [];
14
+ }
15
+ const byName = new Map(records.map((record) => [record.name, record]));
16
+ const versionsByIndexId = await (0, index_1.listVersionsByIndexId)(context, records.map((record) => record.id));
17
+ const actions = definitions.map((definition) => {
18
+ const record = byName.get(definition.name);
19
+ const shared = {
20
+ index: definition.name,
21
+ definition_hash: definition.definition_hash,
22
+ };
23
+ if (!record) {
24
+ return {
25
+ ...shared,
26
+ action: "create",
27
+ physical_name: (0, versions_1.versionPhysicalName)(definition, 1),
28
+ version: 1,
29
+ };
30
+ }
31
+ const versions = versionsByIndexId.get(record.id) ?? [];
32
+ const activeVersion = record.active_version != null
33
+ ? versions.find((v) => v.version === record.active_version)
34
+ : undefined;
35
+ if (activeVersion &&
36
+ activeVersion.definition_hash === definition.definition_hash &&
37
+ activeVersion.provider === definition.provider) {
38
+ return {
39
+ ...shared,
40
+ action: "noop",
41
+ physical_name: activeVersion.physical_name,
42
+ };
43
+ }
44
+ // Already building or built, waiting to be seeded — nothing new to plan.
45
+ const pending = versions.find((v) => (record.active_version == null ||
46
+ v.version > record.active_version) &&
47
+ v.definition_hash === definition.definition_hash &&
48
+ v.provider === definition.provider);
49
+ if (pending) {
50
+ return {
51
+ ...shared,
52
+ action: "noop",
53
+ physical_name: pending.physical_name,
54
+ };
55
+ }
56
+ const highest = versions[0]?.version ?? 0;
57
+ const version = highest + 1;
58
+ const physicalName = (0, versions_1.versionPhysicalName)(definition, version);
59
+ if (!activeVersion) {
60
+ // A `SearchIndex` row exists — an earlier `create` ran — but nothing
61
+ // ever went live, so this is still a `create`, not a `migrate`.
62
+ return {
63
+ ...shared,
64
+ action: "create",
65
+ physical_name: physicalName,
66
+ version,
67
+ };
68
+ }
69
+ return {
70
+ ...shared,
71
+ action: "migrate",
72
+ physical_name: physicalName,
73
+ version,
74
+ active_physical_name: activeVersion.physical_name,
75
+ active_definition_hash: activeVersion.definition_hash,
76
+ provider: definition.provider,
77
+ ...(activeVersion.provider !== definition.provider
78
+ ? { previous_provider: activeVersion.provider }
79
+ : {}),
80
+ };
81
+ });
82
+ const declared = new Set(definitions.map((definition) => definition.name));
83
+ for (const record of records) {
84
+ if (declared.has(record.name)) {
85
+ continue;
86
+ }
87
+ actions.push({
88
+ action: "drop",
89
+ index: record.name,
90
+ // Newest version first, the order `listVersionsByIndexId` returns.
91
+ physical_names: (versionsByIndexId.get(record.id) ?? []).map((version) => version.physical_name),
92
+ });
93
+ }
94
+ return actions;
95
+ }
96
+ // Note: Seeding data is done by the application start hook, migrations are reserved for schema changes.
97
+ async function executeIndexMigrationPlan(context, actions) {
98
+ for (const action of actions) {
99
+ if (action.action === "drop") {
100
+ await (0, versions_1.dropIndex)(context, action.index);
101
+ continue;
102
+ }
103
+ if (action.action === "noop") {
104
+ // Nothing changed for this index, but a version below the active one —
105
+ // left behind by an earlier migration's swap — still needs cleaning up.
106
+ // That cannot wait for the index to drift again.
107
+ const record = await getOrCreateIndexRecord(context, action.index);
108
+ await (0, versions_1.cleanupStaleVersions)(context, record);
109
+ continue;
110
+ }
111
+ const definition = (0, index_1.retrieveIndexDefinition)(context.indexes, action.index);
112
+ const provider = context.providers.retrieve(definition.provider);
113
+ await provider.upsertIndex({
114
+ index: { ...definition, physical_name: action.physical_name },
115
+ });
116
+ const record = await getOrCreateIndexRecord(context, definition.name);
117
+ await context.versionService.create([
118
+ {
119
+ search_index_id: record.id,
120
+ version: action.version,
121
+ provider: definition.provider,
122
+ physical_name: action.physical_name,
123
+ definition_hash: action.definition_hash,
124
+ // Built but not filled; the seed at startup makes it ready.
125
+ status: index_1.SearchIndexState.PENDING,
126
+ },
127
+ ]);
128
+ await (0, versions_1.cleanupStaleVersions)(context, record);
129
+ }
130
+ }
131
+ async function getOrCreateIndexRecord(context, name) {
132
+ const [existing] = (await context.indexService.list({
133
+ name,
134
+ }));
135
+ if (existing) {
136
+ return existing;
137
+ }
138
+ const [created] = (await context.indexService.create([
139
+ { name, active_version: null },
140
+ ]));
141
+ return created;
142
+ }
143
+ //# sourceMappingURL=migrations.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/utils/migrations.ts"],"names":[],"mappings":";;AAaA,4DA0HC;AAGD,8DA0CC;AAlLD,mCAIgB;AAChB,yCAImB;AAEZ,KAAK,UAAU,wBAAwB,CAC5C,OAA4B;IAE5B,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAEjD,uEAAuE;IACvE,8DAA8D;IAC9D,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAC9C,EAAE,EACF,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAwB,CAAA;IAEzB,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3C,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAqB,EACnD,OAAO,EACP,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CACnC,CAAA;IAED,MAAM,OAAO,GAA6C,WAAW,CAAC,GAAG,CACvE,CAAC,UAAU,EAAE,EAAE;QACb,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAC1C,MAAM,MAAM,GAAG;YACb,KAAK,EAAE,UAAU,CAAC,IAAI;YACtB,eAAe,EAAE,UAAU,CAAC,eAAe;SAC5C,CAAA;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAiB;gBACzB,aAAa,EAAE,IAAA,8BAAmB,EAAC,UAAU,EAAE,CAAC,CAAC;gBACjD,OAAO,EAAE,CAAC;aACX,CAAA;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;QACvD,MAAM,aAAa,GACjB,MAAM,CAAC,cAAc,IAAI,IAAI;YAC3B,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,cAAc,CAAC;YAC3D,CAAC,CAAC,SAAS,CAAA;QAEf,IACE,aAAa;YACb,aAAa,CAAC,eAAe,KAAK,UAAU,CAAC,eAAe;YAC5D,aAAa,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,EAC9C,CAAC;YACD,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,MAAe;gBACvB,aAAa,EAAE,aAAa,CAAC,aAAa;aAC3C,CAAA;QACH,CAAC;QAED,yEAAyE;QACzE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,MAAM,CAAC,cAAc,IAAI,IAAI;YAC5B,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,CAAC;YACpC,CAAC,CAAC,eAAe,KAAK,UAAU,CAAC,eAAe;YAChD,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CACrC,CAAA;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,MAAe;gBACvB,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAA;QACH,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAA;QACzC,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,CAAA;QAC3B,MAAM,YAAY,GAAG,IAAA,8BAAmB,EAAC,UAAU,EAAE,OAAO,CAAC,CAAA;QAE7D,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,qEAAqE;YACrE,gEAAgE;YAChE,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAiB;gBACzB,aAAa,EAAE,YAAY;gBAC3B,OAAO;aACR,CAAA;QACH,CAAC;QAED,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,SAAkB;YAC1B,aAAa,EAAE,YAAY;YAC3B,OAAO;YACP,oBAAoB,EAAE,aAAa,CAAC,aAAa;YACjD,sBAAsB,EAAE,aAAa,CAAC,eAAe;YACrD,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,GAAG,CAAC,aAAa,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ;gBAChD,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,CAAC,QAAQ,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;SACR,CAAA;IACH,CAAC,CACF,CAAA;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;IAE1E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,SAAQ;QACV,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,MAAM,CAAC,IAAI;YAClB,mEAAmE;YACnE,cAAc,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC1D,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,CACnC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,wGAAwG;AACjG,KAAK,UAAU,yBAAyB,CAC7C,OAA4B,EAC5B,OAAiD;IAEjD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAA,oBAAS,EAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;YACtC,SAAQ;QACV,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,uEAAuE;YACvE,wEAAwE;YACxE,iDAAiD;YACjD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;YAClE,MAAM,IAAA,+BAAoB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YAC3C,SAAQ;QACV,CAAC;QAED,MAAM,UAAU,GAAG,IAAA,+BAAuB,EAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;QAEhE,MAAM,QAAQ,CAAC,WAAW,CAAC;YACzB,KAAK,EAAE,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE;SAC9D,CAAC,CAAA;QAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;QAErE,MAAM,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;YAClC;gBACE,eAAe,EAAE,MAAM,CAAC,EAAE;gBAC1B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,4DAA4D;gBAC5D,MAAM,EAAE,wBAAgB,CAAC,OAAO;aACjC;SACF,CAAC,CAAA;QAEF,MAAM,IAAA,+BAAoB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC7C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,OAA4B,EAC5B,IAAY;IAEZ,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;QAClD,IAAI;KACL,CAAC,CAAwB,CAAA;IAE1B,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC;QACnD,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE;KAC/B,CAAC,CAAwB,CAAA;IAE1B,OAAO,OAAO,CAAA;AAChB,CAAC"}
@@ -0,0 +1,9 @@
1
+ import { Logger } from "@etohq/framework/types";
2
+ export declare function isRateLimitError(error: unknown): boolean;
3
+ export declare function parseRetryAfter(header: string | null | undefined): number | undefined;
4
+ export declare function rateLimitDelay(attempt: number, retryAfter?: number, random?: () => number): number;
5
+ export declare function retryOnRateLimit<T>(operation: () => Promise<T>, { label, logger }: {
6
+ label: string;
7
+ logger?: Pick<Logger, "warn">;
8
+ }): Promise<T>;
9
+ //# sourceMappingURL=rate-limit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rate-limit.d.ts","sourceRoot":"","sources":["../../src/utils/rate-limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAQ/C,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAExD;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAChC,MAAM,GAAG,SAAS,CAIpB;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,MAAM,MAAoB,GACjC,MAAM,CASR;AAMD,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAClE,OAAO,CAAC,CAAC,CAAC,CAuBZ"}
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.isRateLimitError = isRateLimitError;
4
+ exports.parseRetryAfter = parseRetryAfter;
5
+ exports.rateLimitDelay = rateLimitDelay;
6
+ exports.retryOnRateLimit = retryOnRateLimit;
7
+ // Seeding is background work with nothing waiting on it, so it can afford to
8
+ // sit out a rate limit rather than fail the run.
9
+ const MAX_RETRIES = 5;
10
+ const INITIAL_DELAY = 2_000;
11
+ const MAX_DELAY = 60_000;
12
+ function isRateLimitError(error) {
13
+ return error?.status === 429;
14
+ }
15
+ function parseRetryAfter(header) {
16
+ const seconds = Number(header);
17
+ return header && Number.isFinite(seconds) ? seconds * 1000 : undefined;
18
+ }
19
+ function rateLimitDelay(attempt, retryAfter, random = Math.random) {
20
+ if (retryAfter !== undefined) {
21
+ return Math.min(retryAfter, MAX_DELAY);
22
+ }
23
+ const exponential = Math.min(INITIAL_DELAY * 2 ** attempt, MAX_DELAY);
24
+ // Jittered so replicas rate limited together don't all come back at once.
25
+ return Math.round(exponential * (0.5 + random() * 0.5));
26
+ }
27
+ function sleep(ms) {
28
+ return new Promise((resolve) => setTimeout(resolve, ms));
29
+ }
30
+ async function retryOnRateLimit(operation, { label, logger }) {
31
+ let attempt = 0;
32
+ for (;;) {
33
+ try {
34
+ return await operation();
35
+ }
36
+ catch (error) {
37
+ if (!isRateLimitError(error) || attempt >= MAX_RETRIES) {
38
+ throw error;
39
+ }
40
+ const delay = rateLimitDelay(attempt, error.retry_after);
41
+ logger?.warn(`[Search] Rate limited on ${label}: waiting ${delay}ms before retry ${attempt + 1}/${MAX_RETRIES} (${error.message})`);
42
+ await sleep(delay);
43
+ attempt++;
44
+ }
45
+ }
46
+ }
47
+ //# sourceMappingURL=rate-limit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rate-limit.js","sourceRoot":"","sources":["../../src/utils/rate-limit.ts"],"names":[],"mappings":";;AAQA,4CAEC;AAED,0CAMC;AAED,wCAaC;AAMD,4CA0BC;AA/DD,6EAA6E;AAC7E,iDAAiD;AACjD,MAAM,WAAW,GAAG,CAAC,CAAA;AACrB,MAAM,aAAa,GAAG,KAAK,CAAA;AAC3B,MAAM,SAAS,GAAG,MAAM,CAAA;AAExB,SAAgB,gBAAgB,CAAC,KAAc;IAC7C,OAAQ,KAA6B,EAAE,MAAM,KAAK,GAAG,CAAA;AACvD,CAAC;AAED,SAAgB,eAAe,CAC7B,MAAiC;IAEjC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAE9B,OAAO,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;AACxE,CAAC;AAED,SAAgB,cAAc,CAC5B,OAAe,EACf,UAAmB,EACnB,SAAuB,IAAI,CAAC,MAAM;IAElC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,IAAI,OAAO,EAAE,SAAS,CAAC,CAAA;IAErE,0EAA0E;IAC1E,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAEM,KAAK,UAAU,gBAAgB,CACpC,SAA2B,EAC3B,EAAE,KAAK,EAAE,MAAM,EAAoD;IAEnE,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,SAAS,CAAC;QACR,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAA;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;gBACvD,MAAM,KAAK,CAAA;YACb,CAAC;YAED,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;YAExD,MAAM,EAAE,IAAI,CACV,4BAA4B,KAAK,aAAa,KAAK,mBACjD,OAAO,GAAG,CACZ,IAAI,WAAW,KAAK,KAAK,CAAC,OAAO,GAAG,CACrC,CAAA;YAED,MAAM,KAAK,CAAC,KAAK,CAAC,CAAA;YAClB,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,29 @@
1
+ import { SearchTypes } from "@etohq/framework/types";
2
+ import { EtoError } from "@etohq/framework/utils";
3
+ import { SearchIndexContext, SearchIndexRegistry, SearchIndexSeedAction, SearchSeedRuntime } from "../types";
4
+ type LockContext = Pick<SearchIndexContext, "locking" | "logger">;
5
+ /**
6
+ * Which indexes need data, and which version it goes into. The highest version
7
+ * above the active one is what a migration built for a schema or provider
8
+ * change; it needs filling and then becoming active. Otherwise an index needs
9
+ * seeding in place if its active version was never filled, its last attempt
10
+ * failed, or it holds nothing — the last catching an engine that lost its data
11
+ * while the record still says ready.
12
+ */
13
+ export declare function createSeedPlan(context: SearchIndexRegistry): Promise<SearchIndexSeedAction[]>;
14
+ export declare function executeSeedPlan(context: SearchSeedRuntime & LockContext, actions: SearchIndexSeedAction[]): Promise<void>;
15
+ export declare function reindexIndexes(context: SearchSeedRuntime & LockContext, input?: SearchTypes.SearchReindexInput): Promise<SearchTypes.SearchReindexResult>;
16
+ /**
17
+ * A seed failed in a way that leaves the index inconsistent with the run's
18
+ * cursor, so the next run has to rebuild from the start rather than carry on
19
+ * from where this one stopped.
20
+ */
21
+ export declare class SearchSeedProgressLost extends EtoError {
22
+ readonly cause?: unknown | undefined;
23
+ constructor(message: string, cause?: unknown | undefined);
24
+ }
25
+ export declare class SearchSeedLockLost extends EtoError {
26
+ constructor(message: string);
27
+ }
28
+ export {};
29
+ //# sourceMappingURL=seeding.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"seeding.d.ts","sourceRoot":"","sources":["../../src/utils/seeding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EACL,kBAAkB,EAElB,mBAAmB,EACnB,qBAAqB,EAIrB,iBAAiB,EAClB,MAAM,QAAQ,CAAA;AAkBf,KAAK,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAA;AAQjE;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA8FlC;AAeD,wBAAsB,eAAe,CACnC,OAAO,EAAE,iBAAiB,GAAG,WAAW,EACxC,OAAO,EAAE,qBAAqB,EAAE,GAC/B,OAAO,CAAC,IAAI,CAAC,CAUf;AA4JD,wBAAsB,cAAc,CAClC,OAAO,EAAE,iBAAiB,GAAG,WAAW,EACxC,KAAK,GAAE,WAAW,CAAC,kBAAuB,GACzC,OAAO,CAAC,WAAW,CAAC,mBAAmB,CAAC,CA2B1C;AA+ZD;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,QAAQ;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO;gBAAzC,OAAO,EAAE,MAAM,EAAW,KAAK,CAAC,EAAE,OAAO,YAAA;CAItD;AAED,qBAAa,kBAAmB,SAAQ,QAAQ;gBAClC,OAAO,EAAE,MAAM;CAI5B"}