@cosmicdrift/kumiko-framework 0.406.0 → 0.409.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/api/framework-db.md +1 -0
- package/agents/api/framework-engine.md +15 -2
- package/agents/api/framework-ui-types.md +15 -0
- package/agents/api/renderer-web.md +156 -2
- package/agents/api/types.md +113 -71
- package/agents/api.md +4 -4
- package/agents/boot-rules.md +743 -0
- package/agents/examples/contact.md +167 -0
- package/agents/examples/delivery-notifications.md +34 -1
- package/agents/examples/record-detail-layout.md +13 -2
- package/agents/examples/search.md +65 -3
- package/agents/examples/secrets-demo.md +96 -2
- package/agents/examples/tenant-inactivity.md +138 -0
- package/agents/examples.md +2 -0
- package/agents/features-index.md +4 -0
- package/agents/features.md +24 -1
- package/agents/widgets.md +12 -1
- package/dist/api/export/export-rows.js +1 -0
- package/dist/compliance/index.d.ts +1 -1
- package/dist/db/event-store-executor-read-where.js +15 -5
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/migrate-generator.js +3 -1
- package/dist/db/queries/event-consumer.js +11 -7
- package/dist/db/queries/event-store.d.ts +2 -1
- package/dist/db/queries/event-store.js +25 -6
- package/dist/db/queries/rls-boot-catalog.js +3 -1
- package/dist/db/queries/session-advisory-lock.d.ts +7 -0
- package/dist/db/queries/session-advisory-lock.js +23 -0
- package/dist/db/query.js +7 -10
- package/dist/db/row-level-security-connection.js +4 -2
- package/dist/engine/active-membership.js +1 -0
- package/dist/engine/boot-validator/access-declarations.js +57 -33
- package/dist/engine/boot-validator/action-wiring.js +8 -6
- package/dist/engine/boot-validator/agent-effect-rules.d.ts +12 -0
- package/dist/engine/boot-validator/agent-effect-rules.js +12 -0
- package/dist/engine/boot-validator/agent-effect.d.ts +2 -0
- package/dist/engine/boot-validator/agent-effect.js +20 -0
- package/dist/engine/boot-validator/boot-context.js +5 -1
- package/dist/engine/boot-validator/boot-environment-rules.d.ts +12 -0
- package/dist/engine/boot-validator/boot-environment-rules.js +12 -0
- package/dist/engine/boot-validator/boot-phases-app.js +5 -1
- package/dist/engine/boot-validator/boot-phases.js +3 -1
- package/dist/engine/boot-validator/boot-rules.d.ts +2619 -0
- package/dist/engine/boot-validator/boot-rules.js +58 -0
- package/dist/engine/boot-validator/bundled-lists-table.js +23 -9
- package/dist/engine/boot-validator/config-deps.js +71 -23
- package/dist/engine/boot-validator/config-edit-action-rules.d.ts +17 -0
- package/dist/engine/boot-validator/config-edit-action-rules.js +17 -0
- package/dist/engine/boot-validator/config-key-rules.d.ts +117 -0
- package/dist/engine/boot-validator/config-key-rules.js +117 -0
- package/dist/engine/boot-validator/custom-screen-write-qns.js +6 -2
- package/dist/engine/boot-validator/derived-sql-fields-rules.d.ts +77 -0
- package/dist/engine/boot-validator/derived-sql-fields-rules.js +77 -0
- package/dist/engine/boot-validator/derived-sql-fields.js +53 -17
- package/dist/engine/boot-validator/detail-screens.js +10 -4
- package/dist/engine/boot-validator/entity-declaration-rules.d.ts +92 -0
- package/dist/engine/boot-validator/entity-declaration-rules.js +92 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.d.ts +102 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.js +102 -0
- package/dist/engine/boot-validator/entity-handler.d.ts +0 -7
- package/dist/engine/boot-validator/entity-handler.js +259 -355
- package/dist/engine/boot-validator/entity-image-variants.d.ts +2 -0
- package/dist/engine/boot-validator/entity-image-variants.js +91 -0
- package/dist/engine/boot-validator/entity-list-aggregate-options.js +8 -2
- package/dist/engine/boot-validator/entity-reference-embedded-rules.d.ts +117 -0
- package/dist/engine/boot-validator/entity-reference-embedded-rules.js +117 -0
- package/dist/engine/boot-validator/entity-select-image-rules.d.ts +122 -0
- package/dist/engine/boot-validator/entity-select-image-rules.js +122 -0
- package/dist/engine/boot-validator/entity-select.d.ts +8 -0
- package/dist/engine/boot-validator/entity-select.js +197 -0
- package/dist/engine/boot-validator/entity-structure-rules.d.ts +107 -0
- package/dist/engine/boot-validator/entity-structure-rules.js +107 -0
- package/dist/engine/boot-validator/field-conditions-rules.d.ts +92 -0
- package/dist/engine/boot-validator/field-conditions-rules.js +92 -0
- package/dist/engine/boot-validator/field-conditions.js +67 -22
- package/dist/engine/boot-validator/form-derivations-rules.d.ts +77 -0
- package/dist/engine/boot-validator/form-derivations-rules.js +77 -0
- package/dist/engine/boot-validator/form-derivations.js +52 -18
- package/dist/engine/boot-validator/global-tenancy.js +13 -7
- package/dist/engine/boot-validator/handler-declaration-rules.d.ts +72 -0
- package/dist/engine/boot-validator/handler-declaration-rules.js +72 -0
- package/dist/engine/boot-validator/handler-field-references-rules.d.ts +22 -0
- package/dist/engine/boot-validator/handler-field-references-rules.js +22 -0
- package/dist/engine/boot-validator/handler-field-references.js +14 -6
- package/dist/engine/boot-validator/list-bulk-actions-rules.d.ts +42 -0
- package/dist/engine/boot-validator/list-bulk-actions-rules.js +42 -0
- package/dist/engine/boot-validator/list-bulk-actions.js +29 -11
- package/dist/engine/boot-validator/list-column-layout.js +22 -10
- package/dist/engine/boot-validator/list-query-rules.d.ts +107 -0
- package/dist/engine/boot-validator/list-query-rules.js +107 -0
- package/dist/engine/boot-validator/list-table-aggregate-options.js +43 -15
- package/dist/engine/boot-validator/list-table-aggregate-query.js +7 -3
- package/dist/engine/boot-validator/list-table-aggregate-rules.d.ts +82 -0
- package/dist/engine/boot-validator/list-table-aggregate-rules.js +82 -0
- package/dist/engine/boot-validator/list-table-export-options.js +19 -7
- package/dist/engine/boot-validator/list-table-options.js +33 -13
- package/dist/engine/boot-validator/list-table-reject-unknown-keys.js +6 -2
- package/dist/engine/boot-validator/list-table-rules.d.ts +132 -0
- package/dist/engine/boot-validator/list-table-rules.js +132 -0
- package/dist/engine/boot-validator/nav-rules.d.ts +42 -0
- package/dist/engine/boot-validator/nav-rules.js +42 -0
- package/dist/engine/boot-validator/nav.js +29 -15
- package/dist/engine/boot-validator/ownership-rules.d.ts +27 -0
- package/dist/engine/boot-validator/ownership-rules.js +27 -0
- package/dist/engine/boot-validator/ownership.js +28 -12
- package/dist/engine/boot-validator/parent-ref.js +32 -20
- package/dist/engine/boot-validator/projection-list-screens.js +76 -35
- package/dist/engine/boot-validator/query-output-columns.js +9 -3
- package/dist/engine/boot-validator/query-refs.js +20 -10
- package/dist/engine/boot-validator/query-rules.d.ts +37 -0
- package/dist/engine/boot-validator/query-rules.js +37 -0
- package/dist/engine/boot-validator/read-gate.js +14 -10
- package/dist/engine/boot-validator/record-owned.js +5 -1
- package/dist/engine/boot-validator/related-list-table-rules.js +17 -7
- package/dist/engine/boot-validator/screens-actions-rules.d.ts +185 -0
- package/dist/engine/boot-validator/screens-actions-rules.js +185 -0
- package/dist/engine/boot-validator/screens-actions.js +169 -93
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.d.ts +87 -0
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.js +87 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.d.ts +227 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.js +227 -0
- package/dist/engine/boot-validator/screens-dashboard.js +222 -85
- package/dist/engine/boot-validator/screens-forms-layout-rules.d.ts +122 -0
- package/dist/engine/boot-validator/screens-forms-layout-rules.js +122 -0
- package/dist/engine/boot-validator/screens-forms-rules.d.ts +152 -0
- package/dist/engine/boot-validator/screens-forms-rules.js +152 -0
- package/dist/engine/boot-validator/screens-forms.js +240 -112
- package/dist/engine/boot-validator/screens-kind-config-edit.js +72 -16
- package/dist/engine/boot-validator/screens-kind-entity-edit.js +15 -7
- package/dist/engine/boot-validator/screens-kind-entity-list.js +57 -32
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.d.ts +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.js +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail.js +85 -47
- package/dist/engine/boot-validator/screens-kind-rules.d.ts +137 -0
- package/dist/engine/boot-validator/screens-kind-rules.js +137 -0
- package/dist/engine/boot-validator/screens-kind-secret-mint.js +18 -8
- package/dist/engine/boot-validator/screens-kind-shared.js +40 -22
- package/dist/engine/boot-validator/screens-lists-rules.d.ts +82 -0
- package/dist/engine/boot-validator/screens-lists-rules.js +82 -0
- package/dist/engine/boot-validator/screens-lists.js +90 -45
- package/dist/engine/boot-validator/screens-rules.d.ts +57 -0
- package/dist/engine/boot-validator/screens-rules.js +57 -0
- package/dist/engine/boot-validator/screens.js +17 -12
- package/dist/engine/boot-validator/transaction-phases.js +12 -4
- package/dist/engine/boot-validator/transfer-graph.js +17 -11
- package/dist/engine/boot-validator/url-prefill-fields.js +9 -5
- package/dist/engine/boot-validator/workspaces.js +11 -5
- package/dist/engine/boot-validator/zod-shape.js +6 -2
- package/dist/engine/entity-handlers.d.ts +1 -0
- package/dist/engine/entity-handlers.js +3 -0
- package/dist/engine/entity-list-runtime-options.js +23 -29
- package/dist/engine/extensions/tenant-data.d.ts +3 -0
- package/dist/engine/factories.d.ts +1 -1
- package/dist/engine/factories.js +4 -3
- package/dist/engine/index.d.ts +4 -2
- package/dist/engine/index.js +2 -0
- package/dist/engine/project-app-schema-for-roles.js +29 -14
- package/dist/engine/registry-facade.js +7 -8
- package/dist/engine/registry-validate.js +5 -1
- package/dist/engine/soft-delete-cleanup.d.ts +1 -0
- package/dist/engine/soft-delete-cleanup.js +5 -0
- package/dist/engine/steps/describe-workflow-step-error.js +1 -0
- package/dist/es-ops/operations-schema.d.ts +1 -1
- package/dist/event-store/ensure-dead-letter-table.d.ts +2 -0
- package/dist/event-store/ensure-dead-letter-table.js +10 -0
- package/dist/event-store/rebuild-dead-letter.js +2 -8
- package/dist/event-store/upcaster-dead-letter.js +2 -8
- package/dist/files/copy-pending-file-refs-to-tenant.d.ts +28 -0
- package/dist/files/copy-pending-file-refs-to-tenant.js +87 -0
- package/dist/files/file-routes.d.ts +1 -1
- package/dist/files/file-routes.js +21 -6
- package/dist/files/index.d.ts +2 -0
- package/dist/files/index.js +1 -0
- package/dist/i18n/required-surface-keys.js +1 -0
- package/dist/jobs/job-runner-helpers.js +1 -0
- package/dist/jobs/job-runner.js +7 -12
- package/dist/logging/utils.js +30 -60
- package/dist/observability/metrics-handle.js +10 -21
- package/dist/pipeline/cascade-handler.js +7 -6
- package/dist/pipeline/dispatch-query.js +18 -5
- package/dist/pipeline/dispatch-shared.d.ts +1 -0
- package/dist/pipeline/dispatch-shared.js +3 -1
- package/dist/pipeline/dispatch-write.js +10 -12
- package/dist/pipeline/dispatcher.js +12 -14
- package/dist/pipeline/event-dispatcher-admin.js +9 -8
- package/dist/pipeline/event-dispatcher-delivery.d.ts +19 -1
- package/dist/pipeline/event-dispatcher-delivery.js +121 -29
- package/dist/pipeline/event-dispatcher.d.ts +5 -1
- package/dist/pipeline/event-dispatcher.js +28 -21
- package/dist/pipeline/handler-context-phase-denials.js +2 -0
- package/dist/pipeline/multi-stream-apply-context.js +12 -20
- package/dist/pipeline/system-hooks.d.ts +4 -1
- package/dist/pipeline/system-hooks.js +79 -6
- package/dist/pipeline/system-identity-switch.js +10 -7
- package/dist/search/build-search-snippet.d.ts +9 -0
- package/dist/search/build-search-snippet.js +40 -0
- package/dist/search/index.d.ts +1 -0
- package/dist/search/index.js +1 -0
- package/dist/search/meilisearch-adapter.d.ts +32 -0
- package/dist/search/meilisearch-adapter.js +153 -56
- package/dist/search/meilisearch-chunking.d.ts +21 -0
- package/dist/search/meilisearch-chunking.js +96 -0
- package/dist/search/reindex-entity.d.ts +1 -0
- package/dist/search/reindex-entity.js +97 -52
- package/dist/ui-types/index.d.ts +1 -1
- package/dist/upgrade-cli.js +3 -14
- package/package.json +3 -3
- package/src/changes.json +73 -0
|
@@ -121,15 +121,20 @@ async function resolveSearchOperationsByTenant(events, ctx, registry) {
|
|
|
121
121
|
const op = await resolveSearchOperation(event, ctx, registry);
|
|
122
122
|
if (!op)
|
|
123
123
|
continue;
|
|
124
|
-
|
|
124
|
+
const key = searchOperationKey(op);
|
|
125
|
+
// Re-insert so the map order follows the winning event: per-tenant ops stay in event order.
|
|
126
|
+
byKey.delete(key);
|
|
127
|
+
byKey.set(key, { op, sourceEventId: event.id });
|
|
125
128
|
}
|
|
126
129
|
const perTenant = new Map();
|
|
127
|
-
for (const
|
|
130
|
+
for (const sourced of byKey.values()) {
|
|
131
|
+
const op = sourced.op;
|
|
128
132
|
let bucket = perTenant.get(op.tenantId);
|
|
129
133
|
if (!bucket) {
|
|
130
|
-
bucket = { toIndex: [], toRemove: [] };
|
|
134
|
+
bucket = { toIndex: [], toRemove: [], sourcedOps: [] };
|
|
131
135
|
perTenant.set(op.tenantId, bucket);
|
|
132
136
|
}
|
|
137
|
+
bucket.sourcedOps.push(sourced);
|
|
133
138
|
if (op.kind === "index")
|
|
134
139
|
bucket.toIndex.push(op.doc);
|
|
135
140
|
else
|
|
@@ -163,7 +168,60 @@ async function flushSearchOperationsForTenant(searchAdapter, tenantId, bucket) {
|
|
|
163
168
|
}
|
|
164
169
|
}
|
|
165
170
|
}
|
|
166
|
-
|
|
171
|
+
// A failed tenant batch is replayed op by op. After the collapse every op targets another
|
|
172
|
+
// entity, so only the ops that fail are held and the rest land, removes included; the first
|
|
173
|
+
// held event is the earliest failing one, which skipPoisonEvent's lastFailedEventId names.
|
|
174
|
+
async function flushTenantHoldingFailedOperations(searchAdapter, tenantId, bucket) {
|
|
175
|
+
try {
|
|
176
|
+
await flushSearchOperationsForTenant(searchAdapter, tenantId, bucket);
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
catch (batchError) {
|
|
180
|
+
// skip: a single op would only repeat the request that just failed.
|
|
181
|
+
if (bucket.sourcedOps.length === 1) {
|
|
182
|
+
return { eventIds: bucket.sourcedOps.map((s) => s.sourceEventId), error: batchError };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const heldEventIds = [];
|
|
186
|
+
let firstError;
|
|
187
|
+
for (const sourced of bucket.sourcedOps) {
|
|
188
|
+
try {
|
|
189
|
+
await applySearchOperation(searchAdapter, sourced.op);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (heldEventIds.length === 0)
|
|
193
|
+
firstError = error;
|
|
194
|
+
heldEventIds.push(sourced.sourceEventId);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return heldEventIds.length === 0 ? null : { eventIds: heldEventIds, error: firstError };
|
|
198
|
+
}
|
|
199
|
+
// Spacing for a tenant whose index keeps rejecting writes: its held events come back on every
|
|
200
|
+
// dispatcher turn, but Meilisearch is only asked again once the delay has passed.
|
|
201
|
+
const TENANT_RETRY_BASE_DELAY_MS = 1_000;
|
|
202
|
+
const TENANT_RETRY_MAX_DELAY_MS = 60_000;
|
|
203
|
+
// Ten failures span about four minutes of backoff: enough for Meilisearch to restart, short
|
|
204
|
+
// enough that a broken index or a poison document reaches ops as an error.
|
|
205
|
+
const TENANT_HOLD_ESCALATION_FAILURES = 10;
|
|
206
|
+
export function createSearchEventConsumer(searchAdapter, registry, options = {}) {
|
|
207
|
+
const now = options.now ?? Date.now;
|
|
208
|
+
// Process memory only: after a restart a held tenant is simply retried at once.
|
|
209
|
+
const tenantRetries = new Map();
|
|
210
|
+
function recordTenantFailure(tenantId, held) {
|
|
211
|
+
const previous = tenantRetries.get(tenantId);
|
|
212
|
+
const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + 1;
|
|
213
|
+
const delayMs = Math.min(TENANT_RETRY_BASE_DELAY_MS * 2 ** (consecutiveFailures - 1), TENANT_RETRY_MAX_DELAY_MS);
|
|
214
|
+
const message = held.error instanceof Error ? held.error.message : String(held.error);
|
|
215
|
+
const reason = consecutiveFailures >= TENANT_HOLD_ESCALATION_FAILURES
|
|
216
|
+
? `search index of tenant ${tenantId} rejected writes ${consecutiveFailures} times in a row, event ${held.eventIds[0]} stays held until the index is fixed or the event is skipped with skipPoisonEvent: ${message}`
|
|
217
|
+
: `search index of tenant ${tenantId} rejected the write, held for retry: ${message}`;
|
|
218
|
+
tenantRetries.set(tenantId, { consecutiveFailures, retryAtMs: now() + delayMs, reason });
|
|
219
|
+
if (!previous)
|
|
220
|
+
log.warn(reason);
|
|
221
|
+
if (consecutiveFailures === TENANT_HOLD_ESCALATION_FAILURES)
|
|
222
|
+
log.error(reason);
|
|
223
|
+
return reason;
|
|
224
|
+
}
|
|
167
225
|
return {
|
|
168
226
|
name: SEARCH_CONSUMER_NAME,
|
|
169
227
|
// ponytail: count-based maxAttempts, not real backoff — retries hammer
|
|
@@ -178,10 +236,25 @@ export function createSearchEventConsumer(searchAdapter, registry) {
|
|
|
178
236
|
if (op)
|
|
179
237
|
await applySearchOperation(searchAdapter, op);
|
|
180
238
|
},
|
|
181
|
-
|
|
239
|
+
// One tenant's failing index must not stall the others: its failing ops are held for a later
|
|
240
|
+
// turn, and while it backs off all of its ops wait, so per-entity order survives.
|
|
241
|
+
batchHandler: async (events, ctx, turn) => {
|
|
182
242
|
const perTenant = await resolveSearchOperationsByTenant(events, ctx, registry);
|
|
183
243
|
for (const [tenantId, bucket] of perTenant) {
|
|
184
|
-
|
|
244
|
+
const retry = tenantRetries.get(tenantId);
|
|
245
|
+
if (retry && retry.retryAtMs > now()) {
|
|
246
|
+
turn.holdForRetry(bucket.sourcedOps.map((s) => s.sourceEventId), retry.reason);
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
const held = await flushTenantHoldingFailedOperations(searchAdapter, tenantId, bucket);
|
|
250
|
+
if (held) {
|
|
251
|
+
turn.holdForRetry(held.eventIds, recordTenantFailure(tenantId, held));
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (retry) {
|
|
255
|
+
tenantRetries.delete(tenantId);
|
|
256
|
+
log.info(`search index of tenant ${tenantId} accepts writes again`);
|
|
257
|
+
}
|
|
185
258
|
}
|
|
186
259
|
},
|
|
187
260
|
};
|
|
@@ -32,6 +32,7 @@ export function isIdentitySwitchAllowed(caller, asUser, hasGrant) {
|
|
|
32
32
|
return false;
|
|
33
33
|
return isSelfDelegation(caller, asUser);
|
|
34
34
|
}
|
|
35
|
+
// guard:dup-ok error factory; only message and reason differ from unsafeAllTenantsDenied
|
|
35
36
|
export function systemIdentitySwitchDenied(callerLabel) {
|
|
36
37
|
return new AccessDeniedError({
|
|
37
38
|
message: `${callerLabel} may not switch identity to SYSTEM — declare r.systemScope() or ` +
|
|
@@ -69,18 +70,18 @@ function reportGrantedSwitch(caller, asUser, hasGrant, audit) {
|
|
|
69
70
|
audit.report("identity-switch", audit.reason, { id: asUser.id, tenantId: asUser.tenantId });
|
|
70
71
|
}
|
|
71
72
|
export function createGatedIdentitySwitch(callerLabel, caller, hasGrant, ungated, audit) {
|
|
72
|
-
const
|
|
73
|
+
const assertSwitchAllowed = (asUser) => {
|
|
73
74
|
if (!isIdentitySwitchAllowed(caller, asUser, hasGrant)) {
|
|
74
75
|
throw identitySwitchDenied(callerLabel, asUser);
|
|
75
76
|
}
|
|
76
77
|
reportGrantedSwitch(caller, asUser, hasGrant, audit);
|
|
78
|
+
};
|
|
79
|
+
const queryAs = async (asUser, qn, payload) => {
|
|
80
|
+
assertSwitchAllowed(asUser);
|
|
77
81
|
return ungated.queryAs(asUser, qn, payload);
|
|
78
82
|
};
|
|
79
83
|
const writeAs = async (asUser, qn, payload) => {
|
|
80
|
-
|
|
81
|
-
throw identitySwitchDenied(callerLabel, asUser);
|
|
82
|
-
}
|
|
83
|
-
reportGrantedSwitch(caller, asUser, hasGrant, audit);
|
|
84
|
+
assertSwitchAllowed(asUser);
|
|
84
85
|
return ungated.writeAs(asUser, qn, payload);
|
|
85
86
|
};
|
|
86
87
|
const gated = { queryAs, writeAs };
|
|
@@ -137,11 +138,13 @@ function readFunctionMember(namespace, key) {
|
|
|
137
138
|
return typeof value === "function" ? value : undefined; // @cast-boundary engine-bridge — checked via typeof above
|
|
138
139
|
}
|
|
139
140
|
function unavailableIdentitySwitchFn(callerLabel, kind) {
|
|
140
|
-
return
|
|
141
|
+
return (
|
|
142
|
+
// guard:dup-ok a throw-only stub; the message is its whole content
|
|
143
|
+
async () => {
|
|
141
144
|
throw new InternalError({
|
|
142
145
|
message: `${callerLabel}: this context has no ${kind} to switch identity with.`,
|
|
143
146
|
});
|
|
144
|
-
};
|
|
147
|
+
});
|
|
145
148
|
}
|
|
146
149
|
function deniedResolveActiveMembership(callerLabel) {
|
|
147
150
|
return async () => {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export type SearchSnippet = {
|
|
2
|
+
readonly before: string;
|
|
3
|
+
readonly hit: string;
|
|
4
|
+
readonly after: string;
|
|
5
|
+
};
|
|
6
|
+
/** The first occurrence of `term` in `text` with about `radius` characters of context on each side. */
|
|
7
|
+
export declare function buildSearchSnippet(text: string, term: string, { radius }?: {
|
|
8
|
+
readonly radius?: number;
|
|
9
|
+
}): SearchSnippet | null;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const DEFAULT_RADIUS = 60;
|
|
2
|
+
const ELLIPSIS = "…";
|
|
3
|
+
const WHITESPACE = /\s/;
|
|
4
|
+
// Cuts at a word boundary so the snippet never starts or ends in the middle of a word.
|
|
5
|
+
function trimStartToWord(text, from) {
|
|
6
|
+
let start = from;
|
|
7
|
+
while (start < text.length && !WHITESPACE.test(text.charAt(start - 1)))
|
|
8
|
+
start++;
|
|
9
|
+
return start;
|
|
10
|
+
}
|
|
11
|
+
function trimEndToWord(text, to) {
|
|
12
|
+
let end = to;
|
|
13
|
+
while (end > 0 && !WHITESPACE.test(text.charAt(end)))
|
|
14
|
+
end--;
|
|
15
|
+
return end;
|
|
16
|
+
}
|
|
17
|
+
/** The first occurrence of `term` in `text` with about `radius` characters of context on each side. */
|
|
18
|
+
export function buildSearchSnippet(text, term, { radius = DEFAULT_RADIUS } = {}) {
|
|
19
|
+
const needle = term.trim();
|
|
20
|
+
if (needle === "")
|
|
21
|
+
return null;
|
|
22
|
+
// A case-insensitive regex finds the hit in the original text, so offsets stay valid even
|
|
23
|
+
// where lower-casing would change the string length.
|
|
24
|
+
const match = new RegExp(needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "iu").exec(text);
|
|
25
|
+
if (match === null)
|
|
26
|
+
return null;
|
|
27
|
+
const hitStart = match.index;
|
|
28
|
+
const hitEnd = hitStart + match[0].length;
|
|
29
|
+
const rawBeforeStart = hitStart - radius;
|
|
30
|
+
const beforeStart = rawBeforeStart <= 0 ? 0 : Math.min(trimStartToWord(text, rawBeforeStart), hitStart);
|
|
31
|
+
const rawAfterEnd = hitEnd + radius;
|
|
32
|
+
const afterEnd = rawAfterEnd >= text.length ? text.length : Math.max(trimEndToWord(text, rawAfterEnd), hitEnd);
|
|
33
|
+
const before = text.slice(beforeStart, hitStart);
|
|
34
|
+
const after = text.slice(hitEnd, afterEnd);
|
|
35
|
+
return {
|
|
36
|
+
before: beforeStart > 0 ? `${ELLIPSIS}${before}` : before,
|
|
37
|
+
hit: text.slice(hitStart, hitEnd),
|
|
38
|
+
after: afterEnd < text.length ? `${after}${ELLIPSIS}` : after,
|
|
39
|
+
};
|
|
40
|
+
}
|
package/dist/search/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { buildSearchSnippet, type SearchSnippet } from "./build-search-snippet.ts";
|
|
1
2
|
export { collectSearchableFieldNames } from "./configure-tenant-index.ts";
|
|
2
3
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.ts";
|
|
3
4
|
export { isEntitySearchable } from "./is-entity-searchable.ts";
|
package/dist/search/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// `@cosmicdrift/kumiko-framework/search/meilisearch`. This keeps the main barrel from loading a
|
|
3
3
|
// Meilisearch client when consumers only use SearchAdapter types, so apps that do not use
|
|
4
4
|
// Meilisearch do not pull in the client code.
|
|
5
|
+
export { buildSearchSnippet } from "./build-search-snippet.js";
|
|
5
6
|
export { collectSearchableFieldNames } from "./configure-tenant-index.js";
|
|
6
7
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.js";
|
|
7
8
|
export { isEntitySearchable } from "./is-entity-searchable.js";
|
|
@@ -1,3 +1,35 @@
|
|
|
1
1
|
import { type MeilisearchAdapterOptions } from "./meilisearch-adapter-internals.ts";
|
|
2
2
|
import type { SearchAdapter } from "./types.ts";
|
|
3
|
+
/** A drop that finds its index already deleted by a concurrent drop (index_not_found, or the
|
|
4
|
+
* vanished index files) has reached its end state. Reads a task error or an API error's cause. */
|
|
5
|
+
export declare function isGoneIndexError(error: unknown): boolean;
|
|
6
|
+
/** The part of the Meilisearch client an index listing needs. */
|
|
7
|
+
export type IndexListingClient = {
|
|
8
|
+
getIndexes(query: {
|
|
9
|
+
limit: number;
|
|
10
|
+
offset: number;
|
|
11
|
+
}): Promise<{
|
|
12
|
+
results: readonly {
|
|
13
|
+
uid: string;
|
|
14
|
+
}[];
|
|
15
|
+
total: number;
|
|
16
|
+
}>;
|
|
17
|
+
tasks: {
|
|
18
|
+
getTasks(query: {
|
|
19
|
+
types: "indexDeletion"[];
|
|
20
|
+
statuses: ("enqueued" | "processing")[];
|
|
21
|
+
limit: number;
|
|
22
|
+
}): Promise<{
|
|
23
|
+
results: readonly {
|
|
24
|
+
uid: number;
|
|
25
|
+
}[];
|
|
26
|
+
}>;
|
|
27
|
+
waitForTasks(taskUids: number[]): Promise<unknown>;
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
/** Every index uid on the instance, paged. Several apps or test runs can share one Meilisearch:
|
|
31
|
+
* a listing that meets another client's index deletion fails as a whole (500 "os error 2"),
|
|
32
|
+
* so it waits for the deletions in flight and reads once more. A deletion enqueued between
|
|
33
|
+
* that wait and the second read still fails the call. */
|
|
34
|
+
export declare function listIndexUids(client: IndexListingClient): Promise<readonly string[]>;
|
|
3
35
|
export declare function createMeilisearchAdapter(options: MeilisearchAdapterOptions): SearchAdapter;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { errorCodeOf } from "@cosmicdrift/kumiko-types/type-guards";
|
|
1
|
+
import { errorCodeOf, propertyOf } from "@cosmicdrift/kumiko-types/type-guards";
|
|
2
2
|
import { importOptionalPeer } from "../utils/import-optional-peer.js";
|
|
3
|
-
import {
|
|
3
|
+
import { meilisearchTenantIndex, } from "./meilisearch-adapter-internals.js";
|
|
4
|
+
import { buildPieceDocuments, entityPiecesFilter, filterTypeClause, stalePiecesFilter, } from "./meilisearch-chunking.js";
|
|
4
5
|
import { MEILISEARCH_PEER } from "./meilisearch-peer.js";
|
|
5
6
|
// meilisearch is an optional peer; a static import would fail with the runtime's bare
|
|
6
7
|
// "Cannot find package" error instead of naming the peer and how to install it.
|
|
@@ -15,7 +16,9 @@ async function awaitSucceededTask(enqueued, toleratedErrorCodes = []) {
|
|
|
15
16
|
const task = await enqueued.waitTask();
|
|
16
17
|
const tolerated = task.error ? toleratedErrorCodes.includes(task.error.code) : false;
|
|
17
18
|
if (task.status !== "succeeded" && !tolerated) {
|
|
18
|
-
|
|
19
|
+
// Only the code: Meili's message can embed the whole rejected document (missing_document_id),
|
|
20
|
+
// and this error ends up in the consumer's lastError and the logs. The task uid leads to it.
|
|
21
|
+
const detail = task.error ? task.error.code : "no error code";
|
|
19
22
|
throw new Error(`Meilisearch task ${task.uid} (${task.type}) ended with status "${task.status}": ${detail}`);
|
|
20
23
|
}
|
|
21
24
|
return task;
|
|
@@ -25,11 +28,21 @@ async function awaitSucceededTask(enqueued, toleratedErrorCodes = []) {
|
|
|
25
28
|
// Without this, a delete event or a DSGVO subject purge for such a tenant
|
|
26
29
|
// would fail forever.
|
|
27
30
|
const REMOVE_TOLERATED_ERROR_CODES = [ErrorStatusCode.INDEX_NOT_FOUND];
|
|
28
|
-
const INDEX_LIST_PAGE_SIZE =
|
|
31
|
+
const INDEX_LIST_PAGE_SIZE = 1000;
|
|
29
32
|
// What an unconfigured Meilisearch index reports as searchableAttributes.
|
|
30
33
|
const MEILI_ALL_ATTRIBUTES = "*";
|
|
31
|
-
|
|
34
|
+
// _entityId and _chunk back the piece filters (stale-piece cleanup, remove); distinct on
|
|
35
|
+
// _entityId makes an entity with several matching pieces appear once in the results.
|
|
36
|
+
const DISTINCT_ATTRIBUTE = "_entityId";
|
|
37
|
+
const DEFAULT_FILTERABLE_ATTRIBUTES = [
|
|
38
|
+
"_type",
|
|
39
|
+
"_weight",
|
|
40
|
+
DISTINCT_ATTRIBUTE,
|
|
41
|
+
"_chunk",
|
|
42
|
+
];
|
|
32
43
|
const DEFAULT_SORTABLE_ATTRIBUTES = ["_weight"];
|
|
44
|
+
// Filters per deleteDocuments call: one task per call, and the filter string stays short.
|
|
45
|
+
const DELETE_FILTER_BATCH = 100;
|
|
33
46
|
function defaultSearchableAttributes(config) {
|
|
34
47
|
return config.rankingFields ?? config.searchableFields;
|
|
35
48
|
}
|
|
@@ -41,19 +54,70 @@ function isSubsetOf(stored, wanted) {
|
|
|
41
54
|
// older default that gained fields since; anything with a foreign attribute is
|
|
42
55
|
// an explicit configure() and stays untouched. Known limit: an explicit configure() that
|
|
43
56
|
// narrowed to a strict subset of the default is indistinguishable and gets widened again.
|
|
44
|
-
function isOutdatedDefault(stored, wanted) {
|
|
57
|
+
function isOutdatedDefault(stored, wanted, distinctMissing) {
|
|
45
58
|
const pairs = [
|
|
46
59
|
[stored.searchable, wanted.searchable],
|
|
47
60
|
[stored.filterable, wanted.filterable],
|
|
48
61
|
[stored.sortable, wanted.sortable],
|
|
49
62
|
];
|
|
50
63
|
return (pairs.every(([have, want]) => isSubsetOf(have, want)) &&
|
|
51
|
-
|
|
64
|
+
(distinctMissing ||
|
|
65
|
+
pairs.some(([have, want]) => want.some((attribute) => !have.includes(attribute)))));
|
|
52
66
|
}
|
|
53
67
|
function isIndexNotFound(err) {
|
|
54
68
|
const cause = typeof err === "object" && err !== null && "cause" in err ? err.cause : null;
|
|
55
69
|
return errorCodeOf(cause) === ErrorStatusCode.INDEX_NOT_FOUND;
|
|
56
70
|
}
|
|
71
|
+
// Meili reports ENOENT as "No such file or directory (os error 2)" when an index's files vanish
|
|
72
|
+
// under a running task.
|
|
73
|
+
const MISSING_INDEX_FILES = "os error 2";
|
|
74
|
+
/** A drop that finds its index already deleted by a concurrent drop (index_not_found, or the
|
|
75
|
+
* vanished index files) has reached its end state. Reads a task error or an API error's cause. */
|
|
76
|
+
export function isGoneIndexError(error) {
|
|
77
|
+
const message = propertyOf(error, "message");
|
|
78
|
+
return (errorCodeOf(error) === ErrorStatusCode.INDEX_NOT_FOUND ||
|
|
79
|
+
(typeof message === "string" && message.includes(MISSING_INDEX_FILES)));
|
|
80
|
+
}
|
|
81
|
+
/** Every index uid on the instance, paged. Several apps or test runs can share one Meilisearch:
|
|
82
|
+
* a listing that meets another client's index deletion fails as a whole (500 "os error 2"),
|
|
83
|
+
* so it waits for the deletions in flight and reads once more. A deletion enqueued between
|
|
84
|
+
* that wait and the second read still fails the call. */
|
|
85
|
+
export async function listIndexUids(client) {
|
|
86
|
+
try {
|
|
87
|
+
return await readIndexUids(client);
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
if (!isGoneIndexError(propertyOf(err, "cause")))
|
|
91
|
+
throw err;
|
|
92
|
+
}
|
|
93
|
+
const pending = await client.tasks.getTasks({
|
|
94
|
+
types: ["indexDeletion"],
|
|
95
|
+
statuses: ["enqueued", "processing"],
|
|
96
|
+
limit: INDEX_LIST_PAGE_SIZE,
|
|
97
|
+
});
|
|
98
|
+
await client.tasks.waitForTasks(pending.results.map((task) => task.uid));
|
|
99
|
+
return readIndexUids(client);
|
|
100
|
+
}
|
|
101
|
+
async function readIndexUids(client) {
|
|
102
|
+
const uids = [];
|
|
103
|
+
for (let offset = 0;; offset += INDEX_LIST_PAGE_SIZE) {
|
|
104
|
+
const page = await client.getIndexes({ limit: INDEX_LIST_PAGE_SIZE, offset });
|
|
105
|
+
uids.push(...page.results.map((index) => index.uid));
|
|
106
|
+
if (page.results.length === 0 || offset + page.results.length >= page.total)
|
|
107
|
+
return uids;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Enqueued together so Meili batches the deletions: every deletion in flight fails other
|
|
111
|
+
// clients' index listings, and one batch keeps that window short.
|
|
112
|
+
async function dropIndexesIfPresent(client, uids) {
|
|
113
|
+
const enqueued = await Promise.all(uids.map((uid) => client.deleteIndex(uid)));
|
|
114
|
+
const tasks = await client.tasks.waitForTasks(enqueued);
|
|
115
|
+
// A concurrent drop that got there first leaves the index gone, which is the goal.
|
|
116
|
+
const failed = tasks.find((task) => task.status !== "succeeded" && !isGoneIndexError(task.error));
|
|
117
|
+
if (failed) {
|
|
118
|
+
throw new Error(`Meilisearch task ${failed.uid} (${failed.type}) ended with status "${failed.status}": ${failed.error ? failed.error.code : "no error code"}`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
57
121
|
// Exported for unit tests (index-name / primary-key shape) without a live Meili.
|
|
58
122
|
// Meilisearch primary-key-ids: alphanumerics, `-`, `_`. UUIDs contain `-` —
|
|
59
123
|
// legal. Replace anything else just in case callers pass unexpected shapes.
|
|
@@ -78,6 +142,42 @@ export function createMeilisearchAdapter(options) {
|
|
|
78
142
|
await awaitSucceededTask(index.updateSearchableAttributes([...defaultSearchableAttributes(config)]));
|
|
79
143
|
await awaitSucceededTask(index.updateFilterableAttributes([...DEFAULT_FILTERABLE_ATTRIBUTES]));
|
|
80
144
|
await awaitSucceededTask(index.updateSortableAttributes([...DEFAULT_SORTABLE_ATTRIBUTES]));
|
|
145
|
+
await awaitSucceededTask(index.updateDistinctAttribute(DISTINCT_ATTRIBUTE));
|
|
146
|
+
}
|
|
147
|
+
// Filter deletes need the filterable attributes, but a tenant whose index was never created has
|
|
148
|
+
// nothing to delete and must not get an empty index out of a delete.
|
|
149
|
+
async function ensureConfiguredIfIndexExists(tenantId) {
|
|
150
|
+
if (!configuredTenants.has(tenantId)) {
|
|
151
|
+
try {
|
|
152
|
+
await client.getIndex(meilisearchTenantIndex(prefix, tenantId));
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
if (isIndexNotFound(err))
|
|
156
|
+
return false;
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
await ensureConfigured(tenantId);
|
|
161
|
+
return true;
|
|
162
|
+
}
|
|
163
|
+
async function deleteByFilters(tenantId, filters) {
|
|
164
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
165
|
+
await writeForgettingConfigOnFailure(tenantId, async () => {
|
|
166
|
+
for (let offset = 0; offset < filters.length; offset += DELETE_FILTER_BATCH) {
|
|
167
|
+
const filter = filters.slice(offset, offset + DELETE_FILTER_BATCH).join(" OR ");
|
|
168
|
+
await awaitSucceededTask(index.deleteDocuments({ filter }), REMOVE_TOLERATED_ERROR_CODES);
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
// All pieces first, then the leftovers of an earlier, longer version: a failure in between
|
|
173
|
+
// leaves a document that is too long, never a hole.
|
|
174
|
+
async function writeDocuments(tenantId, docs) {
|
|
175
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
176
|
+
const pieces = docs.flatMap(buildPieceDocuments);
|
|
177
|
+
await writeForgettingConfigOnFailure(tenantId, async () => {
|
|
178
|
+
await awaitSucceededTask(index.addDocuments(pieces, { primaryKey: "_id" }));
|
|
179
|
+
});
|
|
180
|
+
await deleteByFilters(tenantId, docs.map((doc) => stalePiecesFilter(doc.entityType, doc.entityId, buildPieceDocuments(doc).length)));
|
|
81
181
|
}
|
|
82
182
|
// Lazily configures a tenant's index off the default config on its first
|
|
83
183
|
// access, so callers that never call `configure` still get a filterable/
|
|
@@ -89,7 +189,9 @@ export function createMeilisearchAdapter(options) {
|
|
|
89
189
|
// configure() must not be overwritten by the registry default, so the default is applied only
|
|
90
190
|
// to an index that is missing, carries Meilisearch's factory settings, or holds an older
|
|
91
191
|
// default that is a strict subset of the current one (fields were added by a deploy).
|
|
92
|
-
|
|
192
|
+
// A kept index still gets the default filterable attributes merged in: remove, purge and the
|
|
193
|
+
// stale-piece delete filter on them, and an index without them rejects every delete.
|
|
194
|
+
async function configActionFor(tenantId, config) {
|
|
93
195
|
try {
|
|
94
196
|
const settings = await client.index(meilisearchTenantIndex(prefix, tenantId)).getSettings();
|
|
95
197
|
const searchable = settings.searchableAttributes ?? [MEILI_ALL_ATTRIBUTES];
|
|
@@ -97,16 +199,22 @@ export function createMeilisearchAdapter(options) {
|
|
|
97
199
|
const factorySettings = searchable.length === 1 &&
|
|
98
200
|
searchable[0] === MEILI_ALL_ATTRIBUTES &&
|
|
99
201
|
!filterable.includes("_type");
|
|
100
|
-
|
|
202
|
+
const needsDefault = factorySettings ||
|
|
101
203
|
isOutdatedDefault({ searchable, filterable, sortable: settings.sortableAttributes ?? [] }, {
|
|
102
204
|
searchable: defaultSearchableAttributes(config),
|
|
103
205
|
filterable: DEFAULT_FILTERABLE_ATTRIBUTES,
|
|
104
206
|
sortable: DEFAULT_SORTABLE_ATTRIBUTES,
|
|
105
|
-
})
|
|
207
|
+
}, settings.distinctAttribute !== DISTINCT_ATTRIBUTE);
|
|
208
|
+
if (needsDefault)
|
|
209
|
+
return { kind: "apply-default" };
|
|
210
|
+
const missing = DEFAULT_FILTERABLE_ATTRIBUTES.filter((attribute) => !filterable.includes(attribute));
|
|
211
|
+
return missing.length === 0
|
|
212
|
+
? { kind: "keep" }
|
|
213
|
+
: { kind: "add-filterable", filterable: [...filterable, ...missing] };
|
|
106
214
|
}
|
|
107
215
|
catch (err) {
|
|
108
216
|
if (isIndexNotFound(err))
|
|
109
|
-
return
|
|
217
|
+
return { kind: "apply-default" };
|
|
110
218
|
throw err;
|
|
111
219
|
}
|
|
112
220
|
}
|
|
@@ -122,8 +230,14 @@ export function createMeilisearchAdapter(options) {
|
|
|
122
230
|
}
|
|
123
231
|
const config = defaultConfig;
|
|
124
232
|
await trackConfigure(tenantId, async () => {
|
|
125
|
-
|
|
233
|
+
const action = await configActionFor(tenantId, config);
|
|
234
|
+
if (action.kind === "apply-default")
|
|
126
235
|
await applyConfig(tenantId, config);
|
|
236
|
+
if (action.kind === "add-filterable") {
|
|
237
|
+
await awaitSucceededTask(client
|
|
238
|
+
.index(meilisearchTenantIndex(prefix, tenantId))
|
|
239
|
+
.updateFilterableAttributes(action.filterable));
|
|
240
|
+
}
|
|
127
241
|
});
|
|
128
242
|
}
|
|
129
243
|
async function trackConfigure(tenantId, run) {
|
|
@@ -139,6 +253,17 @@ export function createMeilisearchAdapter(options) {
|
|
|
139
253
|
throw err;
|
|
140
254
|
}
|
|
141
255
|
}
|
|
256
|
+
// A rejected write can mean another writer recreated or reconfigured the index since this
|
|
257
|
+
// process configured it; forgetting the tenant makes the next write re-check its settings.
|
|
258
|
+
async function writeForgettingConfigOnFailure(tenantId, write) {
|
|
259
|
+
try {
|
|
260
|
+
await write();
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
configuredTenants.delete(tenantId);
|
|
264
|
+
throw err;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
142
267
|
return {
|
|
143
268
|
setDefaultConfig(config) {
|
|
144
269
|
defaultConfig = config;
|
|
@@ -148,77 +273,47 @@ export function createMeilisearchAdapter(options) {
|
|
|
148
273
|
},
|
|
149
274
|
async index(tenantId, doc) {
|
|
150
275
|
await ensureConfigured(tenantId);
|
|
151
|
-
|
|
152
|
-
await awaitSucceededTask(index.addDocuments([
|
|
153
|
-
{
|
|
154
|
-
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
155
|
-
_type: doc.entityType,
|
|
156
|
-
_weight: doc.weight,
|
|
157
|
-
_entityId: doc.entityId,
|
|
158
|
-
...doc.fields,
|
|
159
|
-
},
|
|
160
|
-
], { primaryKey: "_id" }));
|
|
276
|
+
await writeDocuments(tenantId, [doc]);
|
|
161
277
|
},
|
|
162
278
|
async indexBatch(tenantId, docs) {
|
|
163
279
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
164
280
|
if (docs.length === 0)
|
|
165
281
|
return;
|
|
166
282
|
await ensureConfigured(tenantId);
|
|
167
|
-
|
|
168
|
-
const payload = docs.map((doc) => ({
|
|
169
|
-
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
170
|
-
_type: doc.entityType,
|
|
171
|
-
_weight: doc.weight,
|
|
172
|
-
_entityId: doc.entityId,
|
|
173
|
-
...doc.fields,
|
|
174
|
-
}));
|
|
175
|
-
// Single Meilisearch task covering all N docs. Meilisearch processes
|
|
176
|
-
// the payload server-side as one indexing job — waitTask blocks until
|
|
177
|
-
// that job is done, but it's one round-trip instead of N.
|
|
178
|
-
await awaitSucceededTask(index.addDocuments(payload, { primaryKey: "_id" }));
|
|
283
|
+
await writeDocuments(tenantId, docs);
|
|
179
284
|
},
|
|
180
285
|
async dropAllIndexes() {
|
|
181
286
|
// An empty prefix matches every index, which would wipe the whole Meili instance.
|
|
182
287
|
if (prefix === "") {
|
|
183
288
|
throw new Error("refusing to drop Meilisearch indexes with an empty prefix");
|
|
184
289
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
const page = await client.getIndexes({ limit: INDEX_LIST_PAGE_SIZE, offset });
|
|
188
|
-
for (const index of page.results) {
|
|
189
|
-
if (isOwnTenantIndex(index.uid))
|
|
190
|
-
uids.push(index.uid);
|
|
191
|
-
}
|
|
192
|
-
if (page.results.length === 0 || offset + page.results.length >= page.total)
|
|
193
|
-
break;
|
|
194
|
-
}
|
|
195
|
-
// Cleared even when a delete throws midway: otherwise ensureConfigured treats the
|
|
196
|
-
// already-dropped tenants as configured and the next index() recreates an
|
|
290
|
+
// Cleared even when the listing or a delete throws midway: otherwise ensureConfigured
|
|
291
|
+
// treats the already-dropped tenants as configured and the next index() recreates an
|
|
197
292
|
// unconfigured index ("not filterable" on filtered search).
|
|
198
293
|
try {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
294
|
+
const uids = (await listIndexUids(client)).filter(isOwnTenantIndex);
|
|
295
|
+
await dropIndexesIfPresent(client, uids);
|
|
296
|
+
return uids.length;
|
|
202
297
|
}
|
|
203
298
|
finally {
|
|
204
299
|
configuredTenants.clear();
|
|
205
300
|
}
|
|
206
|
-
return uids.length;
|
|
207
301
|
},
|
|
208
302
|
async removeBatch(tenantId, items) {
|
|
209
303
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
210
304
|
if (items.length === 0)
|
|
211
305
|
return;
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
306
|
+
// skip: the tenant has no index, so nothing is left to remove
|
|
307
|
+
if (!(await ensureConfiguredIfIndexExists(tenantId)))
|
|
308
|
+
return;
|
|
309
|
+
await deleteByFilters(tenantId, items.map((item) => entityPiecesFilter(item.entityType, item.entityId)));
|
|
215
310
|
},
|
|
216
311
|
async search(tenantId, query, options) {
|
|
217
312
|
await ensureConfigured(tenantId);
|
|
218
313
|
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
219
314
|
const filter = [];
|
|
220
315
|
if (options?.filterType) {
|
|
221
|
-
filter.push(
|
|
316
|
+
filter.push(filterTypeClause(options.filterType));
|
|
222
317
|
}
|
|
223
318
|
const searchParams = {
|
|
224
319
|
limit: options?.limit ?? 50,
|
|
@@ -233,8 +328,10 @@ export function createMeilisearchAdapter(options) {
|
|
|
233
328
|
}));
|
|
234
329
|
},
|
|
235
330
|
async remove(tenantId, entityType, entityId) {
|
|
236
|
-
|
|
237
|
-
|
|
331
|
+
// skip: the tenant has no index, so nothing is left to remove
|
|
332
|
+
if (!(await ensureConfiguredIfIndexExists(tenantId)))
|
|
333
|
+
return;
|
|
334
|
+
await deleteByFilters(tenantId, [entityPiecesFilter(entityType, entityId)]);
|
|
238
335
|
},
|
|
239
336
|
};
|
|
240
337
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { EntityId } from "../engine/types/identifiers.ts";
|
|
2
|
+
import type { SearchDocument } from "./types.ts";
|
|
3
|
+
export declare const CHUNK_MAX_CHARS = 8000;
|
|
4
|
+
export declare const CHUNK_OVERLAP_CHARS = 200;
|
|
5
|
+
/** Splits a long value into pieces of at most CHUNK_MAX_CHARS, cut at whitespace. */
|
|
6
|
+
export declare function splitLongText(value: string): string[];
|
|
7
|
+
export type MeilisearchPieceDocument = Record<string, unknown> & {
|
|
8
|
+
readonly _id: string;
|
|
9
|
+
};
|
|
10
|
+
export declare function pieceDocId(baseId: string, chunk: number): string;
|
|
11
|
+
/**
|
|
12
|
+
* One Meilisearch document per piece. Piece 0 keeps the id the entity always had (existing
|
|
13
|
+
* documents stay valid, no duplicates) and carries all short fields; piece n holds the n-th
|
|
14
|
+
* parts of the long fields.
|
|
15
|
+
*/
|
|
16
|
+
export declare function buildPieceDocuments(doc: SearchDocument): MeilisearchPieceDocument[];
|
|
17
|
+
/** Filter matching every piece (and a legacy unchunked document) of one entity. */
|
|
18
|
+
export declare function entityPiecesFilter(entityType: string, entityId: EntityId): string;
|
|
19
|
+
/** Filter matching the pieces of one entity from `fromChunk` on: the leftovers of a longer earlier version. */
|
|
20
|
+
export declare function stalePiecesFilter(entityType: string, entityId: EntityId, fromChunk: number): string;
|
|
21
|
+
export declare function filterTypeClause(entityType: string): string;
|