@cosmicdrift/kumiko-framework 0.405.0 → 0.409.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/api/framework-db.md +1 -0
- package/agents/api/framework-engine.md +18 -2
- package/agents/api/framework-ui-types.md +15 -0
- package/agents/api/renderer-web.md +156 -2
- package/agents/api/types.md +113 -71
- package/agents/api.md +4 -4
- package/agents/boot-rules.md +743 -0
- package/agents/examples/contact.md +167 -0
- package/agents/examples/delivery-notifications.md +34 -1
- package/agents/examples/lane-routing.md +13 -0
- package/agents/examples/record-detail-layout.md +13 -2
- package/agents/examples/search.md +65 -3
- package/agents/examples/secrets-demo.md +96 -2
- package/agents/examples/tenant-inactivity.md +138 -0
- package/agents/examples.md +2 -0
- package/agents/features-index.md +4 -0
- package/agents/features.md +24 -1
- package/agents/widgets.md +12 -1
- package/dist/api/export/export-rows.js +1 -0
- package/dist/compliance/index.d.ts +1 -1
- package/dist/db/event-store-executor-read-where.js +15 -5
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/migrate-generator.js +3 -1
- package/dist/db/queries/event-consumer.js +11 -7
- package/dist/db/queries/event-store.d.ts +2 -1
- package/dist/db/queries/event-store.js +25 -6
- package/dist/db/queries/rls-boot-catalog.js +3 -1
- package/dist/db/queries/session-advisory-lock.d.ts +7 -0
- package/dist/db/queries/session-advisory-lock.js +23 -0
- package/dist/db/query.js +7 -10
- package/dist/db/row-level-security-connection.js +4 -2
- package/dist/engine/active-membership.js +1 -0
- package/dist/engine/boot-validator/access-declarations.js +57 -33
- package/dist/engine/boot-validator/action-wiring.js +8 -6
- package/dist/engine/boot-validator/agent-effect-rules.d.ts +12 -0
- package/dist/engine/boot-validator/agent-effect-rules.js +12 -0
- package/dist/engine/boot-validator/agent-effect.d.ts +2 -0
- package/dist/engine/boot-validator/agent-effect.js +20 -0
- package/dist/engine/boot-validator/boot-context.js +5 -1
- package/dist/engine/boot-validator/boot-environment-rules.d.ts +12 -0
- package/dist/engine/boot-validator/boot-environment-rules.js +12 -0
- package/dist/engine/boot-validator/boot-phases-app.js +5 -1
- package/dist/engine/boot-validator/boot-phases.js +3 -1
- package/dist/engine/boot-validator/boot-rules.d.ts +2619 -0
- package/dist/engine/boot-validator/boot-rules.js +58 -0
- package/dist/engine/boot-validator/bundled-lists-table.js +23 -9
- package/dist/engine/boot-validator/config-deps.js +71 -23
- package/dist/engine/boot-validator/config-edit-action-rules.d.ts +17 -0
- package/dist/engine/boot-validator/config-edit-action-rules.js +17 -0
- package/dist/engine/boot-validator/config-key-rules.d.ts +117 -0
- package/dist/engine/boot-validator/config-key-rules.js +117 -0
- package/dist/engine/boot-validator/custom-screen-write-qns.js +6 -2
- package/dist/engine/boot-validator/derived-sql-fields-rules.d.ts +77 -0
- package/dist/engine/boot-validator/derived-sql-fields-rules.js +77 -0
- package/dist/engine/boot-validator/derived-sql-fields.js +53 -17
- package/dist/engine/boot-validator/detail-screens.js +10 -4
- package/dist/engine/boot-validator/entity-declaration-rules.d.ts +92 -0
- package/dist/engine/boot-validator/entity-declaration-rules.js +92 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.d.ts +102 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.js +102 -0
- package/dist/engine/boot-validator/entity-handler.d.ts +0 -7
- package/dist/engine/boot-validator/entity-handler.js +259 -355
- package/dist/engine/boot-validator/entity-image-variants.d.ts +2 -0
- package/dist/engine/boot-validator/entity-image-variants.js +91 -0
- package/dist/engine/boot-validator/entity-list-aggregate-options.js +8 -2
- package/dist/engine/boot-validator/entity-reference-embedded-rules.d.ts +117 -0
- package/dist/engine/boot-validator/entity-reference-embedded-rules.js +117 -0
- package/dist/engine/boot-validator/entity-select-image-rules.d.ts +122 -0
- package/dist/engine/boot-validator/entity-select-image-rules.js +122 -0
- package/dist/engine/boot-validator/entity-select.d.ts +8 -0
- package/dist/engine/boot-validator/entity-select.js +197 -0
- package/dist/engine/boot-validator/entity-structure-rules.d.ts +107 -0
- package/dist/engine/boot-validator/entity-structure-rules.js +107 -0
- package/dist/engine/boot-validator/field-conditions-rules.d.ts +92 -0
- package/dist/engine/boot-validator/field-conditions-rules.js +92 -0
- package/dist/engine/boot-validator/field-conditions.js +67 -22
- package/dist/engine/boot-validator/form-derivations-rules.d.ts +77 -0
- package/dist/engine/boot-validator/form-derivations-rules.js +77 -0
- package/dist/engine/boot-validator/form-derivations.js +52 -18
- package/dist/engine/boot-validator/global-tenancy.js +13 -7
- package/dist/engine/boot-validator/handler-declaration-rules.d.ts +72 -0
- package/dist/engine/boot-validator/handler-declaration-rules.js +72 -0
- package/dist/engine/boot-validator/handler-field-references-rules.d.ts +22 -0
- package/dist/engine/boot-validator/handler-field-references-rules.js +22 -0
- package/dist/engine/boot-validator/handler-field-references.js +14 -6
- package/dist/engine/boot-validator/list-bulk-actions-rules.d.ts +42 -0
- package/dist/engine/boot-validator/list-bulk-actions-rules.js +42 -0
- package/dist/engine/boot-validator/list-bulk-actions.js +29 -11
- package/dist/engine/boot-validator/list-column-layout.js +22 -10
- package/dist/engine/boot-validator/list-query-rules.d.ts +107 -0
- package/dist/engine/boot-validator/list-query-rules.js +107 -0
- package/dist/engine/boot-validator/list-table-aggregate-options.js +43 -15
- package/dist/engine/boot-validator/list-table-aggregate-query.js +7 -3
- package/dist/engine/boot-validator/list-table-aggregate-rules.d.ts +82 -0
- package/dist/engine/boot-validator/list-table-aggregate-rules.js +82 -0
- package/dist/engine/boot-validator/list-table-export-options.js +19 -7
- package/dist/engine/boot-validator/list-table-options.js +33 -13
- package/dist/engine/boot-validator/list-table-reject-unknown-keys.js +6 -2
- package/dist/engine/boot-validator/list-table-rules.d.ts +132 -0
- package/dist/engine/boot-validator/list-table-rules.js +132 -0
- package/dist/engine/boot-validator/nav-rules.d.ts +42 -0
- package/dist/engine/boot-validator/nav-rules.js +42 -0
- package/dist/engine/boot-validator/nav.js +29 -15
- package/dist/engine/boot-validator/ownership-rules.d.ts +27 -0
- package/dist/engine/boot-validator/ownership-rules.js +27 -0
- package/dist/engine/boot-validator/ownership.js +28 -12
- package/dist/engine/boot-validator/parent-ref.js +32 -20
- package/dist/engine/boot-validator/projection-list-screens.js +76 -35
- package/dist/engine/boot-validator/query-output-columns.js +9 -3
- package/dist/engine/boot-validator/query-refs.js +20 -10
- package/dist/engine/boot-validator/query-rules.d.ts +37 -0
- package/dist/engine/boot-validator/query-rules.js +37 -0
- package/dist/engine/boot-validator/read-gate.js +14 -10
- package/dist/engine/boot-validator/record-owned.js +5 -1
- package/dist/engine/boot-validator/related-list-table-rules.js +17 -7
- package/dist/engine/boot-validator/screens-actions-rules.d.ts +185 -0
- package/dist/engine/boot-validator/screens-actions-rules.js +185 -0
- package/dist/engine/boot-validator/screens-actions.js +169 -93
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.d.ts +87 -0
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.js +87 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.d.ts +227 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.js +227 -0
- package/dist/engine/boot-validator/screens-dashboard.js +222 -85
- package/dist/engine/boot-validator/screens-forms-layout-rules.d.ts +122 -0
- package/dist/engine/boot-validator/screens-forms-layout-rules.js +122 -0
- package/dist/engine/boot-validator/screens-forms-rules.d.ts +152 -0
- package/dist/engine/boot-validator/screens-forms-rules.js +152 -0
- package/dist/engine/boot-validator/screens-forms.js +240 -112
- package/dist/engine/boot-validator/screens-kind-config-edit.js +72 -16
- package/dist/engine/boot-validator/screens-kind-entity-edit.js +15 -7
- package/dist/engine/boot-validator/screens-kind-entity-list.js +57 -32
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.d.ts +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.js +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail.js +85 -47
- package/dist/engine/boot-validator/screens-kind-rules.d.ts +137 -0
- package/dist/engine/boot-validator/screens-kind-rules.js +137 -0
- package/dist/engine/boot-validator/screens-kind-secret-mint.js +18 -8
- package/dist/engine/boot-validator/screens-kind-shared.js +40 -22
- package/dist/engine/boot-validator/screens-lists-rules.d.ts +82 -0
- package/dist/engine/boot-validator/screens-lists-rules.js +82 -0
- package/dist/engine/boot-validator/screens-lists.js +90 -45
- package/dist/engine/boot-validator/screens-rules.d.ts +57 -0
- package/dist/engine/boot-validator/screens-rules.js +57 -0
- package/dist/engine/boot-validator/screens.js +17 -12
- package/dist/engine/boot-validator/transaction-phases.js +12 -4
- package/dist/engine/boot-validator/transfer-graph.js +17 -11
- package/dist/engine/boot-validator/url-prefill-fields.js +9 -5
- package/dist/engine/boot-validator/workspaces.js +11 -5
- package/dist/engine/boot-validator/zod-shape.js +6 -2
- package/dist/engine/entity-handlers.d.ts +1 -0
- package/dist/engine/entity-handlers.js +3 -0
- package/dist/engine/entity-list-runtime-options.js +23 -29
- package/dist/engine/extensions/tenant-data.d.ts +3 -0
- package/dist/engine/extensions/tenant-resource.d.ts +1 -0
- package/dist/engine/factories.d.ts +1 -1
- package/dist/engine/factories.js +4 -3
- package/dist/engine/feature-config-events-jobs.js +7 -0
- package/dist/engine/index.d.ts +4 -2
- package/dist/engine/index.js +2 -0
- package/dist/engine/project-app-schema-for-roles.js +29 -14
- package/dist/engine/registry-facade.js +7 -8
- package/dist/engine/registry-validate.js +5 -1
- package/dist/engine/soft-delete-cleanup.d.ts +1 -0
- package/dist/engine/soft-delete-cleanup.js +5 -0
- package/dist/engine/steps/describe-workflow-step-error.js +1 -0
- package/dist/es-ops/operations-schema.d.ts +1 -1
- package/dist/event-store/ensure-dead-letter-table.d.ts +2 -0
- package/dist/event-store/ensure-dead-letter-table.js +10 -0
- package/dist/event-store/rebuild-dead-letter.js +2 -8
- package/dist/event-store/upcaster-dead-letter.js +2 -8
- package/dist/files/copy-pending-file-refs-to-tenant.d.ts +28 -0
- package/dist/files/copy-pending-file-refs-to-tenant.js +87 -0
- package/dist/files/file-routes.d.ts +1 -1
- package/dist/files/file-routes.js +21 -6
- package/dist/files/index.d.ts +2 -0
- package/dist/files/index.js +1 -0
- package/dist/i18n/required-surface-keys.js +1 -0
- package/dist/jobs/job-runner-helpers.js +1 -0
- package/dist/jobs/job-runner-schedulers.d.ts +2 -2
- package/dist/jobs/job-runner-schedulers.js +8 -4
- package/dist/jobs/job-runner.js +14 -16
- package/dist/logging/utils.js +30 -60
- package/dist/observability/metrics-handle.js +10 -21
- package/dist/pipeline/cascade-handler.js +7 -6
- package/dist/pipeline/dispatch-query.js +18 -5
- package/dist/pipeline/dispatch-shared.d.ts +1 -0
- package/dist/pipeline/dispatch-shared.js +3 -1
- package/dist/pipeline/dispatch-write.js +10 -12
- package/dist/pipeline/dispatcher.js +12 -14
- package/dist/pipeline/event-dispatcher-admin.js +9 -8
- package/dist/pipeline/event-dispatcher-delivery.d.ts +19 -1
- package/dist/pipeline/event-dispatcher-delivery.js +121 -29
- package/dist/pipeline/event-dispatcher.d.ts +5 -1
- package/dist/pipeline/event-dispatcher.js +28 -21
- package/dist/pipeline/handler-context-phase-denials.js +2 -0
- package/dist/pipeline/multi-stream-apply-context.js +12 -20
- package/dist/pipeline/system-hooks.d.ts +4 -1
- package/dist/pipeline/system-hooks.js +79 -6
- package/dist/pipeline/system-identity-switch.js +10 -7
- package/dist/search/build-search-snippet.d.ts +9 -0
- package/dist/search/build-search-snippet.js +40 -0
- package/dist/search/index.d.ts +1 -0
- package/dist/search/index.js +1 -0
- package/dist/search/meilisearch-adapter.d.ts +32 -0
- package/dist/search/meilisearch-adapter.js +153 -56
- package/dist/search/meilisearch-chunking.d.ts +21 -0
- package/dist/search/meilisearch-chunking.js +96 -0
- package/dist/search/reindex-entity.d.ts +1 -0
- package/dist/search/reindex-entity.js +97 -52
- package/dist/ui-types/index.d.ts +1 -1
- package/dist/upgrade-cli.js +3 -14
- package/package.json +3 -3
- package/src/changes.json +85 -0
|
@@ -7,7 +7,7 @@ import { createFallbackLogger } from "../logging/utils.js";
|
|
|
7
7
|
import { getFallbackMeter, getFallbackTracer } from "../observability/fallback.js";
|
|
8
8
|
import { emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, } from "../observability/standard-metrics.js";
|
|
9
9
|
import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
|
|
10
|
-
import { acquireConsumerState, consumerInstanceId,
|
|
10
|
+
import { acquireConsumerState, consumerInstanceId, deliverConsumerTurn, emitLagFromTx, fetchPendingEvents, markProcessing, pendingFetchLimit, persistConsumerOutcome, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
|
|
11
11
|
import { mergeContiguousGapRanges, partitionBurntGaps, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
|
|
12
12
|
const DEFAULT_BATCH_SIZE = 200;
|
|
13
13
|
const DEFAULT_POLL_MS = 100;
|
|
@@ -15,6 +15,7 @@ const DEFAULT_MAX_ATTEMPTS = 10;
|
|
|
15
15
|
const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
|
|
16
16
|
const DEFAULT_MAX_REARM_COUNT = 3;
|
|
17
17
|
const PENDING_GAPS_WARN_THRESHOLD = 1000;
|
|
18
|
+
const DEFAULT_MAX_PENDING_GAP_RANGES = 2 * PENDING_GAPS_WARN_THRESHOLD;
|
|
18
19
|
// Only these say the database socket itself is gone; timeouts, schema or pool errors do not
|
|
19
20
|
// imply the dedicated LISTEN connection dropped.
|
|
20
21
|
const CONNECTION_LOSS_CODES = new Set([
|
|
@@ -36,7 +37,7 @@ function isConnectionLossError(e) {
|
|
|
36
37
|
const PASS_FAILURE_BASE_BACKOFF_MS = 1_000;
|
|
37
38
|
const PASS_FAILURE_MAX_BACKOFF_MS = 60_000;
|
|
38
39
|
export function createEventDispatcher(options) {
|
|
39
|
-
const { db, consumers, context, batchSize = DEFAULT_BATCH_SIZE, pollIntervalMs = DEFAULT_POLL_MS, maxAttempts = DEFAULT_MAX_ATTEMPTS, rearmCooldownMs = DEFAULT_REARM_COOLDOWN_MS, maxRearmCount = DEFAULT_MAX_REARM_COUNT, } = options;
|
|
40
|
+
const { db, consumers, context, batchSize = DEFAULT_BATCH_SIZE, pollIntervalMs = DEFAULT_POLL_MS, maxAttempts = DEFAULT_MAX_ATTEMPTS, rearmCooldownMs = DEFAULT_REARM_COOLDOWN_MS, maxRearmCount = DEFAULT_MAX_REARM_COUNT, maxPendingGapRanges = DEFAULT_MAX_PENDING_GAP_RANGES, } = options;
|
|
40
41
|
// Fail-fast on misconfigured per-instance wiring. Catching this at
|
|
41
42
|
// construction surfaces the problem in boot logs instead of first
|
|
42
43
|
// delivery attempt — where it would land as a confusing preRegister
|
|
@@ -344,14 +345,16 @@ export function createEventDispatcher(options) {
|
|
|
344
345
|
// xmin BEFORE the fetch: a range is provably burnt only once this has
|
|
345
346
|
// passed the xmax recorded with it. Only needed when ranges exist.
|
|
346
347
|
const xminNow = pendingGaps.length > 0 ? await selectSnapshotXmin(tx) : "0";
|
|
347
|
-
const
|
|
348
|
-
const
|
|
349
|
-
const
|
|
348
|
+
const pendingRanges = toIdRanges(pendingGaps);
|
|
349
|
+
const pendingLimit = pendingFetchLimit(consumer, batchSize);
|
|
350
|
+
const events = await fetchPendingEvents(tx, oldCursor, batchSize, pendingRanges, pendingLimit);
|
|
351
|
+
const pendingFetchedIds = events.filter((e) => e.id <= oldCursor).map((e) => e.id);
|
|
352
|
+
const pendingTruncated = pendingFetchedIds.length === pendingLimit;
|
|
350
353
|
// Burnt: no visible id this turn, the fetch is proven to have
|
|
351
|
-
// covered the range (LIMIT can otherwise cut it off early), and
|
|
354
|
+
// covered the range (the pending LIMIT can otherwise cut it off early), and
|
|
352
355
|
// xmin passed its recorded xmax — every xact that could still
|
|
353
356
|
// produce a row has finished without one ever appearing.
|
|
354
|
-
const { burnt: burntGaps, surviving: survivingGaps } = partitionBurntGaps(pendingGaps,
|
|
357
|
+
const { burnt: burntGaps, surviving: survivingGaps } = partitionBurntGaps(pendingGaps, pendingFetchedIds, pendingTruncated, xminNow);
|
|
355
358
|
// skip: nothing to deliver and no burnt gap to clean up. The
|
|
356
359
|
// doPass idle pre-check (selectIdleConsumerKeys) already keeps the
|
|
357
360
|
// common idle case out of this transaction entirely — reaching
|
|
@@ -365,18 +368,19 @@ export function createEventDispatcher(options) {
|
|
|
365
368
|
return;
|
|
366
369
|
}
|
|
367
370
|
await markProcessing(tx, consumer.name, instanceId);
|
|
368
|
-
const outcome =
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
371
|
+
const { deliverable, outcome } = await deliverConsumerTurn({
|
|
372
|
+
tx,
|
|
373
|
+
consumer,
|
|
374
|
+
events,
|
|
375
|
+
state: acquired.state,
|
|
376
|
+
context,
|
|
377
|
+
maxAttempts,
|
|
378
|
+
pendingRanges,
|
|
379
|
+
pendingLimit,
|
|
380
|
+
pendingGapCount: survivingGaps.length,
|
|
381
|
+
maxPendingGapRanges,
|
|
382
|
+
});
|
|
383
|
+
const fetchedIds = deliverable.map((e) => e.id);
|
|
380
384
|
processed = outcome.processed;
|
|
381
385
|
failed = outcome.failed;
|
|
382
386
|
// Carve delivered/skip-applied ids out of the surviving ranges —
|
|
@@ -396,8 +400,11 @@ export function createEventDispatcher(options) {
|
|
|
396
400
|
newGapBounds.push([prev + 1n, id - 1n]);
|
|
397
401
|
prev = id;
|
|
398
402
|
}
|
|
399
|
-
|
|
400
|
-
|
|
403
|
+
// Held ids are visible rows, so partitionBurntGaps never burns them; xmax only keeps
|
|
404
|
+
// the entry shape uniform.
|
|
405
|
+
const heldBounds = outcome.heldNewIds.map((id) => [id, id]);
|
|
406
|
+
const xmaxNow = newGapBounds.length > 0 || heldBounds.length > 0 ? await selectSnapshotXmax(tx) : "";
|
|
407
|
+
const newGaps = [...newGapBounds, ...heldBounds].map(([from, to]) => ({
|
|
401
408
|
from: from.toString(),
|
|
402
409
|
to: to.toString(),
|
|
403
410
|
xmax: xmaxNow,
|
|
@@ -21,11 +21,13 @@ export function dropOutsideTransaction(handlerContext) {
|
|
|
21
21
|
if (handlerContext.db)
|
|
22
22
|
attachOutsideTransaction(handlerContext.db, undefined);
|
|
23
23
|
}
|
|
24
|
+
// guard:dup-ok error factory; only the message differs from its siblings, InternalError is the shared constructor
|
|
24
25
|
function beforeTransactionDenied(surface) {
|
|
25
26
|
return new InternalError({
|
|
26
27
|
message: `ctx.${surface} is not available before the transaction; move it into the handler`,
|
|
27
28
|
});
|
|
28
29
|
}
|
|
30
|
+
// guard:dup-ok error factory; only the message differs from its siblings, InternalError is the shared constructor
|
|
29
31
|
export function withoutTransactionDenied(surface) {
|
|
30
32
|
return new InternalError({
|
|
31
33
|
message: `ctx.${surface} is not available without an open transaction; run the write through ctx.dispatch.write so it gets its own transaction`,
|
|
@@ -3,32 +3,24 @@ import { upcastStoredEvents } from "../event-store/upcaster.js";
|
|
|
3
3
|
import { appendDomainEventCore } from "./append-event-core.js";
|
|
4
4
|
import { asTypedAppendEventFn } from "./typed-append-event.js";
|
|
5
5
|
export function createMultiStreamApplyContext(deps) {
|
|
6
|
+
const appendWithLabel = async (callSiteLabel, args) => {
|
|
7
|
+
await appendDomainEventCore({
|
|
8
|
+
registry: deps.registry,
|
|
9
|
+
db: deps.db,
|
|
10
|
+
tenantId: deps.tenantId,
|
|
11
|
+
userId: deps.userId,
|
|
12
|
+
callSiteLabel,
|
|
13
|
+
...(deps.callerFeature && { callerFeature: deps.callerFeature }),
|
|
14
|
+
}, args);
|
|
15
|
+
};
|
|
6
16
|
return {
|
|
7
17
|
registry: deps.registry,
|
|
8
18
|
...(deps.files ? { files: deps.files } : {}),
|
|
9
19
|
...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
|
|
10
20
|
...(deps.secrets ? { secrets: deps.secrets } : {}),
|
|
11
21
|
...(deps.jobs ? { jobs: deps.jobs } : {}),
|
|
12
|
-
appendEvent: asTypedAppendEventFn(
|
|
13
|
-
|
|
14
|
-
registry: deps.registry,
|
|
15
|
-
db: deps.db,
|
|
16
|
-
tenantId: deps.tenantId,
|
|
17
|
-
userId: deps.userId,
|
|
18
|
-
callSiteLabel: "MSP-apply ctx.appendEvent",
|
|
19
|
-
...(deps.callerFeature && { callerFeature: deps.callerFeature }),
|
|
20
|
-
}, args);
|
|
21
|
-
}),
|
|
22
|
-
unsafeAppendEvent: async (args) => {
|
|
23
|
-
await appendDomainEventCore({
|
|
24
|
-
registry: deps.registry,
|
|
25
|
-
db: deps.db,
|
|
26
|
-
tenantId: deps.tenantId,
|
|
27
|
-
userId: deps.userId,
|
|
28
|
-
callSiteLabel: "MSP-apply ctx.unsafeAppendEvent",
|
|
29
|
-
...(deps.callerFeature && { callerFeature: deps.callerFeature }),
|
|
30
|
-
}, args);
|
|
31
|
-
},
|
|
22
|
+
appendEvent: asTypedAppendEventFn((args) => appendWithLabel("MSP-apply ctx.appendEvent", args)),
|
|
23
|
+
unsafeAppendEvent: (args) => appendWithLabel("MSP-apply ctx.unsafeAppendEvent", args),
|
|
32
24
|
loadAggregate: async (aggregateId, options) => {
|
|
33
25
|
const events = options?.asOf
|
|
34
26
|
? await loadAggregateAsOf(deps.db, aggregateId, deps.tenantId, options.asOf)
|
|
@@ -14,7 +14,10 @@ export type SearchOperation = {
|
|
|
14
14
|
readonly entityType: string;
|
|
15
15
|
readonly entityId: EntityId;
|
|
16
16
|
};
|
|
17
|
-
export
|
|
17
|
+
export type SearchEventConsumerOptions = {
|
|
18
|
+
readonly now?: () => number;
|
|
19
|
+
};
|
|
20
|
+
export declare function createSearchEventConsumer(searchAdapter: SearchAdapter, registry: Registry, options?: SearchEventConsumerOptions): EventConsumer;
|
|
18
21
|
export declare function decryptSearchableSubjectFields(entityName: string, state: Record<string, unknown>, registry: Registry): Promise<Record<string, unknown>>;
|
|
19
22
|
export declare function hasErasedSearchableSubjectField(entityName: string, state: Record<string, unknown>, registry: Registry): boolean;
|
|
20
23
|
export declare function buildSearchDocument(entityName: string, entityId: EntityId, state: Record<string, unknown>, registry: Registry): Promise<SearchDocument | null>;
|
|
@@ -121,15 +121,20 @@ async function resolveSearchOperationsByTenant(events, ctx, registry) {
|
|
|
121
121
|
const op = await resolveSearchOperation(event, ctx, registry);
|
|
122
122
|
if (!op)
|
|
123
123
|
continue;
|
|
124
|
-
|
|
124
|
+
const key = searchOperationKey(op);
|
|
125
|
+
// Re-insert so the map order follows the winning event: per-tenant ops stay in event order.
|
|
126
|
+
byKey.delete(key);
|
|
127
|
+
byKey.set(key, { op, sourceEventId: event.id });
|
|
125
128
|
}
|
|
126
129
|
const perTenant = new Map();
|
|
127
|
-
for (const
|
|
130
|
+
for (const sourced of byKey.values()) {
|
|
131
|
+
const op = sourced.op;
|
|
128
132
|
let bucket = perTenant.get(op.tenantId);
|
|
129
133
|
if (!bucket) {
|
|
130
|
-
bucket = { toIndex: [], toRemove: [] };
|
|
134
|
+
bucket = { toIndex: [], toRemove: [], sourcedOps: [] };
|
|
131
135
|
perTenant.set(op.tenantId, bucket);
|
|
132
136
|
}
|
|
137
|
+
bucket.sourcedOps.push(sourced);
|
|
133
138
|
if (op.kind === "index")
|
|
134
139
|
bucket.toIndex.push(op.doc);
|
|
135
140
|
else
|
|
@@ -163,7 +168,60 @@ async function flushSearchOperationsForTenant(searchAdapter, tenantId, bucket) {
|
|
|
163
168
|
}
|
|
164
169
|
}
|
|
165
170
|
}
|
|
166
|
-
|
|
171
|
+
// A failed tenant batch is replayed op by op. After the collapse every op targets another
|
|
172
|
+
// entity, so only the ops that fail are held and the rest land, removes included; the first
|
|
173
|
+
// held event is the earliest failing one, which skipPoisonEvent's lastFailedEventId names.
|
|
174
|
+
async function flushTenantHoldingFailedOperations(searchAdapter, tenantId, bucket) {
|
|
175
|
+
try {
|
|
176
|
+
await flushSearchOperationsForTenant(searchAdapter, tenantId, bucket);
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
catch (batchError) {
|
|
180
|
+
// skip: a single op would only repeat the request that just failed.
|
|
181
|
+
if (bucket.sourcedOps.length === 1) {
|
|
182
|
+
return { eventIds: bucket.sourcedOps.map((s) => s.sourceEventId), error: batchError };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const heldEventIds = [];
|
|
186
|
+
let firstError;
|
|
187
|
+
for (const sourced of bucket.sourcedOps) {
|
|
188
|
+
try {
|
|
189
|
+
await applySearchOperation(searchAdapter, sourced.op);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (heldEventIds.length === 0)
|
|
193
|
+
firstError = error;
|
|
194
|
+
heldEventIds.push(sourced.sourceEventId);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return heldEventIds.length === 0 ? null : { eventIds: heldEventIds, error: firstError };
|
|
198
|
+
}
|
|
199
|
+
// Spacing for a tenant whose index keeps rejecting writes: its held events come back on every
|
|
200
|
+
// dispatcher turn, but Meilisearch is only asked again once the delay has passed.
|
|
201
|
+
const TENANT_RETRY_BASE_DELAY_MS = 1_000;
|
|
202
|
+
const TENANT_RETRY_MAX_DELAY_MS = 60_000;
|
|
203
|
+
// Ten failures span about four minutes of backoff: enough for Meilisearch to restart, short
|
|
204
|
+
// enough that a broken index or a poison document reaches ops as an error.
|
|
205
|
+
const TENANT_HOLD_ESCALATION_FAILURES = 10;
|
|
206
|
+
export function createSearchEventConsumer(searchAdapter, registry, options = {}) {
|
|
207
|
+
const now = options.now ?? Date.now;
|
|
208
|
+
// Process memory only: after a restart a held tenant is simply retried at once.
|
|
209
|
+
const tenantRetries = new Map();
|
|
210
|
+
function recordTenantFailure(tenantId, held) {
|
|
211
|
+
const previous = tenantRetries.get(tenantId);
|
|
212
|
+
const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + 1;
|
|
213
|
+
const delayMs = Math.min(TENANT_RETRY_BASE_DELAY_MS * 2 ** (consecutiveFailures - 1), TENANT_RETRY_MAX_DELAY_MS);
|
|
214
|
+
const message = held.error instanceof Error ? held.error.message : String(held.error);
|
|
215
|
+
const reason = consecutiveFailures >= TENANT_HOLD_ESCALATION_FAILURES
|
|
216
|
+
? `search index of tenant ${tenantId} rejected writes ${consecutiveFailures} times in a row, event ${held.eventIds[0]} stays held until the index is fixed or the event is skipped with skipPoisonEvent: ${message}`
|
|
217
|
+
: `search index of tenant ${tenantId} rejected the write, held for retry: ${message}`;
|
|
218
|
+
tenantRetries.set(tenantId, { consecutiveFailures, retryAtMs: now() + delayMs, reason });
|
|
219
|
+
if (!previous)
|
|
220
|
+
log.warn(reason);
|
|
221
|
+
if (consecutiveFailures === TENANT_HOLD_ESCALATION_FAILURES)
|
|
222
|
+
log.error(reason);
|
|
223
|
+
return reason;
|
|
224
|
+
}
|
|
167
225
|
return {
|
|
168
226
|
name: SEARCH_CONSUMER_NAME,
|
|
169
227
|
// ponytail: count-based maxAttempts, not real backoff — retries hammer
|
|
@@ -178,10 +236,25 @@ export function createSearchEventConsumer(searchAdapter, registry) {
|
|
|
178
236
|
if (op)
|
|
179
237
|
await applySearchOperation(searchAdapter, op);
|
|
180
238
|
},
|
|
181
|
-
|
|
239
|
+
// One tenant's failing index must not stall the others: its failing ops are held for a later
|
|
240
|
+
// turn, and while it backs off all of its ops wait, so per-entity order survives.
|
|
241
|
+
batchHandler: async (events, ctx, turn) => {
|
|
182
242
|
const perTenant = await resolveSearchOperationsByTenant(events, ctx, registry);
|
|
183
243
|
for (const [tenantId, bucket] of perTenant) {
|
|
184
|
-
|
|
244
|
+
const retry = tenantRetries.get(tenantId);
|
|
245
|
+
if (retry && retry.retryAtMs > now()) {
|
|
246
|
+
turn.holdForRetry(bucket.sourcedOps.map((s) => s.sourceEventId), retry.reason);
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
const held = await flushTenantHoldingFailedOperations(searchAdapter, tenantId, bucket);
|
|
250
|
+
if (held) {
|
|
251
|
+
turn.holdForRetry(held.eventIds, recordTenantFailure(tenantId, held));
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (retry) {
|
|
255
|
+
tenantRetries.delete(tenantId);
|
|
256
|
+
log.info(`search index of tenant ${tenantId} accepts writes again`);
|
|
257
|
+
}
|
|
185
258
|
}
|
|
186
259
|
},
|
|
187
260
|
};
|
|
@@ -32,6 +32,7 @@ export function isIdentitySwitchAllowed(caller, asUser, hasGrant) {
|
|
|
32
32
|
return false;
|
|
33
33
|
return isSelfDelegation(caller, asUser);
|
|
34
34
|
}
|
|
35
|
+
// guard:dup-ok error factory; only message and reason differ from unsafeAllTenantsDenied
|
|
35
36
|
export function systemIdentitySwitchDenied(callerLabel) {
|
|
36
37
|
return new AccessDeniedError({
|
|
37
38
|
message: `${callerLabel} may not switch identity to SYSTEM — declare r.systemScope() or ` +
|
|
@@ -69,18 +70,18 @@ function reportGrantedSwitch(caller, asUser, hasGrant, audit) {
|
|
|
69
70
|
audit.report("identity-switch", audit.reason, { id: asUser.id, tenantId: asUser.tenantId });
|
|
70
71
|
}
|
|
71
72
|
export function createGatedIdentitySwitch(callerLabel, caller, hasGrant, ungated, audit) {
|
|
72
|
-
const
|
|
73
|
+
const assertSwitchAllowed = (asUser) => {
|
|
73
74
|
if (!isIdentitySwitchAllowed(caller, asUser, hasGrant)) {
|
|
74
75
|
throw identitySwitchDenied(callerLabel, asUser);
|
|
75
76
|
}
|
|
76
77
|
reportGrantedSwitch(caller, asUser, hasGrant, audit);
|
|
78
|
+
};
|
|
79
|
+
const queryAs = async (asUser, qn, payload) => {
|
|
80
|
+
assertSwitchAllowed(asUser);
|
|
77
81
|
return ungated.queryAs(asUser, qn, payload);
|
|
78
82
|
};
|
|
79
83
|
const writeAs = async (asUser, qn, payload) => {
|
|
80
|
-
|
|
81
|
-
throw identitySwitchDenied(callerLabel, asUser);
|
|
82
|
-
}
|
|
83
|
-
reportGrantedSwitch(caller, asUser, hasGrant, audit);
|
|
84
|
+
assertSwitchAllowed(asUser);
|
|
84
85
|
return ungated.writeAs(asUser, qn, payload);
|
|
85
86
|
};
|
|
86
87
|
const gated = { queryAs, writeAs };
|
|
@@ -137,11 +138,13 @@ function readFunctionMember(namespace, key) {
|
|
|
137
138
|
return typeof value === "function" ? value : undefined; // @cast-boundary engine-bridge — checked via typeof above
|
|
138
139
|
}
|
|
139
140
|
function unavailableIdentitySwitchFn(callerLabel, kind) {
|
|
140
|
-
return
|
|
141
|
+
return (
|
|
142
|
+
// guard:dup-ok a throw-only stub; the message is its whole content
|
|
143
|
+
async () => {
|
|
141
144
|
throw new InternalError({
|
|
142
145
|
message: `${callerLabel}: this context has no ${kind} to switch identity with.`,
|
|
143
146
|
});
|
|
144
|
-
};
|
|
147
|
+
});
|
|
145
148
|
}
|
|
146
149
|
function deniedResolveActiveMembership(callerLabel) {
|
|
147
150
|
return async () => {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export type SearchSnippet = {
|
|
2
|
+
readonly before: string;
|
|
3
|
+
readonly hit: string;
|
|
4
|
+
readonly after: string;
|
|
5
|
+
};
|
|
6
|
+
/** The first occurrence of `term` in `text` with about `radius` characters of context on each side. */
|
|
7
|
+
export declare function buildSearchSnippet(text: string, term: string, { radius }?: {
|
|
8
|
+
readonly radius?: number;
|
|
9
|
+
}): SearchSnippet | null;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const DEFAULT_RADIUS = 60;
|
|
2
|
+
const ELLIPSIS = "…";
|
|
3
|
+
const WHITESPACE = /\s/;
|
|
4
|
+
// Cuts at a word boundary so the snippet never starts or ends in the middle of a word.
|
|
5
|
+
function trimStartToWord(text, from) {
|
|
6
|
+
let start = from;
|
|
7
|
+
while (start < text.length && !WHITESPACE.test(text.charAt(start - 1)))
|
|
8
|
+
start++;
|
|
9
|
+
return start;
|
|
10
|
+
}
|
|
11
|
+
function trimEndToWord(text, to) {
|
|
12
|
+
let end = to;
|
|
13
|
+
while (end > 0 && !WHITESPACE.test(text.charAt(end)))
|
|
14
|
+
end--;
|
|
15
|
+
return end;
|
|
16
|
+
}
|
|
17
|
+
/** The first occurrence of `term` in `text` with about `radius` characters of context on each side. */
|
|
18
|
+
export function buildSearchSnippet(text, term, { radius = DEFAULT_RADIUS } = {}) {
|
|
19
|
+
const needle = term.trim();
|
|
20
|
+
if (needle === "")
|
|
21
|
+
return null;
|
|
22
|
+
// A case-insensitive regex finds the hit in the original text, so offsets stay valid even
|
|
23
|
+
// where lower-casing would change the string length.
|
|
24
|
+
const match = new RegExp(needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "iu").exec(text);
|
|
25
|
+
if (match === null)
|
|
26
|
+
return null;
|
|
27
|
+
const hitStart = match.index;
|
|
28
|
+
const hitEnd = hitStart + match[0].length;
|
|
29
|
+
const rawBeforeStart = hitStart - radius;
|
|
30
|
+
const beforeStart = rawBeforeStart <= 0 ? 0 : Math.min(trimStartToWord(text, rawBeforeStart), hitStart);
|
|
31
|
+
const rawAfterEnd = hitEnd + radius;
|
|
32
|
+
const afterEnd = rawAfterEnd >= text.length ? text.length : Math.max(trimEndToWord(text, rawAfterEnd), hitEnd);
|
|
33
|
+
const before = text.slice(beforeStart, hitStart);
|
|
34
|
+
const after = text.slice(hitEnd, afterEnd);
|
|
35
|
+
return {
|
|
36
|
+
before: beforeStart > 0 ? `${ELLIPSIS}${before}` : before,
|
|
37
|
+
hit: text.slice(hitStart, hitEnd),
|
|
38
|
+
after: afterEnd < text.length ? `${after}${ELLIPSIS}` : after,
|
|
39
|
+
};
|
|
40
|
+
}
|
package/dist/search/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { buildSearchSnippet, type SearchSnippet } from "./build-search-snippet.ts";
|
|
1
2
|
export { collectSearchableFieldNames } from "./configure-tenant-index.ts";
|
|
2
3
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.ts";
|
|
3
4
|
export { isEntitySearchable } from "./is-entity-searchable.ts";
|
package/dist/search/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// `@cosmicdrift/kumiko-framework/search/meilisearch`. This keeps the main barrel from loading a
|
|
3
3
|
// Meilisearch client when consumers only use SearchAdapter types, so apps that do not use
|
|
4
4
|
// Meilisearch do not pull in the client code.
|
|
5
|
+
export { buildSearchSnippet } from "./build-search-snippet.js";
|
|
5
6
|
export { collectSearchableFieldNames } from "./configure-tenant-index.js";
|
|
6
7
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.js";
|
|
7
8
|
export { isEntitySearchable } from "./is-entity-searchable.js";
|
|
@@ -1,3 +1,35 @@
|
|
|
1
1
|
import { type MeilisearchAdapterOptions } from "./meilisearch-adapter-internals.ts";
|
|
2
2
|
import type { SearchAdapter } from "./types.ts";
|
|
3
|
+
/** A drop that finds its index already deleted by a concurrent drop (index_not_found, or the
|
|
4
|
+
* vanished index files) has reached its end state. Reads a task error or an API error's cause. */
|
|
5
|
+
export declare function isGoneIndexError(error: unknown): boolean;
|
|
6
|
+
/** The part of the Meilisearch client an index listing needs. */
|
|
7
|
+
export type IndexListingClient = {
|
|
8
|
+
getIndexes(query: {
|
|
9
|
+
limit: number;
|
|
10
|
+
offset: number;
|
|
11
|
+
}): Promise<{
|
|
12
|
+
results: readonly {
|
|
13
|
+
uid: string;
|
|
14
|
+
}[];
|
|
15
|
+
total: number;
|
|
16
|
+
}>;
|
|
17
|
+
tasks: {
|
|
18
|
+
getTasks(query: {
|
|
19
|
+
types: "indexDeletion"[];
|
|
20
|
+
statuses: ("enqueued" | "processing")[];
|
|
21
|
+
limit: number;
|
|
22
|
+
}): Promise<{
|
|
23
|
+
results: readonly {
|
|
24
|
+
uid: number;
|
|
25
|
+
}[];
|
|
26
|
+
}>;
|
|
27
|
+
waitForTasks(taskUids: number[]): Promise<unknown>;
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
/** Every index uid on the instance, paged. Several apps or test runs can share one Meilisearch:
|
|
31
|
+
* a listing that meets another client's index deletion fails as a whole (500 "os error 2"),
|
|
32
|
+
* so it waits for the deletions in flight and reads once more. A deletion enqueued between
|
|
33
|
+
* that wait and the second read still fails the call. */
|
|
34
|
+
export declare function listIndexUids(client: IndexListingClient): Promise<readonly string[]>;
|
|
3
35
|
export declare function createMeilisearchAdapter(options: MeilisearchAdapterOptions): SearchAdapter;
|