@cosmicdrift/kumiko-framework 0.405.0 → 0.409.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/api/framework-db.md +1 -0
- package/agents/api/framework-engine.md +18 -2
- package/agents/api/framework-ui-types.md +15 -0
- package/agents/api/renderer-web.md +156 -2
- package/agents/api/types.md +113 -71
- package/agents/api.md +4 -4
- package/agents/boot-rules.md +743 -0
- package/agents/examples/contact.md +167 -0
- package/agents/examples/delivery-notifications.md +34 -1
- package/agents/examples/lane-routing.md +13 -0
- package/agents/examples/record-detail-layout.md +13 -2
- package/agents/examples/search.md +65 -3
- package/agents/examples/secrets-demo.md +96 -2
- package/agents/examples/tenant-inactivity.md +138 -0
- package/agents/examples.md +2 -0
- package/agents/features-index.md +4 -0
- package/agents/features.md +24 -1
- package/agents/widgets.md +12 -1
- package/dist/api/export/export-rows.js +1 -0
- package/dist/compliance/index.d.ts +1 -1
- package/dist/db/event-store-executor-read-where.js +15 -5
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/migrate-generator.js +3 -1
- package/dist/db/queries/event-consumer.js +11 -7
- package/dist/db/queries/event-store.d.ts +2 -1
- package/dist/db/queries/event-store.js +25 -6
- package/dist/db/queries/rls-boot-catalog.js +3 -1
- package/dist/db/queries/session-advisory-lock.d.ts +7 -0
- package/dist/db/queries/session-advisory-lock.js +23 -0
- package/dist/db/query.js +7 -10
- package/dist/db/row-level-security-connection.js +4 -2
- package/dist/engine/active-membership.js +1 -0
- package/dist/engine/boot-validator/access-declarations.js +57 -33
- package/dist/engine/boot-validator/action-wiring.js +8 -6
- package/dist/engine/boot-validator/agent-effect-rules.d.ts +12 -0
- package/dist/engine/boot-validator/agent-effect-rules.js +12 -0
- package/dist/engine/boot-validator/agent-effect.d.ts +2 -0
- package/dist/engine/boot-validator/agent-effect.js +20 -0
- package/dist/engine/boot-validator/boot-context.js +5 -1
- package/dist/engine/boot-validator/boot-environment-rules.d.ts +12 -0
- package/dist/engine/boot-validator/boot-environment-rules.js +12 -0
- package/dist/engine/boot-validator/boot-phases-app.js +5 -1
- package/dist/engine/boot-validator/boot-phases.js +3 -1
- package/dist/engine/boot-validator/boot-rules.d.ts +2619 -0
- package/dist/engine/boot-validator/boot-rules.js +58 -0
- package/dist/engine/boot-validator/bundled-lists-table.js +23 -9
- package/dist/engine/boot-validator/config-deps.js +71 -23
- package/dist/engine/boot-validator/config-edit-action-rules.d.ts +17 -0
- package/dist/engine/boot-validator/config-edit-action-rules.js +17 -0
- package/dist/engine/boot-validator/config-key-rules.d.ts +117 -0
- package/dist/engine/boot-validator/config-key-rules.js +117 -0
- package/dist/engine/boot-validator/custom-screen-write-qns.js +6 -2
- package/dist/engine/boot-validator/derived-sql-fields-rules.d.ts +77 -0
- package/dist/engine/boot-validator/derived-sql-fields-rules.js +77 -0
- package/dist/engine/boot-validator/derived-sql-fields.js +53 -17
- package/dist/engine/boot-validator/detail-screens.js +10 -4
- package/dist/engine/boot-validator/entity-declaration-rules.d.ts +92 -0
- package/dist/engine/boot-validator/entity-declaration-rules.js +92 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.d.ts +102 -0
- package/dist/engine/boot-validator/entity-handler-access-rules.js +102 -0
- package/dist/engine/boot-validator/entity-handler.d.ts +0 -7
- package/dist/engine/boot-validator/entity-handler.js +259 -355
- package/dist/engine/boot-validator/entity-image-variants.d.ts +2 -0
- package/dist/engine/boot-validator/entity-image-variants.js +91 -0
- package/dist/engine/boot-validator/entity-list-aggregate-options.js +8 -2
- package/dist/engine/boot-validator/entity-reference-embedded-rules.d.ts +117 -0
- package/dist/engine/boot-validator/entity-reference-embedded-rules.js +117 -0
- package/dist/engine/boot-validator/entity-select-image-rules.d.ts +122 -0
- package/dist/engine/boot-validator/entity-select-image-rules.js +122 -0
- package/dist/engine/boot-validator/entity-select.d.ts +8 -0
- package/dist/engine/boot-validator/entity-select.js +197 -0
- package/dist/engine/boot-validator/entity-structure-rules.d.ts +107 -0
- package/dist/engine/boot-validator/entity-structure-rules.js +107 -0
- package/dist/engine/boot-validator/field-conditions-rules.d.ts +92 -0
- package/dist/engine/boot-validator/field-conditions-rules.js +92 -0
- package/dist/engine/boot-validator/field-conditions.js +67 -22
- package/dist/engine/boot-validator/form-derivations-rules.d.ts +77 -0
- package/dist/engine/boot-validator/form-derivations-rules.js +77 -0
- package/dist/engine/boot-validator/form-derivations.js +52 -18
- package/dist/engine/boot-validator/global-tenancy.js +13 -7
- package/dist/engine/boot-validator/handler-declaration-rules.d.ts +72 -0
- package/dist/engine/boot-validator/handler-declaration-rules.js +72 -0
- package/dist/engine/boot-validator/handler-field-references-rules.d.ts +22 -0
- package/dist/engine/boot-validator/handler-field-references-rules.js +22 -0
- package/dist/engine/boot-validator/handler-field-references.js +14 -6
- package/dist/engine/boot-validator/list-bulk-actions-rules.d.ts +42 -0
- package/dist/engine/boot-validator/list-bulk-actions-rules.js +42 -0
- package/dist/engine/boot-validator/list-bulk-actions.js +29 -11
- package/dist/engine/boot-validator/list-column-layout.js +22 -10
- package/dist/engine/boot-validator/list-query-rules.d.ts +107 -0
- package/dist/engine/boot-validator/list-query-rules.js +107 -0
- package/dist/engine/boot-validator/list-table-aggregate-options.js +43 -15
- package/dist/engine/boot-validator/list-table-aggregate-query.js +7 -3
- package/dist/engine/boot-validator/list-table-aggregate-rules.d.ts +82 -0
- package/dist/engine/boot-validator/list-table-aggregate-rules.js +82 -0
- package/dist/engine/boot-validator/list-table-export-options.js +19 -7
- package/dist/engine/boot-validator/list-table-options.js +33 -13
- package/dist/engine/boot-validator/list-table-reject-unknown-keys.js +6 -2
- package/dist/engine/boot-validator/list-table-rules.d.ts +132 -0
- package/dist/engine/boot-validator/list-table-rules.js +132 -0
- package/dist/engine/boot-validator/nav-rules.d.ts +42 -0
- package/dist/engine/boot-validator/nav-rules.js +42 -0
- package/dist/engine/boot-validator/nav.js +29 -15
- package/dist/engine/boot-validator/ownership-rules.d.ts +27 -0
- package/dist/engine/boot-validator/ownership-rules.js +27 -0
- package/dist/engine/boot-validator/ownership.js +28 -12
- package/dist/engine/boot-validator/parent-ref.js +32 -20
- package/dist/engine/boot-validator/projection-list-screens.js +76 -35
- package/dist/engine/boot-validator/query-output-columns.js +9 -3
- package/dist/engine/boot-validator/query-refs.js +20 -10
- package/dist/engine/boot-validator/query-rules.d.ts +37 -0
- package/dist/engine/boot-validator/query-rules.js +37 -0
- package/dist/engine/boot-validator/read-gate.js +14 -10
- package/dist/engine/boot-validator/record-owned.js +5 -1
- package/dist/engine/boot-validator/related-list-table-rules.js +17 -7
- package/dist/engine/boot-validator/screens-actions-rules.d.ts +185 -0
- package/dist/engine/boot-validator/screens-actions-rules.js +185 -0
- package/dist/engine/boot-validator/screens-actions.js +169 -93
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.d.ts +87 -0
- package/dist/engine/boot-validator/screens-dashboard-chart-rules.js +87 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.d.ts +227 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.js +227 -0
- package/dist/engine/boot-validator/screens-dashboard.js +222 -85
- package/dist/engine/boot-validator/screens-forms-layout-rules.d.ts +122 -0
- package/dist/engine/boot-validator/screens-forms-layout-rules.js +122 -0
- package/dist/engine/boot-validator/screens-forms-rules.d.ts +152 -0
- package/dist/engine/boot-validator/screens-forms-rules.js +152 -0
- package/dist/engine/boot-validator/screens-forms.js +240 -112
- package/dist/engine/boot-validator/screens-kind-config-edit.js +72 -16
- package/dist/engine/boot-validator/screens-kind-entity-edit.js +15 -7
- package/dist/engine/boot-validator/screens-kind-entity-list.js +57 -32
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.d.ts +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.js +92 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail.js +85 -47
- package/dist/engine/boot-validator/screens-kind-rules.d.ts +137 -0
- package/dist/engine/boot-validator/screens-kind-rules.js +137 -0
- package/dist/engine/boot-validator/screens-kind-secret-mint.js +18 -8
- package/dist/engine/boot-validator/screens-kind-shared.js +40 -22
- package/dist/engine/boot-validator/screens-lists-rules.d.ts +82 -0
- package/dist/engine/boot-validator/screens-lists-rules.js +82 -0
- package/dist/engine/boot-validator/screens-lists.js +90 -45
- package/dist/engine/boot-validator/screens-rules.d.ts +57 -0
- package/dist/engine/boot-validator/screens-rules.js +57 -0
- package/dist/engine/boot-validator/screens.js +17 -12
- package/dist/engine/boot-validator/transaction-phases.js +12 -4
- package/dist/engine/boot-validator/transfer-graph.js +17 -11
- package/dist/engine/boot-validator/url-prefill-fields.js +9 -5
- package/dist/engine/boot-validator/workspaces.js +11 -5
- package/dist/engine/boot-validator/zod-shape.js +6 -2
- package/dist/engine/entity-handlers.d.ts +1 -0
- package/dist/engine/entity-handlers.js +3 -0
- package/dist/engine/entity-list-runtime-options.js +23 -29
- package/dist/engine/extensions/tenant-data.d.ts +3 -0
- package/dist/engine/extensions/tenant-resource.d.ts +1 -0
- package/dist/engine/factories.d.ts +1 -1
- package/dist/engine/factories.js +4 -3
- package/dist/engine/feature-config-events-jobs.js +7 -0
- package/dist/engine/index.d.ts +4 -2
- package/dist/engine/index.js +2 -0
- package/dist/engine/project-app-schema-for-roles.js +29 -14
- package/dist/engine/registry-facade.js +7 -8
- package/dist/engine/registry-validate.js +5 -1
- package/dist/engine/soft-delete-cleanup.d.ts +1 -0
- package/dist/engine/soft-delete-cleanup.js +5 -0
- package/dist/engine/steps/describe-workflow-step-error.js +1 -0
- package/dist/es-ops/operations-schema.d.ts +1 -1
- package/dist/event-store/ensure-dead-letter-table.d.ts +2 -0
- package/dist/event-store/ensure-dead-letter-table.js +10 -0
- package/dist/event-store/rebuild-dead-letter.js +2 -8
- package/dist/event-store/upcaster-dead-letter.js +2 -8
- package/dist/files/copy-pending-file-refs-to-tenant.d.ts +28 -0
- package/dist/files/copy-pending-file-refs-to-tenant.js +87 -0
- package/dist/files/file-routes.d.ts +1 -1
- package/dist/files/file-routes.js +21 -6
- package/dist/files/index.d.ts +2 -0
- package/dist/files/index.js +1 -0
- package/dist/i18n/required-surface-keys.js +1 -0
- package/dist/jobs/job-runner-helpers.js +1 -0
- package/dist/jobs/job-runner-schedulers.d.ts +2 -2
- package/dist/jobs/job-runner-schedulers.js +8 -4
- package/dist/jobs/job-runner.js +14 -16
- package/dist/logging/utils.js +30 -60
- package/dist/observability/metrics-handle.js +10 -21
- package/dist/pipeline/cascade-handler.js +7 -6
- package/dist/pipeline/dispatch-query.js +18 -5
- package/dist/pipeline/dispatch-shared.d.ts +1 -0
- package/dist/pipeline/dispatch-shared.js +3 -1
- package/dist/pipeline/dispatch-write.js +10 -12
- package/dist/pipeline/dispatcher.js +12 -14
- package/dist/pipeline/event-dispatcher-admin.js +9 -8
- package/dist/pipeline/event-dispatcher-delivery.d.ts +19 -1
- package/dist/pipeline/event-dispatcher-delivery.js +121 -29
- package/dist/pipeline/event-dispatcher.d.ts +5 -1
- package/dist/pipeline/event-dispatcher.js +28 -21
- package/dist/pipeline/handler-context-phase-denials.js +2 -0
- package/dist/pipeline/multi-stream-apply-context.js +12 -20
- package/dist/pipeline/system-hooks.d.ts +4 -1
- package/dist/pipeline/system-hooks.js +79 -6
- package/dist/pipeline/system-identity-switch.js +10 -7
- package/dist/search/build-search-snippet.d.ts +9 -0
- package/dist/search/build-search-snippet.js +40 -0
- package/dist/search/index.d.ts +1 -0
- package/dist/search/index.js +1 -0
- package/dist/search/meilisearch-adapter.d.ts +32 -0
- package/dist/search/meilisearch-adapter.js +153 -56
- package/dist/search/meilisearch-chunking.d.ts +21 -0
- package/dist/search/meilisearch-chunking.js +96 -0
- package/dist/search/reindex-entity.d.ts +1 -0
- package/dist/search/reindex-entity.js +97 -52
- package/dist/ui-types/index.d.ts +1 -1
- package/dist/upgrade-cli.js +3 -14
- package/package.json +3 -3
- package/src/changes.json +85 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { errorCodeOf } from "@cosmicdrift/kumiko-types/type-guards";
|
|
1
|
+
import { errorCodeOf, propertyOf } from "@cosmicdrift/kumiko-types/type-guards";
|
|
2
2
|
import { importOptionalPeer } from "../utils/import-optional-peer.js";
|
|
3
|
-
import {
|
|
3
|
+
import { meilisearchTenantIndex, } from "./meilisearch-adapter-internals.js";
|
|
4
|
+
import { buildPieceDocuments, entityPiecesFilter, filterTypeClause, stalePiecesFilter, } from "./meilisearch-chunking.js";
|
|
4
5
|
import { MEILISEARCH_PEER } from "./meilisearch-peer.js";
|
|
5
6
|
// meilisearch is an optional peer; a static import would fail with the runtime's bare
|
|
6
7
|
// "Cannot find package" error instead of naming the peer and how to install it.
|
|
@@ -15,7 +16,9 @@ async function awaitSucceededTask(enqueued, toleratedErrorCodes = []) {
|
|
|
15
16
|
const task = await enqueued.waitTask();
|
|
16
17
|
const tolerated = task.error ? toleratedErrorCodes.includes(task.error.code) : false;
|
|
17
18
|
if (task.status !== "succeeded" && !tolerated) {
|
|
18
|
-
|
|
19
|
+
// Only the code: Meili's message can embed the whole rejected document (missing_document_id),
|
|
20
|
+
// and this error ends up in the consumer's lastError and the logs. The task uid leads to it.
|
|
21
|
+
const detail = task.error ? task.error.code : "no error code";
|
|
19
22
|
throw new Error(`Meilisearch task ${task.uid} (${task.type}) ended with status "${task.status}": ${detail}`);
|
|
20
23
|
}
|
|
21
24
|
return task;
|
|
@@ -25,11 +28,21 @@ async function awaitSucceededTask(enqueued, toleratedErrorCodes = []) {
|
|
|
25
28
|
// Without this, a delete event or a DSGVO subject purge for such a tenant
|
|
26
29
|
// would fail forever.
|
|
27
30
|
const REMOVE_TOLERATED_ERROR_CODES = [ErrorStatusCode.INDEX_NOT_FOUND];
|
|
28
|
-
const INDEX_LIST_PAGE_SIZE =
|
|
31
|
+
const INDEX_LIST_PAGE_SIZE = 1000;
|
|
29
32
|
// What an unconfigured Meilisearch index reports as searchableAttributes.
|
|
30
33
|
const MEILI_ALL_ATTRIBUTES = "*";
|
|
31
|
-
|
|
34
|
+
// _entityId and _chunk back the piece filters (stale-piece cleanup, remove); distinct on
|
|
35
|
+
// _entityId makes an entity with several matching pieces appear once in the results.
|
|
36
|
+
const DISTINCT_ATTRIBUTE = "_entityId";
|
|
37
|
+
const DEFAULT_FILTERABLE_ATTRIBUTES = [
|
|
38
|
+
"_type",
|
|
39
|
+
"_weight",
|
|
40
|
+
DISTINCT_ATTRIBUTE,
|
|
41
|
+
"_chunk",
|
|
42
|
+
];
|
|
32
43
|
const DEFAULT_SORTABLE_ATTRIBUTES = ["_weight"];
|
|
44
|
+
// Filters per deleteDocuments call: one task per call, and the filter string stays short.
|
|
45
|
+
const DELETE_FILTER_BATCH = 100;
|
|
33
46
|
function defaultSearchableAttributes(config) {
|
|
34
47
|
return config.rankingFields ?? config.searchableFields;
|
|
35
48
|
}
|
|
@@ -41,19 +54,70 @@ function isSubsetOf(stored, wanted) {
|
|
|
41
54
|
// older default that gained fields since; anything with a foreign attribute is
|
|
42
55
|
// an explicit configure() and stays untouched. Known limit: an explicit configure() that
|
|
43
56
|
// narrowed to a strict subset of the default is indistinguishable and gets widened again.
|
|
44
|
-
function isOutdatedDefault(stored, wanted) {
|
|
57
|
+
function isOutdatedDefault(stored, wanted, distinctMissing) {
|
|
45
58
|
const pairs = [
|
|
46
59
|
[stored.searchable, wanted.searchable],
|
|
47
60
|
[stored.filterable, wanted.filterable],
|
|
48
61
|
[stored.sortable, wanted.sortable],
|
|
49
62
|
];
|
|
50
63
|
return (pairs.every(([have, want]) => isSubsetOf(have, want)) &&
|
|
51
|
-
|
|
64
|
+
(distinctMissing ||
|
|
65
|
+
pairs.some(([have, want]) => want.some((attribute) => !have.includes(attribute)))));
|
|
52
66
|
}
|
|
53
67
|
function isIndexNotFound(err) {
|
|
54
68
|
const cause = typeof err === "object" && err !== null && "cause" in err ? err.cause : null;
|
|
55
69
|
return errorCodeOf(cause) === ErrorStatusCode.INDEX_NOT_FOUND;
|
|
56
70
|
}
|
|
71
|
+
// Meili reports ENOENT as "No such file or directory (os error 2)" when an index's files vanish
|
|
72
|
+
// under a running task.
|
|
73
|
+
const MISSING_INDEX_FILES = "os error 2";
|
|
74
|
+
/** A drop that finds its index already deleted by a concurrent drop (index_not_found, or the
|
|
75
|
+
* vanished index files) has reached its end state. Reads a task error or an API error's cause. */
|
|
76
|
+
export function isGoneIndexError(error) {
|
|
77
|
+
const message = propertyOf(error, "message");
|
|
78
|
+
return (errorCodeOf(error) === ErrorStatusCode.INDEX_NOT_FOUND ||
|
|
79
|
+
(typeof message === "string" && message.includes(MISSING_INDEX_FILES)));
|
|
80
|
+
}
|
|
81
|
+
/** Every index uid on the instance, paged. Several apps or test runs can share one Meilisearch:
|
|
82
|
+
* a listing that meets another client's index deletion fails as a whole (500 "os error 2"),
|
|
83
|
+
* so it waits for the deletions in flight and reads once more. A deletion enqueued between
|
|
84
|
+
* that wait and the second read still fails the call. */
|
|
85
|
+
export async function listIndexUids(client) {
|
|
86
|
+
try {
|
|
87
|
+
return await readIndexUids(client);
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
if (!isGoneIndexError(propertyOf(err, "cause")))
|
|
91
|
+
throw err;
|
|
92
|
+
}
|
|
93
|
+
const pending = await client.tasks.getTasks({
|
|
94
|
+
types: ["indexDeletion"],
|
|
95
|
+
statuses: ["enqueued", "processing"],
|
|
96
|
+
limit: INDEX_LIST_PAGE_SIZE,
|
|
97
|
+
});
|
|
98
|
+
await client.tasks.waitForTasks(pending.results.map((task) => task.uid));
|
|
99
|
+
return readIndexUids(client);
|
|
100
|
+
}
|
|
101
|
+
async function readIndexUids(client) {
|
|
102
|
+
const uids = [];
|
|
103
|
+
for (let offset = 0;; offset += INDEX_LIST_PAGE_SIZE) {
|
|
104
|
+
const page = await client.getIndexes({ limit: INDEX_LIST_PAGE_SIZE, offset });
|
|
105
|
+
uids.push(...page.results.map((index) => index.uid));
|
|
106
|
+
if (page.results.length === 0 || offset + page.results.length >= page.total)
|
|
107
|
+
return uids;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Enqueued together so Meili batches the deletions: every deletion in flight fails other
|
|
111
|
+
// clients' index listings, and one batch keeps that window short.
|
|
112
|
+
async function dropIndexesIfPresent(client, uids) {
|
|
113
|
+
const enqueued = await Promise.all(uids.map((uid) => client.deleteIndex(uid)));
|
|
114
|
+
const tasks = await client.tasks.waitForTasks(enqueued);
|
|
115
|
+
// A concurrent drop that got there first leaves the index gone, which is the goal.
|
|
116
|
+
const failed = tasks.find((task) => task.status !== "succeeded" && !isGoneIndexError(task.error));
|
|
117
|
+
if (failed) {
|
|
118
|
+
throw new Error(`Meilisearch task ${failed.uid} (${failed.type}) ended with status "${failed.status}": ${failed.error ? failed.error.code : "no error code"}`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
57
121
|
// Exported for unit tests (index-name / primary-key shape) without a live Meili.
|
|
58
122
|
// Meilisearch primary-key-ids: alphanumerics, `-`, `_`. UUIDs contain `-` —
|
|
59
123
|
// legal. Replace anything else just in case callers pass unexpected shapes.
|
|
@@ -78,6 +142,42 @@ export function createMeilisearchAdapter(options) {
|
|
|
78
142
|
await awaitSucceededTask(index.updateSearchableAttributes([...defaultSearchableAttributes(config)]));
|
|
79
143
|
await awaitSucceededTask(index.updateFilterableAttributes([...DEFAULT_FILTERABLE_ATTRIBUTES]));
|
|
80
144
|
await awaitSucceededTask(index.updateSortableAttributes([...DEFAULT_SORTABLE_ATTRIBUTES]));
|
|
145
|
+
await awaitSucceededTask(index.updateDistinctAttribute(DISTINCT_ATTRIBUTE));
|
|
146
|
+
}
|
|
147
|
+
// Filter deletes need the filterable attributes, but a tenant whose index was never created has
|
|
148
|
+
// nothing to delete and must not get an empty index out of a delete.
|
|
149
|
+
async function ensureConfiguredIfIndexExists(tenantId) {
|
|
150
|
+
if (!configuredTenants.has(tenantId)) {
|
|
151
|
+
try {
|
|
152
|
+
await client.getIndex(meilisearchTenantIndex(prefix, tenantId));
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
if (isIndexNotFound(err))
|
|
156
|
+
return false;
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
await ensureConfigured(tenantId);
|
|
161
|
+
return true;
|
|
162
|
+
}
|
|
163
|
+
async function deleteByFilters(tenantId, filters) {
|
|
164
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
165
|
+
await writeForgettingConfigOnFailure(tenantId, async () => {
|
|
166
|
+
for (let offset = 0; offset < filters.length; offset += DELETE_FILTER_BATCH) {
|
|
167
|
+
const filter = filters.slice(offset, offset + DELETE_FILTER_BATCH).join(" OR ");
|
|
168
|
+
await awaitSucceededTask(index.deleteDocuments({ filter }), REMOVE_TOLERATED_ERROR_CODES);
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
// All pieces first, then the leftovers of an earlier, longer version: a failure in between
|
|
173
|
+
// leaves a document that is too long, never a hole.
|
|
174
|
+
async function writeDocuments(tenantId, docs) {
|
|
175
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
176
|
+
const pieces = docs.flatMap(buildPieceDocuments);
|
|
177
|
+
await writeForgettingConfigOnFailure(tenantId, async () => {
|
|
178
|
+
await awaitSucceededTask(index.addDocuments(pieces, { primaryKey: "_id" }));
|
|
179
|
+
});
|
|
180
|
+
await deleteByFilters(tenantId, docs.map((doc) => stalePiecesFilter(doc.entityType, doc.entityId, buildPieceDocuments(doc).length)));
|
|
81
181
|
}
|
|
82
182
|
// Lazily configures a tenant's index off the default config on its first
|
|
83
183
|
// access, so callers that never call `configure` still get a filterable/
|
|
@@ -89,7 +189,9 @@ export function createMeilisearchAdapter(options) {
|
|
|
89
189
|
// configure() must not be overwritten by the registry default, so the default is applied only
|
|
90
190
|
// to an index that is missing, carries Meilisearch's factory settings, or holds an older
|
|
91
191
|
// default that is a strict subset of the current one (fields were added by a deploy).
|
|
92
|
-
|
|
192
|
+
// A kept index still gets the default filterable attributes merged in: remove, purge and the
|
|
193
|
+
// stale-piece delete filter on them, and an index without them rejects every delete.
|
|
194
|
+
async function configActionFor(tenantId, config) {
|
|
93
195
|
try {
|
|
94
196
|
const settings = await client.index(meilisearchTenantIndex(prefix, tenantId)).getSettings();
|
|
95
197
|
const searchable = settings.searchableAttributes ?? [MEILI_ALL_ATTRIBUTES];
|
|
@@ -97,16 +199,22 @@ export function createMeilisearchAdapter(options) {
|
|
|
97
199
|
const factorySettings = searchable.length === 1 &&
|
|
98
200
|
searchable[0] === MEILI_ALL_ATTRIBUTES &&
|
|
99
201
|
!filterable.includes("_type");
|
|
100
|
-
|
|
202
|
+
const needsDefault = factorySettings ||
|
|
101
203
|
isOutdatedDefault({ searchable, filterable, sortable: settings.sortableAttributes ?? [] }, {
|
|
102
204
|
searchable: defaultSearchableAttributes(config),
|
|
103
205
|
filterable: DEFAULT_FILTERABLE_ATTRIBUTES,
|
|
104
206
|
sortable: DEFAULT_SORTABLE_ATTRIBUTES,
|
|
105
|
-
})
|
|
207
|
+
}, settings.distinctAttribute !== DISTINCT_ATTRIBUTE);
|
|
208
|
+
if (needsDefault)
|
|
209
|
+
return { kind: "apply-default" };
|
|
210
|
+
const missing = DEFAULT_FILTERABLE_ATTRIBUTES.filter((attribute) => !filterable.includes(attribute));
|
|
211
|
+
return missing.length === 0
|
|
212
|
+
? { kind: "keep" }
|
|
213
|
+
: { kind: "add-filterable", filterable: [...filterable, ...missing] };
|
|
106
214
|
}
|
|
107
215
|
catch (err) {
|
|
108
216
|
if (isIndexNotFound(err))
|
|
109
|
-
return
|
|
217
|
+
return { kind: "apply-default" };
|
|
110
218
|
throw err;
|
|
111
219
|
}
|
|
112
220
|
}
|
|
@@ -122,8 +230,14 @@ export function createMeilisearchAdapter(options) {
|
|
|
122
230
|
}
|
|
123
231
|
const config = defaultConfig;
|
|
124
232
|
await trackConfigure(tenantId, async () => {
|
|
125
|
-
|
|
233
|
+
const action = await configActionFor(tenantId, config);
|
|
234
|
+
if (action.kind === "apply-default")
|
|
126
235
|
await applyConfig(tenantId, config);
|
|
236
|
+
if (action.kind === "add-filterable") {
|
|
237
|
+
await awaitSucceededTask(client
|
|
238
|
+
.index(meilisearchTenantIndex(prefix, tenantId))
|
|
239
|
+
.updateFilterableAttributes(action.filterable));
|
|
240
|
+
}
|
|
127
241
|
});
|
|
128
242
|
}
|
|
129
243
|
async function trackConfigure(tenantId, run) {
|
|
@@ -139,6 +253,17 @@ export function createMeilisearchAdapter(options) {
|
|
|
139
253
|
throw err;
|
|
140
254
|
}
|
|
141
255
|
}
|
|
256
|
+
// A rejected write can mean another writer recreated or reconfigured the index since this
|
|
257
|
+
// process configured it; forgetting the tenant makes the next write re-check its settings.
|
|
258
|
+
async function writeForgettingConfigOnFailure(tenantId, write) {
|
|
259
|
+
try {
|
|
260
|
+
await write();
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
configuredTenants.delete(tenantId);
|
|
264
|
+
throw err;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
142
267
|
return {
|
|
143
268
|
setDefaultConfig(config) {
|
|
144
269
|
defaultConfig = config;
|
|
@@ -148,77 +273,47 @@ export function createMeilisearchAdapter(options) {
|
|
|
148
273
|
},
|
|
149
274
|
async index(tenantId, doc) {
|
|
150
275
|
await ensureConfigured(tenantId);
|
|
151
|
-
|
|
152
|
-
await awaitSucceededTask(index.addDocuments([
|
|
153
|
-
{
|
|
154
|
-
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
155
|
-
_type: doc.entityType,
|
|
156
|
-
_weight: doc.weight,
|
|
157
|
-
_entityId: doc.entityId,
|
|
158
|
-
...doc.fields,
|
|
159
|
-
},
|
|
160
|
-
], { primaryKey: "_id" }));
|
|
276
|
+
await writeDocuments(tenantId, [doc]);
|
|
161
277
|
},
|
|
162
278
|
async indexBatch(tenantId, docs) {
|
|
163
279
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
164
280
|
if (docs.length === 0)
|
|
165
281
|
return;
|
|
166
282
|
await ensureConfigured(tenantId);
|
|
167
|
-
|
|
168
|
-
const payload = docs.map((doc) => ({
|
|
169
|
-
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
170
|
-
_type: doc.entityType,
|
|
171
|
-
_weight: doc.weight,
|
|
172
|
-
_entityId: doc.entityId,
|
|
173
|
-
...doc.fields,
|
|
174
|
-
}));
|
|
175
|
-
// Single Meilisearch task covering all N docs. Meilisearch processes
|
|
176
|
-
// the payload server-side as one indexing job — waitTask blocks until
|
|
177
|
-
// that job is done, but it's one round-trip instead of N.
|
|
178
|
-
await awaitSucceededTask(index.addDocuments(payload, { primaryKey: "_id" }));
|
|
283
|
+
await writeDocuments(tenantId, docs);
|
|
179
284
|
},
|
|
180
285
|
async dropAllIndexes() {
|
|
181
286
|
// An empty prefix matches every index, which would wipe the whole Meili instance.
|
|
182
287
|
if (prefix === "") {
|
|
183
288
|
throw new Error("refusing to drop Meilisearch indexes with an empty prefix");
|
|
184
289
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
const page = await client.getIndexes({ limit: INDEX_LIST_PAGE_SIZE, offset });
|
|
188
|
-
for (const index of page.results) {
|
|
189
|
-
if (isOwnTenantIndex(index.uid))
|
|
190
|
-
uids.push(index.uid);
|
|
191
|
-
}
|
|
192
|
-
if (page.results.length === 0 || offset + page.results.length >= page.total)
|
|
193
|
-
break;
|
|
194
|
-
}
|
|
195
|
-
// Cleared even when a delete throws midway: otherwise ensureConfigured treats the
|
|
196
|
-
// already-dropped tenants as configured and the next index() recreates an
|
|
290
|
+
// Cleared even when the listing or a delete throws midway: otherwise ensureConfigured
|
|
291
|
+
// treats the already-dropped tenants as configured and the next index() recreates an
|
|
197
292
|
// unconfigured index ("not filterable" on filtered search).
|
|
198
293
|
try {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
294
|
+
const uids = (await listIndexUids(client)).filter(isOwnTenantIndex);
|
|
295
|
+
await dropIndexesIfPresent(client, uids);
|
|
296
|
+
return uids.length;
|
|
202
297
|
}
|
|
203
298
|
finally {
|
|
204
299
|
configuredTenants.clear();
|
|
205
300
|
}
|
|
206
|
-
return uids.length;
|
|
207
301
|
},
|
|
208
302
|
async removeBatch(tenantId, items) {
|
|
209
303
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
210
304
|
if (items.length === 0)
|
|
211
305
|
return;
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
306
|
+
// skip: the tenant has no index, so nothing is left to remove
|
|
307
|
+
if (!(await ensureConfiguredIfIndexExists(tenantId)))
|
|
308
|
+
return;
|
|
309
|
+
await deleteByFilters(tenantId, items.map((item) => entityPiecesFilter(item.entityType, item.entityId)));
|
|
215
310
|
},
|
|
216
311
|
async search(tenantId, query, options) {
|
|
217
312
|
await ensureConfigured(tenantId);
|
|
218
313
|
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
219
314
|
const filter = [];
|
|
220
315
|
if (options?.filterType) {
|
|
221
|
-
filter.push(
|
|
316
|
+
filter.push(filterTypeClause(options.filterType));
|
|
222
317
|
}
|
|
223
318
|
const searchParams = {
|
|
224
319
|
limit: options?.limit ?? 50,
|
|
@@ -233,8 +328,10 @@ export function createMeilisearchAdapter(options) {
|
|
|
233
328
|
}));
|
|
234
329
|
},
|
|
235
330
|
async remove(tenantId, entityType, entityId) {
|
|
236
|
-
|
|
237
|
-
|
|
331
|
+
// skip: the tenant has no index, so nothing is left to remove
|
|
332
|
+
if (!(await ensureConfiguredIfIndexExists(tenantId)))
|
|
333
|
+
return;
|
|
334
|
+
await deleteByFilters(tenantId, [entityPiecesFilter(entityType, entityId)]);
|
|
238
335
|
},
|
|
239
336
|
};
|
|
240
337
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { EntityId } from "../engine/types/identifiers.ts";
|
|
2
|
+
import type { SearchDocument } from "./types.ts";
|
|
3
|
+
export declare const CHUNK_MAX_CHARS = 8000;
|
|
4
|
+
export declare const CHUNK_OVERLAP_CHARS = 200;
|
|
5
|
+
/** Splits a long value into pieces of at most CHUNK_MAX_CHARS, cut at whitespace. */
|
|
6
|
+
export declare function splitLongText(value: string): string[];
|
|
7
|
+
export type MeilisearchPieceDocument = Record<string, unknown> & {
|
|
8
|
+
readonly _id: string;
|
|
9
|
+
};
|
|
10
|
+
export declare function pieceDocId(baseId: string, chunk: number): string;
|
|
11
|
+
/**
|
|
12
|
+
* One Meilisearch document per piece. Piece 0 keeps the id the entity always had (existing
|
|
13
|
+
* documents stay valid, no duplicates) and carries all short fields; piece n holds the n-th
|
|
14
|
+
* parts of the long fields.
|
|
15
|
+
*/
|
|
16
|
+
export declare function buildPieceDocuments(doc: SearchDocument): MeilisearchPieceDocument[];
|
|
17
|
+
/** Filter matching every piece (and a legacy unchunked document) of one entity. */
|
|
18
|
+
export declare function entityPiecesFilter(entityType: string, entityId: EntityId): string;
|
|
19
|
+
/** Filter matching the pieces of one entity from `fromChunk` on: the leftovers of a longer earlier version. */
|
|
20
|
+
export declare function stalePiecesFilter(entityType: string, entityId: EntityId, fromChunk: number): string;
|
|
21
|
+
export declare function filterTypeClause(entityType: string): string;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { meilisearchDocId } from "./meilisearch-adapter-internals.js";
|
|
2
|
+
// Meilisearch indexes at most 65_535 word positions per attribute, and a hard
|
|
3
|
+
// separator advances the position by up to 8. 8_000 characters yield at most
|
|
4
|
+
// 8_000 words (single-character words separated by one space), so a piece of this
|
|
5
|
+
// size stays far below the limit even in the worst case. Longer values are split
|
|
6
|
+
// into pieces instead of being cut: nothing is silently dropped from the index.
|
|
7
|
+
export const CHUNK_MAX_CHARS = 8_000;
|
|
8
|
+
export const CHUNK_OVERLAP_CHARS = 200;
|
|
9
|
+
const WHITESPACE = /\s/;
|
|
10
|
+
function lastWhitespaceBefore(value, end, floor) {
|
|
11
|
+
for (let i = end - 1; i > floor; i--) {
|
|
12
|
+
if (WHITESPACE.test(value.charAt(i)))
|
|
13
|
+
return i;
|
|
14
|
+
}
|
|
15
|
+
return -1;
|
|
16
|
+
}
|
|
17
|
+
// Start of the next piece: about CHUNK_OVERLAP_CHARS before the cut, moved forward to a word
|
|
18
|
+
// start, so a word or short phrase straddling the cut is whole in at least one piece.
|
|
19
|
+
function nextPieceStart(value, pieceStart, cut) {
|
|
20
|
+
const rawStart = Math.max(cut - CHUNK_OVERLAP_CHARS, pieceStart + 1);
|
|
21
|
+
for (let i = rawStart; i < cut; i++) {
|
|
22
|
+
if (WHITESPACE.test(value.charAt(i - 1)))
|
|
23
|
+
return i;
|
|
24
|
+
}
|
|
25
|
+
return cut;
|
|
26
|
+
}
|
|
27
|
+
/** Splits a long value into pieces of at most CHUNK_MAX_CHARS, cut at whitespace. */
|
|
28
|
+
export function splitLongText(value) {
|
|
29
|
+
if (value.length <= CHUNK_MAX_CHARS)
|
|
30
|
+
return [value];
|
|
31
|
+
const pieces = [];
|
|
32
|
+
let start = 0;
|
|
33
|
+
while (value.length - start > CHUNK_MAX_CHARS) {
|
|
34
|
+
const limit = start + CHUNK_MAX_CHARS;
|
|
35
|
+
const space = lastWhitespaceBefore(value, limit, start + CHUNK_OVERLAP_CHARS);
|
|
36
|
+
const cut = space === -1 ? limit : space;
|
|
37
|
+
pieces.push(value.slice(start, cut));
|
|
38
|
+
start = nextPieceStart(value, start, cut);
|
|
39
|
+
}
|
|
40
|
+
pieces.push(value.slice(start));
|
|
41
|
+
return pieces;
|
|
42
|
+
}
|
|
43
|
+
export function pieceDocId(baseId, chunk) {
|
|
44
|
+
return chunk === 0 ? baseId : `${baseId}_c${chunk}`;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* One Meilisearch document per piece. Piece 0 keeps the id the entity always had (existing
|
|
48
|
+
* documents stay valid, no duplicates) and carries all short fields; piece n holds the n-th
|
|
49
|
+
* parts of the long fields.
|
|
50
|
+
*/
|
|
51
|
+
export function buildPieceDocuments(doc) {
|
|
52
|
+
const shortFields = {};
|
|
53
|
+
const longPieces = [];
|
|
54
|
+
for (const [key, value] of Object.entries(doc.fields)) {
|
|
55
|
+
if (typeof value === "string" && value.length > CHUNK_MAX_CHARS) {
|
|
56
|
+
longPieces.push([key, splitLongText(value)]);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
shortFields[key] = value;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const baseId = meilisearchDocId(doc.entityType, doc.entityId);
|
|
63
|
+
const pieceCount = Math.max(1, ...longPieces.map(([, pieces]) => pieces.length));
|
|
64
|
+
const documents = [];
|
|
65
|
+
for (let chunk = 0; chunk < pieceCount; chunk++) {
|
|
66
|
+
const piece = chunk === 0 ? { ...shortFields } : {};
|
|
67
|
+
for (const [key, pieces] of longPieces) {
|
|
68
|
+
const part = pieces[chunk];
|
|
69
|
+
if (part !== undefined)
|
|
70
|
+
piece[key] = part;
|
|
71
|
+
}
|
|
72
|
+
documents.push({
|
|
73
|
+
...piece,
|
|
74
|
+
_id: pieceDocId(baseId, chunk),
|
|
75
|
+
_type: doc.entityType,
|
|
76
|
+
_weight: doc.weight,
|
|
77
|
+
_entityId: doc.entityId,
|
|
78
|
+
_chunk: chunk,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return documents;
|
|
82
|
+
}
|
|
83
|
+
function quoted(value) {
|
|
84
|
+
return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
|
85
|
+
}
|
|
86
|
+
/** Filter matching every piece (and a legacy unchunked document) of one entity. */
|
|
87
|
+
export function entityPiecesFilter(entityType, entityId) {
|
|
88
|
+
return `(_type = ${quoted(entityType)} AND _entityId = ${quoted(String(entityId))})`;
|
|
89
|
+
}
|
|
90
|
+
/** Filter matching the pieces of one entity from `fromChunk` on: the leftovers of a longer earlier version. */
|
|
91
|
+
export function stalePiecesFilter(entityType, entityId, fromChunk) {
|
|
92
|
+
return `(_type = ${quoted(entityType)} AND _entityId = ${quoted(String(entityId))} AND _chunk >= ${fromChunk})`;
|
|
93
|
+
}
|
|
94
|
+
export function filterTypeClause(entityType) {
|
|
95
|
+
return `_type = ${quoted(entityType)}`;
|
|
96
|
+
}
|
|
@@ -14,5 +14,6 @@ export type ReindexEntityResult = {
|
|
|
14
14
|
export type ReindexEntityOptions = {
|
|
15
15
|
readonly batchSize?: number;
|
|
16
16
|
readonly dryRun?: boolean;
|
|
17
|
+
readonly afterBatchRead?: () => Promise<void>;
|
|
17
18
|
};
|
|
18
19
|
export declare function reindexEntity(db: DbRunner, registry: Registry, searchAdapter: SearchAdapter, entityName: string, tenantId: TenantId, options?: ReindexEntityOptions): Promise<ReindexEntityResult>;
|
|
@@ -40,6 +40,87 @@ async function readBatchWithoutStatementTimeout(db, sqlText, params) {
|
|
|
40
40
|
return executeRawQueryRead(tx, sqlText, params);
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
|
+
async function buildBatchDocs(batch, rows, failures) {
|
|
44
|
+
const docs = [];
|
|
45
|
+
for (const row of rows) {
|
|
46
|
+
const entityId = String(row["id"]);
|
|
47
|
+
try {
|
|
48
|
+
const rawState = rowToState(row, batch.fieldNames);
|
|
49
|
+
// `pii + searchable` fields hold ciphertext on the read-table row —
|
|
50
|
+
// decrypt the same way the live write-path consumer does. An
|
|
51
|
+
// erased subject key doesn't throw, it swaps the value for
|
|
52
|
+
// PII_ERASED_SENTINEL (see decryptSearchableSubjectFields) — check
|
|
53
|
+
// for that AFTER decrypting, mirroring the softDelete filter in reindexEntity:
|
|
54
|
+
// resurrecting a crypto-shredded row here would undo
|
|
55
|
+
// purgeSearchDocumentsForSubject and make it findable again.
|
|
56
|
+
const state = await decryptSearchableSubjectFields(batch.entityName, rawState, batch.registry);
|
|
57
|
+
if (hasErasedSearchableSubjectField(batch.entityName, state, batch.registry))
|
|
58
|
+
continue;
|
|
59
|
+
const doc = await buildSearchDocument(batch.entityName, entityId, state, batch.registry);
|
|
60
|
+
if (doc)
|
|
61
|
+
docs.push({ entityId, doc });
|
|
62
|
+
}
|
|
63
|
+
catch (e) {
|
|
64
|
+
failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return docs;
|
|
68
|
+
}
|
|
69
|
+
async function writeBatchDocs(searchAdapter, tenantId, docs, failures) {
|
|
70
|
+
if (searchAdapter.indexBatch) {
|
|
71
|
+
try {
|
|
72
|
+
await searchAdapter.indexBatch(tenantId, docs.map((d) => d.doc));
|
|
73
|
+
return docs.length;
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
77
|
+
for (const { entityId } of docs)
|
|
78
|
+
failures.push({ entityId, reason });
|
|
79
|
+
return 0;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
let written = 0;
|
|
83
|
+
for (const { entityId, doc } of docs) {
|
|
84
|
+
try {
|
|
85
|
+
await searchAdapter.index(tenantId, doc);
|
|
86
|
+
written++;
|
|
87
|
+
}
|
|
88
|
+
catch (e) {
|
|
89
|
+
failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return written;
|
|
93
|
+
}
|
|
94
|
+
// The live consumer keeps running while a reindex scans. A row deleted, purged or edited after
|
|
95
|
+
// its batch was read would otherwise leave stale text in the index, the one thing a delete or a
|
|
96
|
+
// subject purge must not leave behind. So the written ids are read again: rows that vanished,
|
|
97
|
+
// were soft-deleted or became erased leave the index, rows whose version moved are rewritten fresh.
|
|
98
|
+
async function reconcileWrittenBatch(batch, searchAdapter, tenantId, readRows, written, failures) {
|
|
99
|
+
const writtenIds = new Set(written.map((w) => w.entityId));
|
|
100
|
+
const versionAtRead = new Map(readRows.map((row) => [String(row["id"]), row["version"]]));
|
|
101
|
+
const ids = [...writtenIds];
|
|
102
|
+
const placeholders = ids.map((_, i) => `$${i + 2}`).join(", ");
|
|
103
|
+
const fresh = await readBatchWithoutStatementTimeout(batch.db, `SELECT * FROM ${quoteIdent(batch.tableName)}
|
|
104
|
+
WHERE ${quoteIdent("tenant_id")} = $1 AND ${quoteIdent("id")} IN (${placeholders})`, [tenantId, ...ids]);
|
|
105
|
+
const live = fresh.filter((row) => !(batch.deletedFilter && row["is_deleted"] === true));
|
|
106
|
+
const liveIds = new Set(live.map((row) => String(row["id"])));
|
|
107
|
+
const changed = live.filter((row) => row["version"] !== versionAtRead.get(String(row["id"])));
|
|
108
|
+
const rewritten = await buildBatchDocs(batch, changed, failures);
|
|
109
|
+
const rewrittenIds = new Set(rewritten.map((w) => w.entityId));
|
|
110
|
+
const changedIds = new Set(changed.map((row) => String(row["id"])));
|
|
111
|
+
const toRemove = ids
|
|
112
|
+
.filter((id) => !liveIds.has(id) || (changedIds.has(id) && !rewrittenIds.has(id)))
|
|
113
|
+
.map((entityId) => ({ entityType: batch.entityName, entityId }));
|
|
114
|
+
if (toRemove.length > 0) {
|
|
115
|
+
if (searchAdapter.removeBatch)
|
|
116
|
+
await searchAdapter.removeBatch(tenantId, toRemove);
|
|
117
|
+
else
|
|
118
|
+
for (const item of toRemove)
|
|
119
|
+
await searchAdapter.remove(tenantId, item.entityType, item.entityId);
|
|
120
|
+
}
|
|
121
|
+
if (rewritten.length > 0)
|
|
122
|
+
await writeBatchDocs(searchAdapter, tenantId, rewritten, failures);
|
|
123
|
+
}
|
|
43
124
|
export async function reindexEntity(db, registry, searchAdapter, entityName, tenantId, options = {}) {
|
|
44
125
|
return runInSystemRowLevelSecurityScope("search reindex reads entity rows of all tenants", async () => {
|
|
45
126
|
const entity = registry.getEntity(entityName);
|
|
@@ -61,6 +142,14 @@ export async function reindexEntity(db, registry, searchAdapter, entityName, ten
|
|
|
61
142
|
const deletedFilter = entity.softDelete === true ? `AND ${quoteIdent("is_deleted")} IS NOT TRUE` : "";
|
|
62
143
|
const failures = [];
|
|
63
144
|
const result = { scannedRows: 0, indexedRows: 0, wouldIndexRows: 0, failures };
|
|
145
|
+
const batch = {
|
|
146
|
+
db,
|
|
147
|
+
registry,
|
|
148
|
+
entityName,
|
|
149
|
+
tableName,
|
|
150
|
+
fieldNames,
|
|
151
|
+
deletedFilter: entity.softDelete === true,
|
|
152
|
+
};
|
|
64
153
|
let lastId;
|
|
65
154
|
// Whether a searchable field lacks a matching read-table column depends
|
|
66
155
|
// only on the table's column set (rowToState/Object.hasOwn), not on any
|
|
@@ -89,60 +178,16 @@ export async function reindexEntity(db, registry, searchAdapter, entityName, ten
|
|
|
89
178
|
`column ${quoteIdent(tableName)} — likely a dropped/never-migrated column.`);
|
|
90
179
|
}
|
|
91
180
|
}
|
|
92
|
-
const docs =
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
try {
|
|
97
|
-
const rawState = rowToState(row, fieldNames);
|
|
98
|
-
// `pii + searchable` fields hold ciphertext on the read-table row —
|
|
99
|
-
// decrypt the same way the live write-path consumer does. An
|
|
100
|
-
// erased subject key doesn't throw, it swaps the value for
|
|
101
|
-
// PII_ERASED_SENTINEL (see decryptSearchableSubjectFields) — check
|
|
102
|
-
// for that AFTER decrypting, mirroring the softDelete filter above:
|
|
103
|
-
// resurrecting a crypto-shredded row here would undo
|
|
104
|
-
// purgeSearchDocumentsForSubject and make it findable again.
|
|
105
|
-
const state = await decryptSearchableSubjectFields(entityName, rawState, registry);
|
|
106
|
-
if (hasErasedSearchableSubjectField(entityName, state, registry))
|
|
107
|
-
continue;
|
|
108
|
-
const doc = await buildSearchDocument(entityName, entityId, state, registry);
|
|
109
|
-
if (doc)
|
|
110
|
-
docs.push({ entityId, doc });
|
|
111
|
-
}
|
|
112
|
-
catch (e) {
|
|
113
|
-
result.failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
if (!options.dryRun && docs.length > 0) {
|
|
117
|
-
if (searchAdapter.indexBatch) {
|
|
118
|
-
try {
|
|
119
|
-
await searchAdapter.indexBatch(tenantId, docs.map((d) => d.doc));
|
|
120
|
-
result.indexedRows += docs.length;
|
|
121
|
-
}
|
|
122
|
-
catch (e) {
|
|
123
|
-
const reason = e instanceof Error ? e.message : String(e);
|
|
124
|
-
for (const { entityId } of docs)
|
|
125
|
-
result.failures.push({ entityId, reason });
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
else {
|
|
129
|
-
for (const { entityId, doc } of docs) {
|
|
130
|
-
try {
|
|
131
|
-
await searchAdapter.index(tenantId, doc);
|
|
132
|
-
result.indexedRows++;
|
|
133
|
-
}
|
|
134
|
-
catch (e) {
|
|
135
|
-
result.failures.push({
|
|
136
|
-
entityId,
|
|
137
|
-
reason: e instanceof Error ? e.message : String(e),
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
else if (options.dryRun) {
|
|
181
|
+
const docs = await buildBatchDocs(batch, rows, result.failures);
|
|
182
|
+
result.scannedRows += rows.length;
|
|
183
|
+
await options.afterBatchRead?.();
|
|
184
|
+
if (options.dryRun) {
|
|
144
185
|
result.wouldIndexRows += docs.length;
|
|
145
186
|
}
|
|
187
|
+
else if (docs.length > 0) {
|
|
188
|
+
result.indexedRows += await writeBatchDocs(searchAdapter, tenantId, docs, result.failures);
|
|
189
|
+
await reconcileWrittenBatch(batch, searchAdapter, tenantId, rows, docs, result.failures);
|
|
190
|
+
}
|
|
146
191
|
lastId = rows[rows.length - 1]?.["id"];
|
|
147
192
|
if (rows.length < batchSize)
|
|
148
193
|
break;
|
package/dist/ui-types/index.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ export type { EmbeddedDerivedCellDef, EntityDefinition, FieldDefinition, Options
|
|
|
11
11
|
export type { AccessRule } from "../engine/types/handlers.ts";
|
|
12
12
|
export { isOpenToAllGranted, isUiAccessGranted } from "../engine/types/handlers.ts";
|
|
13
13
|
export type { IconKey, NavDefinition, NavIconKey } from "../engine/types/nav.ts";
|
|
14
|
-
export type { ActionFormFooterAction, ActionFormLoad, ActionFormRedirect, ActionFormScreenDefinition, ActionFormSuccessFact, ActionFormSuccessLink, ConfigEditScreenDefinition, CustomScreenDefinition, DashboardChartMarkerKind, DashboardChartPanel, DashboardCurrencyFormat, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelEmptyState, DashboardPanelQueryOptions, DashboardPanelSpan, DashboardPanelVisibility, DashboardProgressListPanel, DashboardProgressTone, DashboardRefresh, DashboardRelativeTimeParam, DashboardScopeDefinition, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextBlock, DashboardTextPanel, DashboardTextParam, DashboardTimeRangeDefinition, EditExtensionSection, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EditWriteFormSection, EmbeddedListEditSpec, EntityEditAction, EntityEditScreenDefinition, EntityListFacetConfig, EntityListScreenDefinition, FieldAffix, FieldCondition, FieldFillAction, FieldIconKey, FieldRenderer, FormWidth, ListBulkAction, ListColumnSpec, ListEmptyStateSpec, ListFacetSpec, ListSortSpec, ListTableOptions, MetricNavigate, PlatformComponent, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, RecordHeaderSubtitlePart, RelatedListEmptyStateSpec, RelatedListEmptyStateVariant, RelatedListRowTone, RelatedListToolbarAction, RowAction, RowActionClientAction, RowActionDisplay, RowActionDrawer, RowActionMode, RowActionNavigate, RowActionWriteHandler, RowFieldExtractor, ScreenDefinition, ScreenFallback, ScreenFallbackByReason, ScreenFallbackReason, ScreenSlots, ScreenVisibility, SecretMintConfirmStep, SecretMintScreenDefinition, SecretsEditScreenDefinition, ToolbarAction, } from "../engine/types/screen.ts";
|
|
14
|
+
export type { ActionFormFooterAction, ActionFormLoad, ActionFormRedirect, ActionFormScreenDefinition, ActionFormSuccessFact, ActionFormSuccessItems, ActionFormSuccessLink, ConfigEditAction, ConfigEditScreenDefinition, CustomScreenDefinition, DashboardChartMarkerKind, DashboardChartPanel, DashboardCurrencyFormat, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelEmptyState, DashboardPanelQueryOptions, DashboardPanelSpan, DashboardPanelVisibility, DashboardProgressListPanel, DashboardProgressTone, DashboardRefresh, DashboardRelativeTimeParam, DashboardScopeDefinition, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextBlock, DashboardTextPanel, DashboardTextParam, DashboardTimeRangeDefinition, EditExtensionSection, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EditWriteFormSection, EmbeddedListEditSpec, EntityEditAction, EntityEditScreenDefinition, EntityListFacetConfig, EntityListScreenDefinition, FieldAffix, FieldCondition, FieldFillAction, FieldIconKey, FieldRenderer, FormWidth, ListBulkAction, ListColumnSpec, ListEmptyStateSpec, ListFacetSpec, ListSortSpec, ListTableOptions, MetricNavigate, PlatformComponent, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, RecordHeaderSubtitlePart, RelatedListEmptyStateSpec, RelatedListEmptyStateVariant, RelatedListRowTone, RelatedListToolbarAction, RowAction, RowActionClientAction, RowActionDisplay, RowActionDrawer, RowActionMode, RowActionNavigate, RowActionWriteHandler, RowFieldExtractor, ScreenDefinition, ScreenFallback, ScreenFallbackByReason, ScreenFallbackReason, ScreenSlots, ScreenVisibility, SecretMintConfirmStep, SecretMintScreenDefinition, SecretsEditScreenDefinition, ToolbarAction, } from "../engine/types/screen.ts";
|
|
15
15
|
export { MAX_BULK_SELECTION } from "../engine/types/screen.ts";
|
|
16
16
|
export type { TargetRef } from "../engine/types/target-ref.ts";
|
|
17
17
|
export type { TreeAction } from "../engine/types/tree-node.ts";
|