@cosmicdrift/kumiko-framework 0.404.1 → 0.406.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/api/framework-engine.md +3 -0
- package/agents/examples/lane-routing.md +13 -0
- package/dist/api/auth-routes-mfa.js +2 -2
- package/dist/api/index.d.ts +2 -2
- package/dist/api/index.js +1 -1
- package/dist/api/readiness.d.ts +0 -11
- package/dist/api/readiness.js +1 -30
- package/dist/crypto/index.d.ts +1 -0
- package/dist/crypto/index.js +1 -0
- package/dist/crypto/opaque-token-hash.d.ts +3 -0
- package/dist/crypto/opaque-token-hash.js +14 -0
- package/dist/crypto/subject-resolver.js +3 -8
- package/dist/db/blind-index-cleanup.js +3 -10
- package/dist/db/query.js +3 -18
- package/dist/derivatives/variant-key.d.ts +4 -0
- package/dist/derivatives/variant-key.js +4 -3
- package/dist/engine/boot-validator/field-conditions.js +2 -7
- package/dist/engine/boot-validator/list-table-aggregate-query.js +1 -1
- package/dist/engine/boot-validator/parent-ref.d.ts +0 -2
- package/dist/engine/boot-validator/parent-ref.js +1 -8
- package/dist/engine/boot-validator/projection-list-screens.js +1 -1
- package/dist/engine/boot-validator/read-gate.js +1 -1
- package/dist/engine/extensions/tenant-resource.d.ts +1 -0
- package/dist/engine/feature-config-events-jobs.js +7 -0
- package/dist/engine/find-entity.d.ts +3 -0
- package/dist/engine/find-entity.js +8 -0
- package/dist/engine/project-app-schema-for-roles.js +19 -55
- package/dist/files/file-handle.js +3 -16
- package/dist/jobs/job-runner-schedulers.d.ts +2 -2
- package/dist/jobs/job-runner-schedulers.js +8 -4
- package/dist/jobs/job-runner.js +7 -4
- package/dist/observability/ids.js +1 -8
- package/dist/pipeline/projection-rebuild.js +3 -17
- package/dist/pipeline/tenant-timezone-cache.js +8 -27
- package/dist/search/index.d.ts +3 -0
- package/dist/search/index.js +3 -0
- package/dist/search/is-entity-searchable.d.ts +2 -0
- package/dist/search/is-entity-searchable.js +4 -0
- package/dist/search/reindex-tenants-search.d.ts +19 -0
- package/dist/search/reindex-tenants-search.js +69 -0
- package/dist/utils/chunk.d.ts +1 -0
- package/dist/utils/chunk.js +7 -0
- package/dist/utils/hex.d.ts +3 -0
- package/dist/utils/hex.js +11 -0
- package/dist/utils/index.d.ts +3 -0
- package/dist/utils/index.js +3 -0
- package/dist/utils/lru-cache.d.ts +17 -0
- package/dist/utils/lru-cache.js +56 -0
- package/dist/utils/run-with-timeout.d.ts +11 -0
- package/dist/utils/run-with-timeout.js +30 -0
- package/package.json +4 -4
- package/src/changes.json +48 -0
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// suffix before the file extension — `foo/bar.jpg` + `"medium-<16 hex>"` →
|
|
11
11
|
// `foo/bar.medium-<16 hex>.jpg`. The suffix must be a `variantSuffix(...)` so
|
|
12
12
|
// the erasure sweep recognizes it. Stable, reversible, no extra lookup tables.
|
|
13
|
-
import { derivativeListPrefix, isDerivativeSuffix } from "../derivatives/variant-key.js";
|
|
13
|
+
import { derivativeListPrefix, isDerivativeSuffix, splitKey } from "../derivatives/variant-key.js";
|
|
14
14
|
import { InternalError } from "../errors/index.js";
|
|
15
15
|
import { assertSafeStorageKey } from "./types.js";
|
|
16
16
|
// `getProvider` is a lazily-resolved, memoized accessor — the provider is
|
|
@@ -100,26 +100,13 @@ export function createFileContext(resolve, tenantId, uploadLedger) {
|
|
|
100
100
|
export function createTenantFileContext(resolver, tenantId) {
|
|
101
101
|
return createFileContext(() => resolver(tenantId), tenantId, resolver.uploadLedgerFor?.(tenantId));
|
|
102
102
|
}
|
|
103
|
-
// Splits a key into the part before the last dot of its last path segment
|
|
104
|
-
// (the "stem") and the extension including the dot. No dot in the last
|
|
105
|
-
// segment: the whole key is the stem, extension is empty.
|
|
106
|
-
function splitKeyExtension(key) {
|
|
107
|
-
const lastSlash = key.lastIndexOf("/");
|
|
108
|
-
const lastSegment = lastSlash === -1 ? key : key.slice(lastSlash + 1);
|
|
109
|
-
const lastDot = lastSegment.lastIndexOf(".");
|
|
110
|
-
if (lastDot === -1)
|
|
111
|
-
return { stem: key, ext: "" };
|
|
112
|
-
const stem = key.slice(0, key.length - lastSegment.length + lastDot);
|
|
113
|
-
const ext = lastSegment.slice(lastDot);
|
|
114
|
-
return { stem, ext };
|
|
115
|
-
}
|
|
116
103
|
// Inserts a suffix before the file extension. Keys without an extension get
|
|
117
104
|
// the suffix appended with a dot: `foo/bar` + `"small"` → `foo/bar.small`.
|
|
118
105
|
// Keys with a dot earlier in the path (e.g. `archive.v2/foo.jpg`) correctly
|
|
119
106
|
// split on the LAST segment only.
|
|
120
107
|
export function deriveKey(key, suffix) {
|
|
121
|
-
const {
|
|
122
|
-
return ext === "" ? `${key}.${suffix}` : `${
|
|
108
|
+
const { base, ext } = splitKey(key);
|
|
109
|
+
return ext === "" ? `${key}.${suffix}` : `${base}.${suffix}${ext}`;
|
|
123
110
|
}
|
|
124
111
|
// The prefix that covers a key AND every `derive()`d variant of it — `foo/bar.jpg` → `foo/bar.`,
|
|
125
112
|
// matching `foo/bar.jpg` and `foo/bar.medium.jpg` alike, since derive() only ever inserts a suffix
|
|
@@ -10,7 +10,7 @@ export declare const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-
|
|
|
10
10
|
* leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828,
|
|
11
11
|
* publicstatus Redis OOM). Strip `.` and `:` from the job QN.
|
|
12
12
|
*/
|
|
13
|
-
export declare function schedulerIdForJobName(jobName: string): string;
|
|
13
|
+
export declare function schedulerIdForJobName(jobName: string, wakesSleepingApp?: boolean): string;
|
|
14
14
|
export declare function bootJobIdForJobName(jobName: string): string;
|
|
15
15
|
export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
|
|
16
16
|
export declare function markSchedulersSeen(queue: Queue, schedulerIds: ReadonlySet<string>): Promise<void>;
|
|
@@ -18,6 +18,6 @@ type OrphanSchedulerLog = {
|
|
|
18
18
|
info(msg: string, data?: Record<string, unknown>): void;
|
|
19
19
|
warn(msg: string, data?: Record<string, unknown>): void;
|
|
20
20
|
};
|
|
21
|
-
export declare function removeOrphanJobSchedulers(queue: Queue, expectedSchedulerIds: ReadonlySet<string>, lane: JobRunIn, log: OrphanSchedulerLog): Promise<void>;
|
|
21
|
+
export declare function removeOrphanJobSchedulers(queue: Queue, expectedSchedulerIds: ReadonlySet<string>, supersededSchedulerIds: ReadonlySet<string>, lane: JobRunIn, log: OrphanSchedulerLog): Promise<void>;
|
|
22
22
|
export declare function perTenantChildJobId(wrapperJobId: string, tenantId: string): string;
|
|
23
23
|
export {};
|
|
@@ -22,9 +22,12 @@ export const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
|
|
|
22
22
|
* leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828,
|
|
23
23
|
* publicstatus Redis OOM). Strip `.` and `:` from the job QN.
|
|
24
24
|
*/
|
|
25
|
-
export function schedulerIdForJobName(jobName) {
|
|
26
|
-
|
|
25
|
+
export function schedulerIdForJobName(jobName, wakesSleepingApp = true) {
|
|
26
|
+
const prefix = wakesSleepingApp ? "scheduler-" : NO_WAKE_SCHEDULER_PREFIX;
|
|
27
|
+
return `${prefix}${jobName.replace(/[.:]/g, "-")}`;
|
|
27
28
|
}
|
|
29
|
+
// Contract with the infra waker of sleeping shared apps: it skips due jobs under this prefix.
|
|
30
|
+
const NO_WAKE_SCHEDULER_PREFIX = "scheduler-nowake-";
|
|
28
31
|
// Same colon-in-BullMQ-id hazard as schedulerIdForJobName —
|
|
29
32
|
// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
|
|
30
33
|
export function bootJobIdForJobName(jobName) {
|
|
@@ -53,7 +56,7 @@ export async function markSchedulersSeen(queue, schedulerIds) {
|
|
|
53
56
|
// A cron job dropped from the app (or an older scheduler-id format) leaves its
|
|
54
57
|
// scheduler in Redis, where it keeps firing jobs the worker no longer knows
|
|
55
58
|
// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
|
|
56
|
-
export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
|
|
59
|
+
export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, supersededSchedulerIds, lane, log) {
|
|
57
60
|
try {
|
|
58
61
|
// Collect first: removing while paging would shift the indices.
|
|
59
62
|
const existingKeys = [];
|
|
@@ -71,7 +74,8 @@ export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lan
|
|
|
71
74
|
if (expectedSchedulerIds.has(key))
|
|
72
75
|
continue;
|
|
73
76
|
const lastSeen = Number(seenAt[key]);
|
|
74
|
-
|
|
77
|
+
const isWithinGrace = key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS;
|
|
78
|
+
if (isWithinGrace && !supersededSchedulerIds.has(key))
|
|
75
79
|
continue;
|
|
76
80
|
await queue.removeJobScheduler(key);
|
|
77
81
|
await client.hdel(schedulerSeenKey(queue), key);
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -294,11 +294,14 @@ export function createJobRunner(options) {
|
|
|
294
294
|
}
|
|
295
295
|
}
|
|
296
296
|
}
|
|
297
|
-
|
|
297
|
+
// `superseded`: the id the job would have with the other wake flag. No runner of any version
|
|
298
|
+
// expects both ids of one job, so a leftover under that id goes without the orphan grace period.
|
|
299
|
+
function cronSchedulerIdsForLane(lane, superseded = false) {
|
|
298
300
|
const ids = new Set();
|
|
299
301
|
for (const [name, jobDef] of allJobs) {
|
|
300
302
|
if (laneForJob(jobDef) === lane && isScheduledTrigger(jobDef.trigger)) {
|
|
301
|
-
|
|
303
|
+
const wakesSleepingApp = jobDef.trigger.wakesSleepingApp !== false;
|
|
304
|
+
ids.add(schedulerIdForJobName(name, superseded ? !wakesSleepingApp : wakesSleepingApp));
|
|
302
305
|
}
|
|
303
306
|
}
|
|
304
307
|
return ids;
|
|
@@ -308,7 +311,7 @@ export function createJobRunner(options) {
|
|
|
308
311
|
if (laneForJob(jobDef) !== lane)
|
|
309
312
|
continue;
|
|
310
313
|
if (isScheduledTrigger(jobDef.trigger)) {
|
|
311
|
-
await queue.upsertJobScheduler(schedulerIdForJobName(name), scheduleRepeatOptions(jobDef.trigger, options.maintenanceWindow), {
|
|
314
|
+
await queue.upsertJobScheduler(schedulerIdForJobName(name, jobDef.trigger.wakesSleepingApp !== false), scheduleRepeatOptions(jobDef.trigger, options.maintenanceWindow), {
|
|
312
315
|
name: jobDef.perTenant ? `${PER_TENANT_PREFIX}${name}` : name,
|
|
313
316
|
data: {},
|
|
314
317
|
// Queue-level defaultJobOptions (jobRetentionOpts) covers
|
|
@@ -423,7 +426,7 @@ export function createJobRunner(options) {
|
|
|
423
426
|
});
|
|
424
427
|
});
|
|
425
428
|
await upsertCronSchedulers(consumerQueue, consumerLane);
|
|
426
|
-
await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
|
|
429
|
+
await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, cronSchedulerIdsForLane(consumerLane, true), consumerLane, {
|
|
427
430
|
info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
|
|
428
431
|
warn: errorLogger.warn,
|
|
429
432
|
});
|
|
@@ -2,14 +2,7 @@
|
|
|
2
2
|
// collectors and UIs understand what we emit:
|
|
3
3
|
// - traceId: 16 random bytes, rendered as 32 lowercase hex chars
|
|
4
4
|
// - spanId: 8 random bytes, rendered as 16 lowercase hex chars
|
|
5
|
-
|
|
6
|
-
let out = "";
|
|
7
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
8
|
-
const b = bytes[i] ?? 0;
|
|
9
|
-
out += b.toString(16).padStart(2, "0");
|
|
10
|
-
}
|
|
11
|
-
return out;
|
|
12
|
-
}
|
|
5
|
+
import { bytesToHex } from "../utils/hex.js";
|
|
13
6
|
export function generateTraceId() {
|
|
14
7
|
const bytes = new Uint8Array(16);
|
|
15
8
|
crypto.getRandomValues(bytes);
|
|
@@ -6,8 +6,9 @@ import { runInSystemRowLevelSecurityScope } from "../db/row-level-security-scope
|
|
|
6
6
|
import { withoutStatementTimeout } from "../db/statement-timeout.js";
|
|
7
7
|
import { extractTableInfo } from "../db/table-info.js";
|
|
8
8
|
import { withExecutorManagedWrites } from "../db/tenant-db.js";
|
|
9
|
-
import { eventsTable, getEventsHighWaterMark, upcastStoredEvent
|
|
9
|
+
import { eventsTable, getEventsHighWaterMark, upcastStoredEvent } from "../event-store/index.js";
|
|
10
10
|
import { createRebuildDeadLetterTable, recordRebuildDeadLetters, } from "../event-store/rebuild-dead-letter.js";
|
|
11
|
+
import { toStoredEvent } from "../event-store/row-to-stored-event.js";
|
|
11
12
|
import { createFallbackLogger } from "../logging/utils.js";
|
|
12
13
|
import { emitProjectionRebuild } from "../observability/standard-metrics.js";
|
|
13
14
|
import { projectionStateTable } from "./projection-state.js";
|
|
@@ -22,21 +23,6 @@ export const REBUILD_BATCH_SIZE = 1000;
|
|
|
22
23
|
// live table is held ACCESS EXCLUSIVE).
|
|
23
24
|
const MAX_UNLOCKED_BATCHES = 10_000;
|
|
24
25
|
const DEFAULT_FENCE_LOCK_TIMEOUT_MS = 5_000;
|
|
25
|
-
function rowToStoredEvent(row) {
|
|
26
|
-
return {
|
|
27
|
-
id: String(row.id),
|
|
28
|
-
aggregateId: row.aggregateId,
|
|
29
|
-
aggregateType: row.aggregateType,
|
|
30
|
-
tenantId: row.tenantId,
|
|
31
|
-
version: row.version,
|
|
32
|
-
type: row.type,
|
|
33
|
-
eventVersion: row.eventVersion,
|
|
34
|
-
payload: row.payload,
|
|
35
|
-
metadata: row.metadata,
|
|
36
|
-
createdAt: row.createdAt,
|
|
37
|
-
createdBy: row.createdBy,
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
26
|
const TENANT_COLUMN_SUFFIX = ".tenant_id";
|
|
41
27
|
// Ops-facing log line for countColumnDrift's result. No-op when clean.
|
|
42
28
|
function warnOnColumnDrift(projectionName, tableName, drift) {
|
|
@@ -93,7 +79,7 @@ async function replayProjectionIntoShadow(projectionName, deps) {
|
|
|
93
79
|
for (const r of rawEvents) {
|
|
94
80
|
deps.signal?.throwIfAborted();
|
|
95
81
|
const row = coerceRow(r, eventsInfo);
|
|
96
|
-
const storedEvent = await upcastStoredEvent(
|
|
82
|
+
const storedEvent = await upcastStoredEvent(toStoredEvent(row), upcasters, {
|
|
97
83
|
db: tx,
|
|
98
84
|
tenantId: row.tenantId, // @cast-boundary db-row
|
|
99
85
|
});
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// the write. The TTL below therefore bounds how long OTHER replicas (and
|
|
9
9
|
// writes that bypass that path: migrations, seeds, direct DB edits) keep
|
|
10
10
|
// serving the old value, hence the short default.
|
|
11
|
+
import { createLruCache } from "../utils/lru-cache.js";
|
|
11
12
|
const DEFAULT_TTL_MS = 30 * 1000;
|
|
12
13
|
const DEFAULT_MAX_ENTRIES = 1000;
|
|
13
14
|
export function createTenantTimezoneCache(opts = {}) {
|
|
@@ -15,37 +16,17 @@ export function createTenantTimezoneCache(opts = {}) {
|
|
|
15
16
|
// 0 or negative would never evict (size < max is never true) and grow unbounded.
|
|
16
17
|
const maxEntries = Math.max(1, opts.maxEntries ?? DEFAULT_MAX_ENTRIES);
|
|
17
18
|
const now = opts.now ?? (() => Date.now());
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
const entries = createLruCache({
|
|
20
|
+
maxEntries,
|
|
21
|
+
ttlMs,
|
|
22
|
+
now,
|
|
23
|
+
});
|
|
20
24
|
let generation = 0;
|
|
21
|
-
function evictOldestIfFull() {
|
|
22
|
-
// skip: cache has room, nothing to evict
|
|
23
|
-
if (entries.size < maxEntries)
|
|
24
|
-
return;
|
|
25
|
-
const oldestKey = entries.keys().next().value;
|
|
26
|
-
// skip: defensive — maxEntries >= 1 guarantees a non-empty map here
|
|
27
|
-
if (oldestKey === undefined)
|
|
28
|
-
return;
|
|
29
|
-
entries.delete(oldestKey);
|
|
30
|
-
}
|
|
31
25
|
function set(tenantId, value) {
|
|
32
|
-
entries.
|
|
33
|
-
evictOldestIfFull();
|
|
34
|
-
entries.set(tenantId, { value, expiresAt: now() + ttlMs });
|
|
26
|
+
entries.set(tenantId, { value });
|
|
35
27
|
}
|
|
36
28
|
return {
|
|
37
|
-
get(tenantId)
|
|
38
|
-
const hit = entries.get(tenantId);
|
|
39
|
-
if (!hit)
|
|
40
|
-
return undefined;
|
|
41
|
-
if (hit.expiresAt <= now()) {
|
|
42
|
-
entries.delete(tenantId);
|
|
43
|
-
return undefined;
|
|
44
|
-
}
|
|
45
|
-
entries.delete(tenantId);
|
|
46
|
-
entries.set(tenantId, hit);
|
|
47
|
-
return { value: hit.value };
|
|
48
|
-
},
|
|
29
|
+
get: (tenantId) => entries.get(tenantId),
|
|
49
30
|
set,
|
|
50
31
|
generation() {
|
|
51
32
|
return generation;
|
package/dist/search/index.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
export { collectSearchableFieldNames } from "./configure-tenant-index.ts";
|
|
1
2
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.ts";
|
|
3
|
+
export { isEntitySearchable } from "./is-entity-searchable.ts";
|
|
2
4
|
export { purgeSearchDocumentsForSubject } from "./purge-subject.ts";
|
|
3
5
|
export { reindexEntity } from "./reindex-entity.ts";
|
|
6
|
+
export { type ReindexTenantsSearchOptions, type ReindexTenantsSearchResult, reindexTenantsSearch, } from "./reindex-tenants-search.ts";
|
|
4
7
|
export type { SearchAdapter, SearchAdapterConfig, } from "./types.ts";
|
package/dist/search/index.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
// `@cosmicdrift/kumiko-framework/search/meilisearch`. This keeps the main barrel from loading a
|
|
3
3
|
// Meilisearch client when consumers only use SearchAdapter types, so apps that do not use
|
|
4
4
|
// Meilisearch do not pull in the client code.
|
|
5
|
+
export { collectSearchableFieldNames } from "./configure-tenant-index.js";
|
|
5
6
|
export { createInMemorySearchAdapter } from "./in-memory-adapter.js";
|
|
7
|
+
export { isEntitySearchable } from "./is-entity-searchable.js";
|
|
6
8
|
export { purgeSearchDocumentsForSubject } from "./purge-subject.js";
|
|
7
9
|
export { reindexEntity } from "./reindex-entity.js";
|
|
10
|
+
export { reindexTenantsSearch, } from "./reindex-tenants-search.js";
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { DbRunner } from "../db/connection.ts";
|
|
2
|
+
import { type Registry, type TenantId } from "../engine/types/index.ts";
|
|
3
|
+
import type { SearchAdapter } from "./types.ts";
|
|
4
|
+
export type ReindexTenantsSearchOptions = {
|
|
5
|
+
readonly tenantIds: readonly TenantId[];
|
|
6
|
+
readonly timeoutMs?: number;
|
|
7
|
+
readonly configureTimeoutMs?: number;
|
|
8
|
+
readonly dryRun?: boolean;
|
|
9
|
+
readonly warn?: (message: string) => void;
|
|
10
|
+
};
|
|
11
|
+
export type ReindexTenantsSearchResult = {
|
|
12
|
+
readonly tenants: number;
|
|
13
|
+
readonly entities: number;
|
|
14
|
+
readonly indexedRows: number;
|
|
15
|
+
readonly failedRows: number;
|
|
16
|
+
readonly failedReindexes: number;
|
|
17
|
+
readonly failedConfigures: number;
|
|
18
|
+
};
|
|
19
|
+
export declare function reindexTenantsSearch(db: DbRunner, registry: Registry, searchAdapter: SearchAdapter, options: ReindexTenantsSearchOptions): Promise<ReindexTenantsSearchResult>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
|
|
2
|
+
import { createFallbackLogger } from "../logging/utils.js";
|
|
3
|
+
import { runWithTimeout } from "../utils/run-with-timeout.js";
|
|
4
|
+
import { collectSearchableFieldNames } from "./configure-tenant-index.js";
|
|
5
|
+
import { isEntitySearchable } from "./is-entity-searchable.js";
|
|
6
|
+
import { reindexEntity } from "./reindex-entity.js";
|
|
7
|
+
const defaultLog = createFallbackLogger("search");
|
|
8
|
+
const DEFAULT_CONFIGURE_TIMEOUT_MS = 5_000;
|
|
9
|
+
const DEFAULT_REINDEX_TIMEOUT_MS = 60_000;
|
|
10
|
+
function messageOf(error) {
|
|
11
|
+
return error instanceof Error ? error.message : String(error);
|
|
12
|
+
}
|
|
13
|
+
// Reindexing an unconfigured index falls back to searchableAttributes ["*"],
|
|
14
|
+
// so a tenant whose configure failed is reported and skipped, not fatal.
|
|
15
|
+
async function configureTolerant(tenantIds, registry, searchAdapter, timeoutMs, warn) {
|
|
16
|
+
const fields = collectSearchableFieldNames(registry);
|
|
17
|
+
const configured = [];
|
|
18
|
+
let failedConfigures = 0;
|
|
19
|
+
for (const tenantId of tenantIds) {
|
|
20
|
+
const outcome = await runWithTimeout(() => searchAdapter.configure(tenantId, { searchableFields: fields, rankingFields: fields }), timeoutMs);
|
|
21
|
+
if (outcome.ok) {
|
|
22
|
+
configured.push(tenantId);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
failedConfigures += 1;
|
|
26
|
+
warn(`[search] configure for tenant ${tenantId} failed: ${messageOf(outcome.error)}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return { configured, failedConfigures };
|
|
30
|
+
}
|
|
31
|
+
export async function reindexTenantsSearch(db, registry, searchAdapter, options) {
|
|
32
|
+
const warn = options.warn ?? ((message) => defaultLog.warn(message));
|
|
33
|
+
const reindexTimeoutMs = options.timeoutMs ?? DEFAULT_REINDEX_TIMEOUT_MS;
|
|
34
|
+
const dryRun = options.dryRun === true;
|
|
35
|
+
const { configured, failedConfigures } = dryRun
|
|
36
|
+
? { configured: [...options.tenantIds], failedConfigures: 0 }
|
|
37
|
+
: await configureTolerant(options.tenantIds, registry, searchAdapter, options.configureTimeoutMs ?? DEFAULT_CONFIGURE_TIMEOUT_MS, warn);
|
|
38
|
+
let entities = 0;
|
|
39
|
+
let indexedRows = 0;
|
|
40
|
+
let failedRows = 0;
|
|
41
|
+
let failedReindexes = 0;
|
|
42
|
+
for (const [entityName, entity] of registry.getAllEntities()) {
|
|
43
|
+
if (!isEntitySearchable(registry, entityName))
|
|
44
|
+
continue;
|
|
45
|
+
entities += 1;
|
|
46
|
+
const targetTenantIds = entity.systemStream === true ? [SYSTEM_TENANT_ID] : configured;
|
|
47
|
+
for (const tenantId of targetTenantIds) {
|
|
48
|
+
const outcome = await runWithTimeout(() => reindexEntity(db, registry, searchAdapter, entityName, tenantId, dryRun ? { dryRun } : {}), reindexTimeoutMs);
|
|
49
|
+
if (!outcome.ok) {
|
|
50
|
+
failedReindexes += 1;
|
|
51
|
+
warn(`[search] reindex "${entityName}" tenant=${tenantId} failed: ${messageOf(outcome.error)}`);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
indexedRows += outcome.value.indexedRows;
|
|
55
|
+
if (outcome.value.failures.length > 0) {
|
|
56
|
+
failedRows += outcome.value.failures.length;
|
|
57
|
+
warn(`[search] reindex "${entityName}" tenant=${tenantId}: ${outcome.value.failures.length} row(s) failed, first: ${outcome.value.failures[0]?.reason}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
tenants: options.tenantIds.length,
|
|
63
|
+
entities,
|
|
64
|
+
indexedRows,
|
|
65
|
+
failedRows,
|
|
66
|
+
failedReindexes,
|
|
67
|
+
failedConfigures,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function chunk<T>(items: readonly T[], size: number): T[][];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function bytesToHex(bytes) {
|
|
2
|
+
let out = "";
|
|
3
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
4
|
+
out += (bytes[i] ?? 0).toString(16).padStart(2, "0");
|
|
5
|
+
}
|
|
6
|
+
return out;
|
|
7
|
+
}
|
|
8
|
+
export async function digestHex(algorithm, text) {
|
|
9
|
+
const digest = await crypto.subtle.digest(algorithm, new TextEncoder().encode(text));
|
|
10
|
+
return bytesToHex(new Uint8Array(digest));
|
|
11
|
+
}
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export { assertUnreachable } from "./assert.ts";
|
|
2
|
+
export { chunk } from "./chunk.ts";
|
|
3
|
+
export { bytesToHex, type DigestAlgorithm, digestHex } from "./hex.ts";
|
|
2
4
|
export { generateDeterministicId, generateId } from "./ids.ts";
|
|
3
5
|
export { isPlainObject } from "./is-plain-object.ts";
|
|
6
|
+
export { createLruCache, type LruCache, type LruCacheOptions } from "./lru-cache.ts";
|
|
4
7
|
export { parseStringArrayJson } from "./parse-string-array-json.ts";
|
|
5
8
|
export { parseJsonOrThrow, parseJsonSafe } from "./safe-json.ts";
|
|
6
9
|
export { parseRoles } from "./serialization.ts";
|
package/dist/utils/index.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export { assertUnreachable } from "./assert.js";
|
|
2
|
+
export { chunk } from "./chunk.js";
|
|
3
|
+
export { bytesToHex, digestHex } from "./hex.js";
|
|
2
4
|
export { generateDeterministicId, generateId } from "./ids.js";
|
|
3
5
|
export { isPlainObject } from "./is-plain-object.js";
|
|
6
|
+
export { createLruCache } from "./lru-cache.js";
|
|
4
7
|
export { parseStringArrayJson } from "./parse-string-array-json.js";
|
|
5
8
|
export { parseJsonOrThrow, parseJsonSafe } from "./safe-json.js";
|
|
6
9
|
export { parseRoles } from "./serialization.js";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type LruCacheOptions<V> = {
|
|
2
|
+
readonly maxEntries: number;
|
|
3
|
+
/** Entries at least this old count as misses; omitted = no expiry. */
|
|
4
|
+
readonly ttlMs?: number;
|
|
5
|
+
readonly now?: () => number;
|
|
6
|
+
/** Summed `sizeOf` budget; a value larger than the whole budget is never cached. */
|
|
7
|
+
readonly maxTotalSize?: number;
|
|
8
|
+
readonly sizeOf?: (value: V) => number;
|
|
9
|
+
};
|
|
10
|
+
export type LruCache<K, V> = {
|
|
11
|
+
get(key: K): V | undefined;
|
|
12
|
+
set(key: K, value: V): void;
|
|
13
|
+
delete(key: K): void;
|
|
14
|
+
clear(): void;
|
|
15
|
+
readonly size: number;
|
|
16
|
+
};
|
|
17
|
+
export declare function createLruCache<K, V>(options: LruCacheOptions<V>): LruCache<K, V>;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// Map iteration is insertion order: re-inserting on a hit keeps the most recently used entry last.
|
|
2
|
+
export function createLruCache(options) {
|
|
3
|
+
const now = options.now ?? Date.now;
|
|
4
|
+
const sizeOf = options.sizeOf ?? (() => 0);
|
|
5
|
+
const entries = new Map();
|
|
6
|
+
let totalSize = 0;
|
|
7
|
+
function remove(key) {
|
|
8
|
+
const stored = entries.get(key);
|
|
9
|
+
// skip: key not cached, nothing to remove
|
|
10
|
+
if (stored === undefined)
|
|
11
|
+
return;
|
|
12
|
+
totalSize -= stored.size;
|
|
13
|
+
entries.delete(key);
|
|
14
|
+
}
|
|
15
|
+
function overBudget() {
|
|
16
|
+
if (entries.size > options.maxEntries)
|
|
17
|
+
return true;
|
|
18
|
+
return options.maxTotalSize !== undefined && totalSize > options.maxTotalSize;
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
get(key) {
|
|
22
|
+
const hit = entries.get(key);
|
|
23
|
+
if (hit === undefined)
|
|
24
|
+
return undefined;
|
|
25
|
+
remove(key);
|
|
26
|
+
if (options.ttlMs !== undefined && now() - hit.storedAt >= options.ttlMs)
|
|
27
|
+
return undefined;
|
|
28
|
+
entries.set(key, hit);
|
|
29
|
+
totalSize += hit.size;
|
|
30
|
+
return hit.value;
|
|
31
|
+
},
|
|
32
|
+
set(key, value) {
|
|
33
|
+
remove(key);
|
|
34
|
+
const size = sizeOf(value);
|
|
35
|
+
// skip: a value larger than the whole budget would evict everything and still not fit
|
|
36
|
+
if (options.maxTotalSize !== undefined && size > options.maxTotalSize)
|
|
37
|
+
return;
|
|
38
|
+
entries.set(key, { value, storedAt: now(), size });
|
|
39
|
+
totalSize += size;
|
|
40
|
+
while (overBudget()) {
|
|
41
|
+
const oldest = entries.keys().next();
|
|
42
|
+
if (oldest.done)
|
|
43
|
+
break;
|
|
44
|
+
remove(oldest.value);
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
delete: remove,
|
|
48
|
+
clear() {
|
|
49
|
+
entries.clear();
|
|
50
|
+
totalSize = 0;
|
|
51
|
+
},
|
|
52
|
+
get size() {
|
|
53
|
+
return entries.size;
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type TimedRunResult<T> = {
|
|
2
|
+
readonly ok: true;
|
|
3
|
+
readonly value: T;
|
|
4
|
+
readonly latencyMs: number;
|
|
5
|
+
} | {
|
|
6
|
+
readonly ok: false;
|
|
7
|
+
readonly error: unknown;
|
|
8
|
+
readonly timedOut: boolean;
|
|
9
|
+
readonly latencyMs: number;
|
|
10
|
+
};
|
|
11
|
+
export declare function runWithTimeout<T>(run: () => Promise<T>, timeoutMs: number): Promise<TimedRunResult<T>>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
class TimeoutError extends Error {
|
|
2
|
+
}
|
|
3
|
+
export async function runWithTimeout(run, timeoutMs) {
|
|
4
|
+
const start = performance.now();
|
|
5
|
+
const elapsedMs = () => Math.round(performance.now() - start);
|
|
6
|
+
let timer;
|
|
7
|
+
try {
|
|
8
|
+
const value = await Promise.race([
|
|
9
|
+
run(),
|
|
10
|
+
new Promise((_, reject) => {
|
|
11
|
+
timer = setTimeout(() => reject(new TimeoutError(`timeout after ${timeoutMs}ms`)), timeoutMs);
|
|
12
|
+
// Don't keep the event loop alive on a hung run during shutdown.
|
|
13
|
+
timer.unref?.();
|
|
14
|
+
}),
|
|
15
|
+
]);
|
|
16
|
+
return { ok: true, value, latencyMs: elapsedMs() };
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
return {
|
|
20
|
+
ok: false,
|
|
21
|
+
error,
|
|
22
|
+
timedOut: error instanceof TimeoutError,
|
|
23
|
+
latencyMs: elapsedMs(),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
finally {
|
|
27
|
+
if (timer)
|
|
28
|
+
clearTimeout(timer);
|
|
29
|
+
}
|
|
30
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.406.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -203,10 +203,10 @@
|
|
|
203
203
|
"./package.json": "./package.json"
|
|
204
204
|
},
|
|
205
205
|
"dependencies": {
|
|
206
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
206
|
+
"@cosmicdrift/kumiko-types": "0.406.0",
|
|
207
207
|
"bullmq": "^5.81.5",
|
|
208
208
|
"hono": "^4.13.14",
|
|
209
|
-
"i18next": "^26.5.
|
|
209
|
+
"i18next": "^26.5.12",
|
|
210
210
|
"ioredis": "^5.11.1",
|
|
211
211
|
"jose": "^6.2.14",
|
|
212
212
|
"pino": "^10.4.0",
|
|
@@ -225,7 +225,7 @@
|
|
|
225
225
|
}
|
|
226
226
|
},
|
|
227
227
|
"devDependencies": {
|
|
228
|
-
"@cosmicdrift/kumiko-testing": "0.
|
|
228
|
+
"@cosmicdrift/kumiko-testing": "0.406.0",
|
|
229
229
|
"bun-types": "^1.4.4",
|
|
230
230
|
"fflate": "^0.8.3",
|
|
231
231
|
"meilisearch": "^0.62.0",
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,52 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.406.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "pending flag on TenantDestroyHookResult",
|
|
6
|
+
"detail": "A destroy hook may return { done: false, processed: 0, pending: true } when it waits on an external operation it has already started, for example a remote teardown job. The field is optional, so existing hooks keep compiling and keep their behavior."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.406.0",
|
|
10
|
+
"type": "improvement",
|
|
11
|
+
"title": "wakesSleepingApp false on cron and maintenance triggers",
|
|
12
|
+
"detail": "A job declared with trigger { cron, wakesSleepingApp: false } or { maintenance: true, wakesSleepingApp: false } is scheduled under the BullMQ scheduler id scheduler-nowake-<job> instead of scheduler-<job>, so its delayed jobs are repeat:scheduler-nowake-<job>:<ts> and the waker of sleeping shared apps skips them; the run happens on the next wake. Leaving the flag out keeps waking the app. Any other value, or the flag on an event or manual trigger, stops the registration. On start the runner removes the scheduler the job had under its other id at once, without the orphan grace period, so flipping the flag never leaves the old scheduler firing."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.405.0",
|
|
16
|
+
"type": "improvement",
|
|
17
|
+
"title": "chunk in framework utils",
|
|
18
|
+
"detail": "@cosmicdrift/kumiko-framework/utils exports chunk(items, size), which splits an array into slices of at most size elements. files-tenant-data, subscription-paddle and template-resolver use it instead of local copies."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"version": "0.405.0",
|
|
22
|
+
"type": "improvement",
|
|
23
|
+
"title": "Search sweep helpers in framework/search",
|
|
24
|
+
"detail": "@cosmicdrift/kumiko-framework/search exports collectSearchableFieldNames, isEntitySearchable and reindexTenantsSearch(db, registry, adapter, { tenantIds, timeoutMs?, configureTimeoutMs?, dryRun?, warn? }). The sweep configures each given tenant index, then reindexes every searchable entity for those tenants (system-stream entities once under the system tenant) and returns counts of indexed rows and failures. A failed configure or reindex is reported through warn and does not stop the sweep. Apps no longer need their own configure-on-boot seed: the server's default search config is applied lazily to a missing or factory-default index."
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"version": "0.405.0",
|
|
28
|
+
"type": "improvement",
|
|
29
|
+
"title": "createLruCache in framework utils",
|
|
30
|
+
"detail": "@cosmicdrift/kumiko-framework/utils exports createLruCache({ maxEntries, ttlMs?, now?, maxTotalSize?, sizeOf? }) with get, set, delete, clear and size. The tenant timezone cache and the managed-pages body cache use it. runWithTimeout moves to the utils module; the api barrel export stays."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"version": "0.405.0",
|
|
34
|
+
"type": "improvement",
|
|
35
|
+
"title": "Engine and db helpers deduplicated",
|
|
36
|
+
"detail": "The engine uses one findEntity lookup instead of three copies, the PII subject resolver and the boot validator use isFileField, insertOne builds its SQL through the same insert helpers as insertMany, the projection rebuild maps event rows through the shared toStoredEvent, and the app-schema role projection filters row, toolbar and related-list actions through one helper. No behavior change."
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"version": "0.405.0",
|
|
40
|
+
"type": "improvement",
|
|
41
|
+
"title": "One key split for file variants and derivatives",
|
|
42
|
+
"detail": "deriveKey and the derivative key helpers split a storage key into base and extension through the same function, so both always agree on where the extension starts. No behavior change."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"version": "0.405.0",
|
|
46
|
+
"type": "improvement",
|
|
47
|
+
"title": "Opaque-token hash and hex helpers exported",
|
|
48
|
+
"detail": "@cosmicdrift/kumiko-framework/crypto exports hashOpaqueToken (async, Web Crypto), hashOpaqueTokenSync (node:crypto, for synchronous server paths) and opaqueTokenHashMatches (timing-safe comparison against a stored SHA-256 hex hash). @cosmicdrift/kumiko-framework/utils exports bytesToHex and digestHex(algorithm, text). The framework's own token hashes (personal access tokens, single-use token store, preauth MFA key, export download tokens, termination revocation) use them and produce the same hex as before. Apps that hash share or heartbeat tokens themselves can switch to these helpers without migrating stored hashes."
|
|
49
|
+
},
|
|
2
50
|
{
|
|
3
51
|
"version": "0.404.0",
|
|
4
52
|
"type": "improvement",
|