@cosmicdrift/kumiko-server-runtime 0.406.0 → 0.409.0

package/dist/boot/search-reindex-on-boot.d.ts

@@ -0,0 +1,34 @@
+import { type DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import type { Registry } from "@cosmicdrift/kumiko-framework/engine";
+import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
+import { type SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
+export declare const SKIP_SEARCH_REINDEX_ENV = "KUMIKO_SKIP_SEARCH_REINDEX";
+export declare const SEARCH_DOCUMENT_FORMAT_VERSION = 2;
+export declare const SEARCH_REINDEX_LOCK_KEY = 1936876392;
+type Log = ReturnType<typeof createFallbackLogger>;
+export type SearchReindexOutcome = {
+    /** A peer holds the lock and does the work. */
+    readonly skippedBecauseLocked: boolean;
+    readonly reindexedEntities: readonly string[];
+    readonly failedEntities: readonly string[];
+};
+/** Changes when the search fields of the entity or the document format change. */
+export declare function searchReindexFingerprint(registry: Registry, entityName: string): string;
+export declare function runSearchReindexOnBoot(opts: {
+    readonly db: DbConnection;
+    readonly registry: Registry;
+    readonly searchAdapter: SearchAdapter;
+    readonly signal?: AbortSignal;
+    readonly log?: Log;
+}): Promise<SearchReindexOutcome>;
+type ShutdownRegistry = {
+    registerShutdownHook(name: string, fn: () => Promise<void>): void;
+};
+export declare function startSearchReindexOnBoot(opts: {
+    readonly db: DbConnection;
+    readonly registry: Registry;
+    readonly extraContext: object;
+    readonly lifecycle: ShutdownRegistry;
+    readonly envSource: Readonly<Record<string, string | undefined>>;
+}): void;
+export {};

package/dist/boot/search-reindex-on-boot.js

@@ -0,0 +1,139 @@
+import { createHash } from "node:crypto";
+import { resolveBaseDbAndSearchAdapter } from "@cosmicdrift/kumiko-framework/api";
+import { fetchOne, insertOne, listEventTenantIds, tryWithSessionAdvisoryLock, } from "@cosmicdrift/kumiko-framework/db";
+import { createEsOperationsTable, esOperationsTable } from "@cosmicdrift/kumiko-framework/es-ops";
+import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
+import { reindexEntity } from "@cosmicdrift/kumiko-framework/search";
+export const SKIP_SEARCH_REINDEX_ENV = "KUMIKO_SKIP_SEARCH_REINDEX";
+// Bump when the way documents are built or stored in the index changes (2: long fields are
+// split into pieces), so every searchable entity is indexed again once.
+export const SEARCH_DOCUMENT_FORMAT_VERSION = 2;
+export const SEARCH_REINDEX_LOCK_KEY = 0x73_72_63_68; // 'srch'
+function isAborted(signal) {
+    return signal?.aborted === true;
+}
+function searchedEntityNames(registry) {
+    return [...registry.getAllEntities().keys()].filter((name) => registry.getSearchableFields(name).length > 0 ||
+        registry.getSearchPayloadExtensions(name).length > 0);
+}
+/** Changes when the search fields of the entity or the document format change. */
+export function searchReindexFingerprint(registry, entityName) {
+    const fields = [...registry.getSearchableFields(entityName)].sort();
+    const extensionCount = registry.getSearchPayloadExtensions(entityName).length;
+    return createHash("sha256")
+        .update(`${SEARCH_DOCUMENT_FORMAT_VERSION}|${fields.join(",")}|${extensionCount}`)
+        .digest("hex")
+        .slice(0, 16);
+}
+// The completion marker is a row in kumiko_es_operations, keyed by entity and fingerprint: a
+// changed fingerprint is a missing marker, and the row is only written after a clean run.
+function markerId(entityName, fingerprint) {
+    return `search-reindex:${entityName}:${fingerprint}`;
+}
+async function hasMarker(db, id) {
+    return (await fetchOne(db, esOperationsTable, { id })) !== undefined;
+}
+async function pendingEntities(db, registry) {
+    const pending = [];
+    for (const name of searchedEntityNames(registry)) {
+        if (!(await hasMarker(db, markerId(name, searchReindexFingerprint(registry, name))))) {
+            pending.push(name);
+        }
+    }
+    return pending;
+}
+export async function runSearchReindexOnBoot(opts) {
+    const { db, registry, searchAdapter, signal } = opts;
+    const log = opts.log ?? createFallbackLogger("search-reindex");
+    await createEsOperationsTable(db);
+    // Cheap exit for the common boot, without taking the lock.
+    if ((await pendingEntities(db, registry)).length === 0) {
+        return { skippedBecauseLocked: false, reindexedEntities: [], failedEntities: [] };
+    }
+    const locked = await tryWithSessionAdvisoryLock(db, SEARCH_REINDEX_LOCK_KEY, async () => {
+        // A peer may have finished between the check above and the lock.
+        const pending = await pendingEntities(db, registry);
+        const tenantIds = pending.length > 0 ? await listEventTenantIds(db) : [];
+        const reindexed = [];
+        const failed = [];
+        for (const entityName of pending) {
+            if (isAborted(signal))
+                break;
+            let clean = true;
+            for (const tenantId of tenantIds) {
+                if (isAborted(signal)) {
+                    clean = false;
+                    break;
+                }
+                try {
+                    const result = await reindexEntity(db, registry, searchAdapter, entityName, tenantId);
+                    if (result.failures.length > 0) {
+                        clean = false;
+                        log.error("search reindex had failing rows; it is retried on the next boot", {
+                            entityName,
+                            tenantId,
+                            failures: result.failures.length,
+                            firstReason: result.failures[0]?.reason,
+                        });
+                    }
+                }
+                catch (e) {
+                    clean = false;
+                    log.error("search reindex failed; it is retried on the next boot", {
+                        entityName,
+                        tenantId,
+                        error: e instanceof Error ? e.message : String(e),
+                    });
+                }
+            }
+            if (!clean) {
+                failed.push(entityName);
+                continue;
+            }
+            await insertOne(db, esOperationsTable, {
+                id: markerId(entityName, searchReindexFingerprint(registry, entityName)),
+                operationType: "search-reindex",
+                durationMs: 0,
+                appliedBy: "boot",
+            });
+            reindexed.push(entityName);
+        }
+        return { reindexed, failed };
+    });
+    if (!locked.acquired) {
+        return { skippedBecauseLocked: true, reindexedEntities: [], failedEntities: [] };
+    }
+    return {
+        skippedBecauseLocked: false,
+        reindexedEntities: locked.value.reindexed,
+        failedEntities: locked.value.failed,
+    };
+}
+// Background like the PII backfill: a reindex scales with the data and must not delay readiness.
+export function startSearchReindexOnBoot(opts) {
+    const { searchAdapter } = resolveBaseDbAndSearchAdapter(opts.extraContext, opts.registry);
+    // skip: no search backend in this process, or the operator opted out
+    if (searchAdapter === undefined || opts.envSource[SKIP_SEARCH_REINDEX_ENV] === "1")
+        return;
+    const log = createFallbackLogger("search-reindex");
+    const abortController = new AbortController();
+    const run = runSearchReindexOnBoot({
+        db: opts.db,
+        registry: opts.registry,
+        searchAdapter,
+        signal: abortController.signal,
+        log,
+    }).then((outcome) => {
+        if (outcome.reindexedEntities.length > 0) {
+            log.info("search reindex finished", { entities: outcome.reindexedEntities });
+        }
+    }, (e) => {
+        log.error("search reindex crashed; it is retried on the next boot", {
+            error: e instanceof Error ? e.message : String(e),
+        });
+    });
+    opts.lifecycle.registerShutdownHook("searchReindex", async () => {
+        abortController.abort();
+        await run;
+    });
+}

package/dist/index.d.ts

@@ -24,3 +24,4 @@ export { runWorkerApp } from "./run-worker-app.ts";
 export type { CspExtension, CspOption, SecurityHeadersOption } from "./security-headers.ts";
 export { buildDefaultCsp } from "./security-headers.ts";
 export { makeSeedCreateFileRef, type SeedCreateFileRef } from "./seed-dispatcher.ts";
+export { type StaticHtmlPage, writeStaticHtmlPages } from "./static-html-pages.ts";

package/dist/index.js

@@ -18,3 +18,4 @@ export { parseRunMode, requireEnv, runProdApp } from "./run-prod-app.js";
 export { runWorkerApp } from "./run-worker-app.js";
 export { buildDefaultCsp } from "./security-headers.js";
 export { makeSeedCreateFileRef } from "./seed-dispatcher.js";
+export { writeStaticHtmlPages } from "./static-html-pages.js";

package/dist/run-prod-app-entrypoint.d.ts

@@ -11,4 +11,4 @@ export type ProdEntrypointOptions = ReturnType<typeof buildProdEntrypointOptions
 export declare function createProdEntrypoint(options: RunProdAppOptions, config: ProdBootConfig, composed: ProdComposedFeatures, connections: ProdConnections, entrypointOptions: ProdEntrypointOptions): ProdEntrypoint;
 export declare function registerProdShutdownHooks(entrypoint: ProdEntrypoint, cacheSync: ProdConnections["cacheSync"], ownedRedisSseBroker: ProdRequestContext["ownedRedisSseBroker"]): void;
 export declare function runProdWireHook(options: RunProdAppOptions, config: ProdBootConfig, composed: ProdComposedFeatures, connections: ProdConnections, entrypoint: ProdEntrypoint): Promise<void>;
-export declare function startProdBackgroundWork(entrypoint: ProdEntrypoint, config: ProdBootConfig, composed: ProdComposedFeatures, connections: ProdConnections): Promise<void>;
+export declare function startProdBackgroundWork(entrypoint: ProdEntrypoint, config: ProdBootConfig, composed: ProdComposedFeatures, connections: ProdConnections, extraContext: object): Promise<void>;

package/dist/run-prod-app-entrypoint.js

@@ -4,6 +4,7 @@ import { createMetricsForFactory } from "@cosmicdrift/kumiko-framework/observabi
 import { jobRunLoggerCallbacks } from "./boot/job-run-logger.js";
 import { startPiiEventBackfillOnBoot } from "./boot/pii-event-backfill-on-boot.js";
 import { recordAppStartedOnBoot } from "./boot/record-app-started-on-boot.js";
+import { startSearchReindexOnBoot } from "./boot/search-reindex-on-boot.js";
 import { makeDispatchSystemQuery, makeDispatchSystemWrite } from "./extra-routes-deps.js";
 import { makeFilesForTenant } from "./files-for-tenant.js";
 import { buildProdAuthRoutes } from "./run-prod-app-auth.js";
@@ -140,7 +141,7 @@ export async function runProdWireHook(options, config, composed, connections, en
         });
     }
 }
-export async function startProdBackgroundWork(entrypoint, config, composed, connections) {
+export async function startProdBackgroundWork(entrypoint, config, composed, connections, extraContext) {
     const { envSource } = config;
     await entrypoint.start();
     await recordAppStartedOnBoot({ db: connections.db, envSource });
@@ -150,4 +151,11 @@ export async function startProdBackgroundWork(entrypoint, config, composed, conn
         lifecycle: entrypoint.lifecycle,
         envSource,
     });
+    startSearchReindexOnBoot({
+        db: connections.db,
+        registry: composed.registry,
+        extraContext,
+        lifecycle: entrypoint.lifecycle,
+        envSource,
+    });
 }

package/dist/run-prod-app-static-files.js

@@ -125,6 +125,8 @@ export function mimeTypeFor(filePath) {
             return "image/svg+xml";
         case "png":
             return "image/png";
+        case "webp":
+            return "image/webp";
         case "jpg":
         case "jpeg":
             return "image/jpeg";

package/dist/run-prod-app.js

@@ -142,7 +142,7 @@ async function bootProdApp(options, registerAbortCleanup) {
     registerProdShutdownHooks(entrypoint, connections.cacheSync, requestContext.ownedRedisSseBroker);
     await runProdWireHook(options, config, composed, connections, entrypoint);
     await runProdBootSeeds({ options, config, composed, connections, requestContext }, entrypoint, entrypointOptions.dispatcherOptions);
-    await startProdBackgroundWork(entrypoint, config, composed, connections);
+    await startProdBackgroundWork(entrypoint, config, composed, connections, requestContext.extraContext);
     const fetch = await buildProdFetchHandler(options, config, entrypoint);
     // Mark lifecycle ready — health/ready flips to 200 after this.
     entrypoint.lifecycle.markReady();

package/dist/run-worker-app.js

@@ -22,6 +22,7 @@ import { logRedisConnectionState } from "@cosmicdrift/kumiko-framework/redis";
 import { runEnvGate } from "./boot/env-gate.js";
 import { startPiiEventBackfillOnBoot } from "./boot/pii-event-backfill-on-boot.js";
 import { recordAppStartedOnBoot } from "./boot/record-app-started-on-boot.js";
+import { startSearchReindexOnBoot } from "./boot/search-reindex-on-boot.js";
 import { runStreamTenantBackfillOnBoot, } from "./boot/stream-tenant-backfill-on-boot.js";
 import { bootWorkerProcess, resolveWorkerEnvSource, } from "./boot/worker-boot-core.js";
 import { startWorkerMetricsServer, } from "./boot/worker-metrics-server.js";
@@ -139,6 +140,13 @@ export async function runWorkerApp(options) {
         lifecycle: entrypoint.lifecycle,
         envSource,
     });
+    startSearchReindexOnBoot({
+        db,
+        registry,
+        extraContext: boot.extraContext,
+        lifecycle: entrypoint.lifecycle,
+        envSource,
+    });
     if (options.wireComponents) {
         await options.wireComponents({
             db: wireHookDb(db, options),

package/dist/static-html-pages.d.ts

@@ -0,0 +1,5 @@
+export type StaticHtmlPage = {
+    readonly path: string;
+    readonly html: string;
+};
+export declare function writeStaticHtmlPages(outDir: string, pages: readonly StaticHtmlPage[]): Promise<void>;

package/dist/static-html-pages.js

@@ -0,0 +1,19 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join } from "node:path";
+function assertSafePagePath(path) {
+    if (isAbsolute(path) || path.startsWith("/") || path.includes("\\")) {
+        throw new Error(`writeStaticHtmlPages: page path "${path}" must be relative and use "/" only`);
+    }
+    if (path.split("/").includes("..")) {
+        throw new Error(`writeStaticHtmlPages: page path "${path}" must not contain ".." segments`);
+    }
+}
+export async function writeStaticHtmlPages(outDir, pages) {
+    for (const page of pages)
+        assertSafePagePath(page.path);
+    for (const page of pages) {
+        const file = join(outDir, page.path, "index.html");
+        await mkdir(dirname(file), { recursive: true });
+        await writeFile(file, page.html, "utf-8");
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-server-runtime",
-  "version": "0.406.0",
+  "version": "0.409.0",
   "description": "Production server-boot runtime for Kumiko apps: connections, schema-drift-gate, seeds, lifecycle, graceful shutdown. Symmetric to kumiko-dev-server's runDevApp, without dev/scaffold tooling.",
   "license": "MIT",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -96,11 +96,11 @@
     }
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-bundled-features": "0.406.0",
-    "@cosmicdrift/kumiko-framework": "0.406.0",
-    "@cosmicdrift/kumiko-renderer": "0.406.0",
-    "@cosmicdrift/kumiko-renderer-mail-html": "0.406.0",
-    "@cosmicdrift/kumiko-types": "0.406.0",
+    "@cosmicdrift/kumiko-bundled-features": "0.409.0",
+    "@cosmicdrift/kumiko-framework": "0.409.0",
+    "@cosmicdrift/kumiko-renderer": "0.409.0",
+    "@cosmicdrift/kumiko-renderer-mail-html": "0.409.0",
+    "@cosmicdrift/kumiko-types": "0.409.0",
     "ioredis": "^5.11.1",
     "temporal-polyfill": "^0.3.2",
     "zod": "^4.6.7"
@@ -109,7 +109,7 @@
     "bun-types": "^1.4.4"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-testing": "0.406.0",
+    "@cosmicdrift/kumiko-testing": "0.409.0",
     "bun-types": "^1.4.4"
   },
   "publishConfig": {

package/src/changes.json

@@ -1,4 +1,16 @@
 [
+  {
+    "version": "0.409.0",
+    "type": "improvement",
+    "title": "Boot-time search reindex when search fields or the document format change",
+    "detail": "runProdApp and runWorkerApp start a background reindex (reindexEntity for every tenant that owns events) for each searchable entity whose search fingerprint, the sorted search fields plus the document format version 2, has no completion marker yet. The marker is a kumiko_es_operations row written only after a run without failing rows, so an aborted or failed run is retried on the next boot. A Postgres advisory lock lets one process run it while the others skip, and boot readiness does not wait for it. It needs a configured SearchAdapter in the process; KUMIKO_SKIP_SEARCH_REINDEX=1 opts out."
+  },
+  {
+    "version": "0.408.0",
+    "type": "improvement",
+    "title": "writeStaticHtmlPages and webp content type",
+    "detail": "writeStaticHtmlPages(outDir, pages) from @cosmicdrift/kumiko-server-runtime writes each { path, html } to <outDir>/<path>/index.html (an empty path writes <outDir>/index.html) and creates the directories. Before writing anything it rejects absolute paths, backslashes and \"..\" segments. Apps that render marketing pages per locale into dist/pages can use it instead of their own node:fs code. The prod static file server now answers .webp files with content-type image/webp instead of application/octet-stream."
+  },
   {
     "version": "0.404.0",
     "type": "improvement",