@coffer-org/server 1.11.0 → 1.12.0

package/dist/batch.d.ts

@@ -0,0 +1,2 @@
+export declare const IN_CHUNK = 500;
+export declare function chunk<T>(items: T[], size?: number): T[][];

package/dist/batch.js

@@ -0,0 +1,9 @@
+export const IN_CHUNK = 500;
+export function chunk(items, size = IN_CHUNK) {
+    if (items.length <= size)
+        return items.length ? [items] : [];
+    const out = [];
+    for (let i = 0; i < items.length; i += size)
+        out.push(items.slice(i, i + size));
+    return out;
+}

package/dist/collection-io.d.ts

@@ -9,6 +9,7 @@ export declare function splitAt(fields: LayoutEl[], input: Row): {
 };
 export declare function writeAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number, collections: Record<string, Row[]>): Promise<void>;
 export declare function readAt(em: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<Record<string, Row[]>>;
+export declare function readAtMany(em: EntityManager, prefix: string, fields: LayoutEl[], parentIds: number[]): Promise<Map<number, Record<string, Row[]>>>;
 export declare function deleteAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<void>;
 export declare const splitCollections: (m: ShelfDef, input: Row) => {
     base: Row;

package/dist/collection-io.js

@@ -1,6 +1,7 @@
 import { serialize } from '@mikro-orm/core';
 import { collectionGroups, fieldEntries, storageColumns } from '@coffer-org/sdk/shelf';
 import { isGroup, isEmbeddedGroup, isJsonStored } from '@coffer-org/sdk/fields';
+import { chunk } from "./batch.js";
 function flattenLayout(fields) {
     const out = [];
     for (const f of fields) {
@@ -78,6 +79,34 @@ export async function readAt(em, prefix, fields, parentId) {
     }
     return out;
 }
+export async function readAtMany(em, prefix, fields, parentIds) {
+    const out = new Map();
+    for (const id of parentIds)
+        out.set(id, {});
+    if (parentIds.length === 0)
+        return out;
+    for (const c of collectionGroups(fields)) {
+        const table = `${prefix}__${c.key}`;
+        const childFields = c.group.fields;
+        const rows = [];
+        for (const ids of chunk(parentIds)) {
+            rows.push(...serialize(await em.find(table, { parent_id: { $in: ids } }, {
+                orderBy: { position: 'asc' },
+            })));
+        }
+        const nestedByChild = await readAtMany(em, table, childFields, rows.map((r) => r.id));
+        for (const id of parentIds)
+            out.get(id)[c.key] = [];
+        for (const row of rows) {
+            const { id, parent_id: pid, position: _pos, ...rest } = row;
+            const bucket = out.get(pid);
+            if (!bucket)
+                continue;
+            bucket[c.key].push({ ...nestEmbeddedAt(childFields, rest), ...(nestedByChild.get(id) ?? {}) });
+        }
+    }
+    return out;
+}
 export async function deleteAt(tx, prefix, fields, parentId) {
     for (const c of collectionGroups(fields)) {
         const table = `${prefix}__${c.key}`;

package/dist/extend-table.d.ts

@@ -3,6 +3,7 @@ import type { ExtendDef } from '@coffer-org/sdk/extend';
 type Row = Record<string, unknown>;
 export declare function extendEntityName(e: ExtendDef): string;
 export declare function getExtendRecord(e: ExtendDef, baseId: number, em?: EntityManager): Promise<Row | undefined>;
+export declare function getExtendRecords(e: ExtendDef, baseIds: number[], em?: EntityManager): Promise<Map<number, Row>>;
 export declare function upsertExtendRecord(em: EntityManager, e: ExtendDef, baseId: number, data: Row): Promise<void>;
 export declare function deleteExtendRecord(em: EntityManager, e: ExtendDef, baseId: number): Promise<void>;
 export {};

package/dist/extend-table.js

@@ -1,6 +1,7 @@
 import { serialize } from '@mikro-orm/core';
 import { getEm } from "./db.js";
-import { splitAt, writeAt, readAt, deleteAt } from "./collection-io.js";
+import { splitAt, writeAt, readAt, readAtMany, deleteAt } from "./collection-io.js";
+import { chunk } from "./batch.js";
 export function extendEntityName(e) {
     return `extend__${e.id}`;
 }
@@ -14,6 +15,26 @@ export async function getExtendRecord(e, baseId, em) {
     const collections = await readAt(fork, name, e.fields, baseId);
     return { ...flat, ...collections };
 }
+export async function getExtendRecords(e, baseIds, em) {
+    const out = new Map();
+    if (baseIds.length === 0)
+        return out;
+    const fork = em ?? getEm().fork();
+    const name = extendEntityName(e);
+    const flats = [];
+    for (const ids of chunk(baseIds)) {
+        flats.push(...serialize(await fork.find(name, { base_id: { $in: ids } })));
+    }
+    if (flats.length === 0)
+        return out;
+    const presentIds = flats.map((r) => Number(r.base_id));
+    const collectionsById = await readAtMany(fork, name, e.fields, presentIds);
+    for (const flat of flats) {
+        const id = Number(flat.base_id);
+        out.set(id, { ...flat, ...(collectionsById.get(id) ?? {}) });
+    }
+    return out;
+}
 export async function upsertExtendRecord(em, e, baseId, data) {
     const name = extendEntityName(e);
     const { base, collections } = splitAt(e.fields, data);

package/dist/index.js

@@ -1,4 +1,4 @@
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { createWriteStream, existsSync } from 'node:fs';
 import { realpath } from 'node:fs/promises';
 import { pipeline } from 'node:stream/promises';
@@ -9,7 +9,7 @@ import cors from '@fastify/cors';
 import multipart from '@fastify/multipart';
 import fastifyStatic from '@fastify/static';
 import { serialize } from '@mikro-orm/core';
-import { textSearchKeys, recordTitle } from '@coffer-org/sdk/shelf';
+import { textSearchKeys, recordTitle, titleKey, storageColumnsFor } from '@coffer-org/sdk/shelf';
 import { tokenize } from '@coffer-org/core/search';
 import { shelfTableName } from "./entity-schema.js";
 import { getEm, closeDb } from "./db.js";
@@ -133,7 +133,18 @@ app.get('/health', async (_req, reply) => {
         return reply.code(503).send({ status: 'db_unavailable' });
     }
 });
-app.get('/api/libraries', () => buildClientSchema());
+let librariesPayload = null;
+app.get('/api/libraries', async (req, reply) => {
+    if (!librariesPayload) {
+        const body = JSON.stringify(buildClientSchema());
+        librariesPayload = { body, etag: `"${createHash('sha1').update(body).digest('base64url')}"` };
+    }
+    reply.header('ETag', librariesPayload.etag);
+    reply.header('Cache-Control', 'no-cache');
+    if (req.headers['if-none-match'] === librariesPayload.etag)
+        return reply.code(304).send();
+    return reply.type('application/json').send(librariesPayload.body);
+});
 app.get('/api/counts', async () => {
     const fork = getEm().fork();
     const out = {};
@@ -147,6 +158,8 @@ app.get('/api/counts', async () => {
     }
     return out;
 });
+const SEARCH_SCAN_LIMIT = 1000;
+const searchLog = getLogger('search');
 app.get('/api/search', async (req) => {
     const { q = '', limit = '20' } = req.query;
     const tokens = tokenize(q);
@@ -164,7 +177,11 @@ app.get('/api/search', async (req) => {
         if (!textSearchKeys(mdef).length)
             continue;
         const [library, type] = k.split('/');
-        const rows = (await fork.find(ename, {}, { limit: 1000 })).map((r) => serialize(r));
+        const cols = storageColumnsFor(mdef, [...textSearchKeys(mdef), titleKey(mdef)]);
+        const rows = (await fork.find(ename, {}, { limit: SEARCH_SCAN_LIMIT, fields: cols })).map((r) => serialize(r));
+        if (rows.length === SEARCH_SCAN_LIMIT) {
+            searchLog.warn('scan cap reached', { shelf: k, limit: SEARCH_SCAN_LIMIT });
+        }
         for (const row of rows) {
             const m = rowMatch(mdef, row, tokens);
             if (!m)
@@ -363,9 +380,12 @@ function maskRecordRow(library, type, row) {
 }
 app.get('/api/:library/:type', async (req, reply) => {
     const { library, type } = req.params;
-    const query = req.query;
+    const { _fields, _extends, ...query } = req.query;
     try {
-        const rows = await recordList(library, type, query);
+        const rows = await recordList(library, type, query, {
+            view: _fields === 'full' ? 'full' : 'list',
+            extends: _extends === '1',
+        });
         return rows.map((r) => maskRecordRow(library, type, r));
     }
     catch (e) {
@@ -456,12 +476,31 @@ await registerAuthApi(app);
 registerOAuthApi(app);
 await registerMcpHttp(app);
 await registerPluginsApi(app);
+const IMMUTABLE = 'public, max-age=31536000, immutable';
+const isHashedAsset = (p) => /-[A-Za-z0-9_-]{8,}\.(js|css)$/.test(p);
 const WEB_DIST = process.env.WEB_DIST;
 if (WEB_DIST && existsSync(join(WEB_DIST, 'index.html'))) {
-    await app.register(fastifyStatic, { root: WEB_DIST, prefix: '/', decorateReply: false });
+    await app.register(fastifyStatic, {
+        root: WEB_DIST,
+        prefix: '/',
+        decorateReply: false,
+        cacheControl: false,
+        setHeaders(res, path) {
+            const hashed = path.includes(`${sep}assets${sep}`) || isHashedAsset(path);
+            res.setHeader('Cache-Control', hashed ? IMMUTABLE : 'no-cache');
+        },
+    });
     const VENDOR_DIST = join(WEB_DIST, '..', 'dist-vendor');
     if (existsSync(VENDOR_DIST)) {
-        await app.register(fastifyStatic, { root: VENDOR_DIST, prefix: '/vendor/', decorateReply: false });
+        await app.register(fastifyStatic, {
+            root: VENDOR_DIST,
+            prefix: '/vendor/',
+            decorateReply: false,
+            cacheControl: false,
+            setHeaders(res, path) {
+                res.setHeader('Cache-Control', isHashedAsset(path) ? IMMUTABLE : 'no-cache');
+            },
+        });
     }
     app.setNotFoundHandler((req, reply) => {
         if (req.method === 'GET' &&

package/dist/plugin-discovery.d.ts

@@ -10,7 +10,7 @@ export interface PluginAssetUrls {
     web?: string;
     css?: string;
 }
-export declare function resolveAssetUrls(id: string, coffer: PluginAssetPaths): PluginAssetUrls;
+export declare function resolveAssetUrls(id: string, coffer: PluginAssetPaths, version?: string): PluginAssetUrls;
 export interface PluginAssetRecord {
     id: string;
     version: string;
@@ -21,6 +21,7 @@ export interface PluginAssetRecord {
     css?: string;
     local?: boolean;
 }
+export declare function invalidatePluginAssets(): void;
 export declare function discoverPluginAssets(): Promise<PluginAssetRecord[]>;
 export interface RuntimeRecord {
     packageName: string;

package/dist/plugin-discovery.js

@@ -31,13 +31,14 @@ async function pkgNames(nm) {
     }
     return names;
 }
-export function resolveAssetUrls(id, coffer) {
+export function resolveAssetUrls(id, coffer, version) {
     const base = `/plugins/${id}`;
-    const out = { schema: `${base}/schema.js` };
+    const v = version ? `?v=${encodeURIComponent(version)}` : '';
+    const out = { schema: `${base}/schema.js${v}` };
     if (coffer.web)
-        out.web = `${base}/web.js`;
+        out.web = `${base}/web.js${v}`;
     if (coffer.css)
-        out.css = `${base}/web.css`;
+        out.css = `${base}/web.css${v}`;
     return out;
 }
 async function isLocalPackage(dir) {
@@ -49,7 +50,13 @@ async function isLocalPackage(dir) {
         return false;
     }
 }
+let assetsCache = null;
+export function invalidatePluginAssets() {
+    assetsCache = null;
+}
 export async function discoverPluginAssets() {
+    if (assetsCache)
+        return assetsCache;
     const nm = join(process.cwd(), 'node_modules');
     const names = await pkgNames(nm);
     const out = [];
@@ -62,10 +69,11 @@ export async function discoverPluginAssets() {
             const manifest = (await import(__rewriteRelativeImportExtension(spec))).default;
             if (!manifest?.id)
                 continue;
-            const urls = resolveAssetUrls(manifest.id, pkg.coffer);
+            const version = pkg.version ?? '0.0.0';
+            const urls = resolveAssetUrls(manifest.id, pkg.coffer, version);
             out.push({
                 id: manifest.id,
-                version: pkg.version ?? '0.0.0',
+                version,
                 dependsOn: manifest.dependsOn ?? [],
                 packageName: name,
                 schema: urls.schema,
@@ -78,6 +86,7 @@ export async function discoverPluginAssets() {
             log.error(`${name}: asset scan skip — ${e.message}`);
         }
     }
+    assetsCache = out;
     return out;
 }
 const RUNTIME_PACKAGE = '@coffer-org/meta';

package/dist/plugins-api.d.ts

@@ -15,5 +15,5 @@ export interface PluginListEntry {
     web?: string;
     css?: string;
 }
-export declare function buildPluginListResponse(plugins: PluginManifest[], assets: PluginAssetRecord[], disabled: Set<string>): Promise<PluginListEntry[]>;
+export declare function buildPluginListResponse(plugins: PluginManifest[], assets: PluginAssetRecord[], disabled: Set<string>, withUpdates?: boolean): Promise<PluginListEntry[]>;
 export declare function registerPluginsApi(app: FastifyInstance): Promise<void>;

package/dist/plugins-api.js

@@ -5,14 +5,14 @@ import { getPlugins, readDisabled } from "./plugin-runtime.js";
 import { checkLatestVersion } from "./plugin-updates.js";
 const nmRoot = () => join(process.cwd(), 'node_modules');
 const ASSET_KEY = { 'schema.js': 'schema', 'web.js': 'web', 'web.css': 'css' };
-export async function buildPluginListResponse(plugins, assets, disabled) {
+export async function buildPluginListResponse(plugins, assets, disabled, withUpdates = true) {
     const assetById = new Map(assets.map((a) => [a.id, a]));
     return Promise.all(plugins.map(async (p) => {
         const a = assetById.get(p.id);
         return {
             id: p.id,
             installedVersion: a?.version ?? p.version,
-            latestVersion: a && !a.local ? await checkLatestVersion(a.packageName) : null,
+            latestVersion: withUpdates && a && !a.local ? await checkLatestVersion(a.packageName) : null,
             packageName: a?.packageName ?? null,
             dependsOn: p.dependsOn,
             enabled: !disabled.has(p.id),
@@ -26,9 +26,10 @@ export async function buildPluginListResponse(plugins, assets, disabled) {
     }));
 }
 export async function registerPluginsApi(app) {
-    app.get('/api/plugins', async () => {
+    app.get('/api/plugins', async (req) => {
+        const withUpdates = req.query.updates === '1';
         const [plugins, assets, disabled] = await Promise.all([getPlugins(), discoverPluginAssets(), readDisabled()]);
-        return buildPluginListResponse(plugins, assets, disabled);
+        return buildPluginListResponse(plugins, assets, disabled, withUpdates);
     });
     app.get('/api/runtime', async () => {
         const runtime = await discoverRuntime();
@@ -63,6 +64,8 @@ export async function registerPluginsApi(app) {
         if (!existsSync(abs))
             return reply.code(404).send({ error: 'asset_missing' });
         reply.type(abs.endsWith('.css') ? 'text/css' : 'text/javascript');
+        const pinned = Boolean(req.query.v) || !key;
+        reply.header('Cache-Control', pinned ? 'public, max-age=31536000, immutable' : 'no-cache');
         return reply.send(createReadStream(abs));
     });
 }

package/dist/records-api.d.ts

@@ -6,13 +6,18 @@ export type RecordListQuery = {
     id?: string;
     [field: string]: string | number | undefined;
 };
+export type RecordListOpts = {
+    view?: 'list' | 'full';
+    extends?: boolean;
+};
 export declare function coerceFilter(v: string, column: string): unknown;
 export declare function withExtends(record: Record<string, unknown>, library: string, shelf: string): Promise<Record<string, unknown>>;
+export declare function withExtendsMany(rows: Record<string, unknown>[], library: string, shelf: string): Promise<Record<string, unknown>[]>;
 export declare function rowMatch(mdef: ShelfDef, row: Record<string, unknown>, tokens: string[]): {
     score: number;
     snippet: string;
 } | null;
-export declare function recordList(library: string, type: string, query?: RecordListQuery): Promise<Record<string, unknown>[]>;
+export declare function recordList(library: string, type: string, query?: RecordListQuery, opts?: RecordListOpts): Promise<Record<string, unknown>[]>;
 export declare function recordGet(library: string, type: string, id: number): Promise<Record<string, unknown> | null>;
 export declare function recordCreate(library: string, type: string, body: unknown, actor?: string): Promise<Record<string, unknown>>;
 export declare function recordUpdate(library: string, type: string, id: number, body: unknown, actor?: string): Promise<Record<string, unknown>>;

package/dist/records-api.js

@@ -1,15 +1,22 @@
-import { fieldMap, textSearchKeys, titleKey, recordTitle } from '@coffer-org/sdk/shelf';
+import { fieldMap, textSearchKeys, titleKey, recordTitle, listKeys, storageColumnsFor } from '@coffer-org/sdk/shelf';
 import { tokenize, matchScoreFolded, foldText } from '@coffer-org/core/search';
 import { serialize } from '@mikro-orm/core';
 import { getShelf, getExtendsFor } from "./registry-context.js";
 import { shelfTableName } from "./entity-schema.js";
-import { getExtendRecord } from "./extend-table.js";
+import { getExtendRecord, getExtendRecords } from "./extend-table.js";
 import { getEm } from "./db.js";
 import { decodeTemporal, dtStringToDate } from "./temporal.js";
 import { splitBody, saveExtends, deleteExtends, validateExtends, readExtends } from "./extend-io.js";
 import { createRecord, updateRecord, getRecord, deleteRecord } from "./mutate.js";
 export class UnknownTypeError extends Error {
 }
+function pickCols(row, cols) {
+    const out = {};
+    for (const c of cols)
+        if (c in row)
+            out[c] = row[c];
+    return out;
+}
 function resolve(library, type) {
     const m = getShelf(library, type);
     if (!m)
@@ -37,6 +44,22 @@ export async function withExtends(record, library, shelf) {
     }));
     return { ...record, _extends };
 }
+export async function withExtendsMany(rows, library, shelf) {
+    const matchedExtends = getExtendsFor(library, shelf);
+    if (!matchedExtends.length || rows.length === 0)
+        return rows;
+    const ids = rows.map((r) => Number(r.id));
+    const byExtend = new Map();
+    await Promise.all(matchedExtends.map(async (e) => {
+        byExtend.set(e.id, await getExtendRecords(e, ids));
+    }));
+    return rows.map((row) => {
+        const _extends = {};
+        for (const e of matchedExtends)
+            _extends[e.id] = byExtend.get(e.id)?.get(Number(row.id)) ?? null;
+        return { ...row, _extends };
+    });
+}
 export function rowMatch(mdef, row, tokens) {
     const textKeys = textSearchKeys(mdef);
     if (!textKeys.length)
@@ -54,7 +77,8 @@ export function rowMatch(mdef, row, tokens) {
     const snippet = raw.length > 120 ? raw.slice(0, 120) + '…' : raw;
     return { score, snippet };
 }
-export async function recordList(library, type, query = {}) {
+export async function recordList(library, type, query = {}, opts = {}) {
+    const { view = 'full', extends: withExt = true } = opts;
     const { m, ename } = resolve(library, type);
     const { q, ...filterParams } = query;
     const where = {};
@@ -77,12 +101,20 @@ export async function recordList(library, type, query = {}) {
     if (m.standalone === false && Object.keys(where).length === 0)
         return [];
     const fork = getEm().fork();
-    let rows = (await fork.find(ename, where)).map((r) => serialize(r));
     const tokens = typeof q === 'string' && q ? tokenize(q) : [];
+    const projected = view === 'list' ? storageColumnsFor(m, listKeys(m)) : null;
+    const findOpts = projected
+        ? { fields: tokens.length ? [...new Set([...projected, ...storageColumnsFor(m, textSearchKeys(m))])] : projected }
+        : undefined;
+    let rows = (await fork.find(ename, where, findOpts)).map((r) => serialize(r));
     if (tokens.length > 0)
         rows = rows.filter((row) => rowMatch(m, row, tokens) !== null);
+    if (projected)
+        rows = rows.map((row) => pickCols(row, projected));
     const decoded = rows.map((r) => decodeTemporal(m, r));
-    return Promise.all(decoded.map((row) => withExtends(row, library, type)));
+    if (!withExt)
+        return decoded;
+    return withExtendsMany(decoded, library, type);
 }
 export async function recordGet(library, type, id) {
     const { m, ename } = resolve(library, type);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coffer-org/server",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "type": "module",
   "engines": {
     "node": ">=24"
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@coffer-org/core": "^1.4.0",
-    "@coffer-org/sdk": "^1.6.0",
+    "@coffer-org/sdk": "^1.7.0",
     "@extractus/oembed-extractor": "^4.1.0",
     "@fastify/cors": "^11.2.0",
     "@fastify/multipart": "^10.0.0",