@coffer-org/server 1.7.1 → 1.8.0

package/dist/auth-api.js

@@ -1,10 +1,10 @@
 import { field } from '@coffer-org/sdk/fields';
 import { countUsers, createUser, findUserByLogin, findUserById, getPasswordHash, listUsers, updateUser, deleteUser, countAdmins, createSession, resolveSession, deleteSession, createApiToken, resolveApiToken, listApiTokens, revokeApiToken, } from "./auth-store.js";
 import { verifyPassword } from "./auth-crypto.js";
-import { maskSecrets, preserveSecrets } from "./secrets.js";
+import { maskSecrets, preserveSecrets } from "./field-masking.js";
 import { rowMatch } from "./records-api.js";
 import { tokenize } from '@coffer-org/core/search';
-import { USERS_MODULE } from '@coffer-org/sdk/users-module';
+import { USERS_SHELF } from '@coffer-org/sdk/users-shelf';
 const PASSWORD_FIELDS = [field.password({ key: 'password' })];
 const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const COOKIE_NAME = 'sid';
@@ -108,7 +108,7 @@ export async function registerAuthApi(app) {
         const tokens = q ? tokenize(q) : [];
         let users = await listUsers();
         if (tokens.length > 0) {
-            users = users.filter((u) => rowMatch(USERS_MODULE, u, tokens) !== null);
+            users = users.filter((u) => rowMatch(USERS_SHELF, u, tokens) !== null);
         }
         return users.map(withMaskedPassword);
     });

package/dist/background-scheduler.d.ts

@@ -0,0 +1,22 @@
+export type TimerHandle = unknown;
+export interface BackgroundTask {
+    name: string;
+    intervalMs: number;
+    run: () => Promise<void>;
+    timeoutMs?: number;
+}
+export interface SchedulerOpts {
+    startupDelayMs?: number;
+    gapMs?: number;
+    taskTimeoutMs?: number;
+    setTimer?: (cb: () => void, ms: number) => TimerHandle;
+    clearTimer?: (h: TimerHandle) => void;
+}
+export interface BackgroundScheduler {
+    register(task: BackgroundTask): void;
+    start(): void;
+    stop(): void;
+}
+export declare function makeScheduler(opts?: SchedulerOpts): BackgroundScheduler;
+export declare function startScheduler(tasks: BackgroundTask[], opts?: SchedulerOpts): void;
+export declare function stopScheduler(): void;

package/dist/background-scheduler.js

@@ -0,0 +1,101 @@
+import { getLogger } from '@coffer-org/sdk/logger';
+const log = getLogger('scheduler');
+const defaultSetTimer = (cb, ms) => {
+    const t = setTimeout(cb, ms);
+    if (typeof t.unref === 'function')
+        t.unref();
+    return t;
+};
+export function makeScheduler(opts = {}) {
+    const startupDelayMs = opts.startupDelayMs ?? (Number(process.env['BG_STARTUP_DELAY_MS']) || 600_000);
+    const gapMs = opts.gapMs ?? (Number(process.env['BG_GAP_MS']) || 30_000);
+    const defaultTimeoutMs = opts.taskTimeoutMs ?? (Number(process.env['BG_TASK_TIMEOUT_MS']) || 600_000);
+    const setTimer = opts.setTimer ?? defaultSetTimer;
+    const clearTimer = opts.clearTimer ?? ((h) => clearTimeout(h));
+    const tasks = [];
+    const queue = [];
+    const queued = new Set();
+    const timers = new Set();
+    let running = false;
+    let stopped = false;
+    let started = false;
+    function arm(cb, ms) {
+        if (stopped)
+            return;
+        const h = setTimer(() => { timers.delete(h); cb(); }, ms);
+        timers.add(h);
+    }
+    function enqueue(task) {
+        if (stopped || queued.has(task.name))
+            return;
+        queued.add(task.name);
+        queue.push(task);
+        pump();
+    }
+    function pump() {
+        if (stopped || running)
+            return;
+        const task = queue.shift();
+        if (!task)
+            return;
+        queued.delete(task.name);
+        running = true;
+        void runOne(task).finally(() => {
+            running = false;
+            arm(() => enqueue(task), task.intervalMs);
+            if (queue.length)
+                arm(pump, gapMs);
+        });
+    }
+    async function runOne(task) {
+        const timeoutMs = task.timeoutMs ?? defaultTimeoutMs;
+        let to;
+        const timeout = new Promise((resolve) => {
+            to = setTimer(() => { timers.delete(to); log.warn(`${task.name}: timed out after ${timeoutMs}ms`); resolve(); }, timeoutMs);
+            timers.add(to);
+        });
+        try {
+            let runPromise;
+            try {
+                runPromise = task.run();
+            }
+            catch (e) {
+                runPromise = Promise.reject(e);
+            }
+            await Promise.race([runPromise.catch((e) => log.warn(`${task.name}: ${e.message}`)), timeout]);
+        }
+        finally {
+            clearTimer(to);
+            timers.delete(to);
+        }
+    }
+    return {
+        register(task) { tasks.push(task); },
+        start() {
+            if (started || stopped)
+                return;
+            started = true;
+            arm(() => { for (const t of tasks)
+                enqueue(t); }, startupDelayMs);
+        },
+        stop() {
+            stopped = true;
+            for (const h of timers)
+                clearTimer(h);
+            timers.clear();
+            queue.length = 0;
+            queued.clear();
+        },
+    };
+}
+let current;
+export function startScheduler(tasks, opts = {}) {
+    current = makeScheduler(opts);
+    for (const t of tasks)
+        current.register(t);
+    current.start();
+}
+export function stopScheduler() {
+    current?.stop();
+    current = undefined;
+}

package/dist/collection-io.d.ts

@@ -1,5 +1,5 @@
 import type { EntityManager } from '@mikro-orm/core';
-import type { ModuleDef } from '@coffer-org/sdk/module';
+import type { ShelfDef } from '@coffer-org/sdk/shelf';
 import type { LayoutEl } from '@coffer-org/sdk/fields';
 type Row = Record<string, unknown>;
 export declare function scalarKeys(fields: LayoutEl[]): string[];
@@ -10,15 +10,15 @@ export declare function splitAt(fields: LayoutEl[], input: Row): {
 export declare function writeAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number, collections: Record<string, Row[]>): Promise<void>;
 export declare function readAt(em: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<Record<string, Row[]>>;
 export declare function deleteAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<void>;
-export declare const splitCollections: (m: ModuleDef, input: Row) => {
+export declare const splitCollections: (m: ShelfDef, input: Row) => {
     base: Row;
     collections: Record<string, Row[]>;
 };
-export declare const writeCollections: (tx: EntityManager, m: ModuleDef, parentId: number, collections: Record<string, Row[]>) => Promise<void>;
-export declare const readCollections: (em: EntityManager, m: ModuleDef, parentId: number) => Promise<Record<string, Row[]>>;
-export declare const deleteCollections: (tx: EntityManager, m: ModuleDef, parentId: number) => Promise<void>;
+export declare const writeCollections: (tx: EntityManager, m: ShelfDef, parentId: number, collections: Record<string, Row[]>) => Promise<void>;
+export declare const readCollections: (em: EntityManager, m: ShelfDef, parentId: number) => Promise<Record<string, Row[]>>;
+export declare const deleteCollections: (tx: EntityManager, m: ShelfDef, parentId: number) => Promise<void>;
 export declare function flattenEmbeddedAt(fields: LayoutEl[], input: Row): Row;
 export declare function nestEmbeddedAt(fields: LayoutEl[], row: Row): Row;
-export declare const flattenEmbedded: (m: ModuleDef, input: Row) => Row;
-export declare const nestEmbedded: (m: ModuleDef, row: Row) => Row;
+export declare const flattenEmbedded: (m: ShelfDef, input: Row) => Row;
+export declare const nestEmbedded: (m: ShelfDef, row: Row) => Row;
 export {};

package/dist/collection-io.js

@@ -1,5 +1,5 @@
 import { serialize } from '@mikro-orm/core';
-import { collectionGroups, fieldEntries, storageColumns } from '@coffer-org/sdk/module';
+import { collectionGroups, fieldEntries, storageColumns } from '@coffer-org/sdk/shelf';
 import { isGroup, isEmbeddedGroup, isJsonStored } from '@coffer-org/sdk/fields';
 function flattenLayout(fields) {
     const out = [];
@@ -88,7 +88,7 @@ export async function deleteAt(tx, prefix, fields, parentId) {
         await tx.nativeDelete(table, { parent_id: parentId });
     }
 }
-const modPrefix = (m) => `${m.vault}__${m.module}`;
+const modPrefix = (m) => `${m.library}__${m.shelf}`;
 export const splitCollections = (m, input) => splitAt(m.fields, input);
 export const writeCollections = (tx, m, parentId, collections) => writeAt(tx, modPrefix(m), m.fields, parentId, collections);
 export const readCollections = (em, m, parentId) => readAt(em, modPrefix(m), m.fields, parentId);

package/dist/connector-identity.d.ts

@@ -0,0 +1,5 @@
+import type { AuthUser } from './auth-store.ts';
+export declare function linkConnectorIdentity(connector: string, connectorUserId: string, userId: number): Promise<void>;
+export declare function resolveConnectorUser(connector: string, connectorUserId: string): Promise<AuthUser | null>;
+export declare function firstAdmin(): Promise<AuthUser | null>;
+export declare function linkToFirstAdmin(connector: string, connectorUserId: string): Promise<void>;

package/dist/connector-identity.js

@@ -0,0 +1,32 @@
+import { getEm } from "./db.js";
+import { findUserById, listUsers } from "./auth-store.js";
+export async function linkConnectorIdentity(connector, connectorUserId, userId) {
+    const em = getEm().fork();
+    await em.upsert('_ConnectorIdentity', {
+        connector,
+        connector_user_id: connectorUserId,
+        user_id: userId,
+    });
+    await em.flush();
+}
+export async function resolveConnectorUser(connector, connectorUserId) {
+    const em = getEm().fork();
+    const row = (await em.findOne('_ConnectorIdentity', {
+        connector,
+        connector_user_id: connectorUserId,
+    }));
+    if (!row)
+        return null;
+    return findUserById(row.user_id);
+}
+export async function firstAdmin() {
+    const users = await listUsers();
+    return users.find((u) => u.role === 'admin' && !u.disabled) ?? null;
+}
+export async function linkToFirstAdmin(connector, connectorUserId) {
+    if (await resolveConnectorUser(connector, connectorUserId))
+        return;
+    const admin = await firstAdmin();
+    if (admin)
+        await linkConnectorIdentity(connector, connectorUserId, admin.id);
+}

package/dist/embed-openai.d.ts

@@ -0,0 +1,10 @@
+export declare const EMBED_MODEL = "text-embedding-3-small";
+export declare const EMBED_DIM = 1024;
+export declare function embedBatch(texts: string[], apiKey: string, fetchImpl?: typeof fetch): Promise<{
+    vectors: number[][];
+    tokens: number;
+}>;
+export declare function embedOne(text: string, apiKey: string, fetchImpl?: typeof fetch): Promise<{
+    vector: number[];
+    tokens: number;
+}>;

package/dist/embed-openai.js

@@ -0,0 +1,28 @@
+const OPENAI_URL = 'https://api.openai.com/v1/embeddings';
+export const EMBED_MODEL = 'text-embedding-3-small';
+export const EMBED_DIM = 1024;
+export async function embedBatch(texts, apiKey, fetchImpl = fetch) {
+    if (texts.length === 0)
+        return { vectors: [], tokens: 0 };
+    if (!apiKey)
+        throw new Error('embeddings: missing API key (OPENAI_API_KEY)');
+    const res = await fetchImpl(OPENAI_URL, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
+        body: JSON.stringify({ model: EMBED_MODEL, input: texts, dimensions: EMBED_DIM }),
+    });
+    if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        throw new Error(`openai embeddings ${res.status}: ${body.slice(0, 200)}`);
+    }
+    const json = (await res.json());
+    const vectors = [...json.data].sort((a, b) => a.index - b.index).map((d) => d.embedding);
+    return { vectors, tokens: json.usage?.total_tokens ?? 0 };
+}
+export async function embedOne(text, apiKey, fetchImpl = fetch) {
+    const { vectors, tokens } = await embedBatch([text], apiKey, fetchImpl);
+    const vector = vectors[0];
+    if (!vector)
+        throw new Error('embeddings: empty response');
+    return { vector, tokens };
+}

package/dist/entity-schema.d.ts

@@ -1,14 +1,14 @@
 import { EntitySchema } from '@mikro-orm/core';
-import type { ModuleDef } from '@coffer-org/sdk/module';
+import type { ShelfDef } from '@coffer-org/sdk/shelf';
 import type { ExtendDef } from '@coffer-org/sdk/extend';
 import type { SettingsDef } from '@coffer-org/sdk/settings';
 import type { PluginManifest } from '@coffer-org/sdk/plugin';
-export declare function moduleTableName(vault: string, module: string): string;
-export declare function buildEntitySchema(m: ModuleDef): EntitySchema;
+export declare function shelfTableName(library: string, shelf: string): string;
+export declare function buildEntitySchema(m: ShelfDef): EntitySchema;
 export declare function buildExtendEntitySchema(e: ExtendDef): EntitySchema;
 export declare function buildSettingsEntitySchema(pluginId: string, settings: SettingsDef): EntitySchema;
-export declare function childTableName(vault: string, module: string, key: string): string;
-export declare function buildCollectionEntities(m: ModuleDef): EntitySchema[];
+export declare function childTableName(library: string, shelf: string, key: string): string;
+export declare function buildCollectionEntities(m: ShelfDef): EntitySchema[];
 export declare function buildExtendCollectionEntities(e: ExtendDef): EntitySchema[];
 export declare const EventSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
 export declare const PluginRowSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
@@ -16,6 +16,7 @@ export declare const MigrationRowSchema: EntitySchema<any, never, import("@mikro
 export declare const SeedRowSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
 export declare const EmbeddingSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
 export declare const PluginStateSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
+export declare const ThreadMessageSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
 export declare function buildPluginEntities(plugins: PluginManifest[]): EntitySchema[];
 export declare const MsgLogSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;
 export declare const UserSchema: EntitySchema<any, never, import("@mikro-orm/core").EntityCtor<any>>;

package/dist/entity-schema.js

@@ -1,5 +1,5 @@
 import { EntitySchema } from '@mikro-orm/core';
-import { fieldEntries, collectionGroups, storageColumns } from '@coffer-org/sdk/module';
+import { fieldEntries, collectionGroups, storageColumns } from '@coffer-org/sdk/shelf';
 function mikroType(col) {
     if (col === 'integer')
         return 'integer';
@@ -34,8 +34,8 @@ function addFieldProps(properties, fields, plainNullable) {
         }
     }
 }
-export function moduleTableName(vault, module) {
-    return `${vault}__${module}`;
+export function shelfTableName(library, shelf) {
+    return `${library}__${shelf}`;
 }
 export function buildEntitySchema(m) {
     const properties = {
@@ -44,7 +44,7 @@ export function buildEntitySchema(m) {
         updated_at: { type: 'text' },
     };
     addFieldProps(properties, m.fields, (f) => !f.required);
-    const name = moduleTableName(m.vault, m.module);
+    const name = shelfTableName(m.library, m.shelf);
     return new EntitySchema({ name, tableName: name, properties });
 }
 export function buildExtendEntitySchema(e) {
@@ -63,8 +63,8 @@ export function buildSettingsEntitySchema(pluginId, settings) {
     const name = `_settings__${pluginId}`;
     return new EntitySchema({ name, tableName: name, properties });
 }
-export function childTableName(vault, module, key) {
-    return `${vault}__${module}__${key}`;
+export function childTableName(library, shelf, key) {
+    return `${library}__${shelf}__${key}`;
 }
 function buildChildSchemaAt(prefix, c) {
     const properties = {
@@ -85,7 +85,7 @@ function buildCollectionEntitiesAt(prefix, fields) {
     return out;
 }
 export function buildCollectionEntities(m) {
-    return buildCollectionEntitiesAt(`${m.vault}__${m.module}`, m.fields);
+    return buildCollectionEntitiesAt(`${m.library}__${m.shelf}`, m.fields);
 }
 export function buildExtendCollectionEntities(e) {
     return buildCollectionEntitiesAt(`extend__${e.id}`, e.fields);
@@ -151,10 +151,24 @@ export const PluginStateSchema = new EntitySchema({
         value: { type: 'text' },
     },
 });
+export const ThreadMessageSchema = new EntitySchema({
+    name: '_ThreadMessage',
+    tableName: '_thread_message',
+    properties: {
+        connector: { type: 'text', primary: true },
+        chat_id: { type: 'text', primary: true },
+        msg_id: { type: 'text', primary: true },
+        role: { type: 'text' },
+        sender: { type: 'text', nullable: true },
+        text: { type: 'text' },
+        ts: { type: 'integer' },
+        reply_to_id: { type: 'text', nullable: true },
+    },
+});
 export function buildPluginEntities(plugins) {
     return plugins.flatMap((p) => [
-        ...(p.vaults ?? []).flatMap((v) => v.modules.flatMap((m) => [buildEntitySchema(m), ...buildCollectionEntities(m)])),
-        ...(p.vaultModules ?? []).flatMap((m) => [buildEntitySchema(m), ...buildCollectionEntities(m)]),
+        ...(p.libraries ?? []).flatMap((v) => v.shelves.flatMap((m) => [buildEntitySchema(m), ...buildCollectionEntities(m)])),
+        ...(p.libraryShelves ?? []).flatMap((m) => [buildEntitySchema(m), ...buildCollectionEntities(m)]),
         ...(p.extends_ ?? []).flatMap((e) => [buildExtendEntitySchema(e), ...buildExtendCollectionEntities(e)]),
         ...(p.settings && p.settings.fields.length > 0 ? [buildSettingsEntitySchema(p.id, p.settings)] : []),
     ]);
@@ -170,7 +184,6 @@ export const MsgLogSchema = new EntitySchema({
         user_id: { type: 'text' },
         role: { type: 'text' },
         text: { type: 'text' },
-        session_id: { type: 'text', nullable: true },
         tokens_in: { type: 'integer', nullable: true },
         tokens_out: { type: 'integer', nullable: true },
         ms: { type: 'integer', nullable: true },
@@ -216,4 +229,5 @@ export const systemEntities = [
     EventSchema, PluginRowSchema, MigrationRowSchema, SeedRowSchema,
     EmbeddingSchema, PluginStateSchema, MsgLogSchema,
     UserSchema, SessionSchema, ApiTokenSchema,
+    ThreadMessageSchema,
 ];

package/dist/extend-io.d.ts

@@ -1,7 +1,9 @@
+import type { EntityManager } from '@mikro-orm/core';
 export declare function splitBody(body: unknown): {
     base: Record<string, unknown>;
     extData: Record<string, Record<string, unknown>>;
 };
-export declare function deleteExtends(vault: string, module: string, id: number): Promise<void>;
-export declare function validateExtends(vault: string, module: string, extData: Record<string, Record<string, unknown>>): void;
-export declare function saveExtends(vault: string, module: string, baseId: number, extData: Record<string, Record<string, unknown>>): Promise<void>;
+export declare function deleteExtends(em: EntityManager, library: string, shelf: string, id: number): Promise<void>;
+export declare function readExtends(em: EntityManager, library: string, shelf: string, id: number): Promise<Record<string, unknown>>;
+export declare function validateExtends(library: string, shelf: string, extData: Record<string, Record<string, unknown>>): void;
+export declare function saveExtends(em: EntityManager, library: string, shelf: string, baseId: number, extData: Record<string, Record<string, unknown>>): Promise<void>;

package/dist/extend-io.js

@@ -1,6 +1,6 @@
 import { getExtendsFor } from "./registry-context.js";
 import { buildExtendZodPartial } from '@coffer-org/sdk/extend';
-import { upsertExtendRecord, deleteExtendRecord } from "./extend-table.js";
+import { upsertExtendRecord, deleteExtendRecord, getExtendRecord } from "./extend-table.js";
 import { toIssue, ValidationError } from "./mutate.js";
 export function splitBody(body) {
     const raw = (body ?? {});
@@ -14,14 +14,21 @@ export function splitBody(body) {
     }
     return { base, extData };
 }
-export async function deleteExtends(vault, module, id) {
-    for (const e of getExtendsFor(vault, module)) {
-        await deleteExtendRecord(e, id);
+export async function deleteExtends(em, library, shelf, id) {
+    for (const e of getExtendsFor(library, shelf)) {
+        await deleteExtendRecord(em, e, id);
     }
 }
-export function validateExtends(vault, module, extData) {
+export async function readExtends(em, library, shelf, id) {
+    const out = {};
+    for (const e of getExtendsFor(library, shelf)) {
+        out[e.id] = (await getExtendRecord(e, id, em)) ?? null;
+    }
+    return out;
+}
+export function validateExtends(library, shelf, extData) {
     const issues = [];
-    for (const e of getExtendsFor(vault, module)) {
+    for (const e of getExtendsFor(library, shelf)) {
         const data = extData[e.id];
         if (data === undefined)
             continue;
@@ -33,15 +40,15 @@ export function validateExtends(vault, module, extData) {
     if (issues.length)
         throw new ValidationError(issues);
 }
-export async function saveExtends(vault, module, baseId, extData) {
-    validateExtends(vault, module, extData);
-    for (const e of getExtendsFor(vault, module)) {
+export async function saveExtends(em, library, shelf, baseId, extData) {
+    validateExtends(library, shelf, extData);
+    for (const e of getExtendsFor(library, shelf)) {
         const data = extData[e.id];
         if (data === undefined)
             continue;
         const parsed = buildExtendZodPartial(e).safeParse(data);
         if (parsed.success && Object.keys(parsed.data).length > 0) {
-            await upsertExtendRecord(e, baseId, parsed.data);
+            await upsertExtendRecord(em, e, baseId, parsed.data);
         }
     }
 }

package/dist/extend-table.d.ts

@@ -1,7 +1,8 @@
+import type { EntityManager } from '@mikro-orm/core';
 import type { ExtendDef } from '@coffer-org/sdk/extend';
 type Row = Record<string, unknown>;
 export declare function extendEntityName(e: ExtendDef): string;
-export declare function getExtendRecord(e: ExtendDef, baseId: number): Promise<Row | undefined>;
-export declare function upsertExtendRecord(e: ExtendDef, baseId: number, data: Row): Promise<void>;
-export declare function deleteExtendRecord(e: ExtendDef, baseId: number): Promise<void>;
+export declare function getExtendRecord(e: ExtendDef, baseId: number, em?: EntityManager): Promise<Row | undefined>;
+export declare function upsertExtendRecord(em: EntityManager, e: ExtendDef, baseId: number, data: Row): Promise<void>;
+export declare function deleteExtendRecord(em: EntityManager, e: ExtendDef, baseId: number): Promise<void>;
 export {};

package/dist/extend-table.js

@@ -4,8 +4,8 @@ import { splitAt, writeAt, readAt, deleteAt } from "./collection-io.js";
 export function extendEntityName(e) {
     return `extend__${e.id}`;
 }
-export async function getExtendRecord(e, baseId) {
-    const fork = getEm().fork();
+export async function getExtendRecord(e, baseId, em) {
+    const fork = em ?? getEm().fork();
     const name = extendEntityName(e);
     const row = await fork.findOne(name, { base_id: baseId });
     if (!row)
@@ -14,24 +14,16 @@ export async function getExtendRecord(e, baseId) {
     const collections = await readAt(fork, name, e.fields, baseId);
     return { ...flat, ...collections };
 }
-export async function upsertExtendRecord(e, baseId, data) {
+export async function upsertExtendRecord(em, e, baseId, data) {
     const name = extendEntityName(e);
     const { base, collections } = splitAt(e.fields, data);
-    await getEm()
-        .fork()
-        .transactional(async (tx) => {
-        await tx.upsert(name, { base_id: baseId, ...base });
-        await writeAt(tx, name, e.fields, baseId, collections);
-    });
+    await em.upsert(name, { base_id: baseId, ...base });
+    await writeAt(em, name, e.fields, baseId, collections);
 }
-export async function deleteExtendRecord(e, baseId) {
+export async function deleteExtendRecord(em, e, baseId) {
     const name = extendEntityName(e);
-    await getEm()
-        .fork()
-        .transactional(async (tx) => {
-        await deleteAt(tx, name, e.fields, baseId);
-        const ref = tx.getReference(name, baseId);
-        tx.remove(ref);
-        await tx.flush();
-    });
+    await deleteAt(em, name, e.fields, baseId);
+    const ref = em.getReference(name, baseId);
+    em.remove(ref);
+    await em.flush();
 }

package/dist/field-masking.d.ts

@@ -0,0 +1,7 @@
+import type { LayoutEl } from '@coffer-org/sdk/fields';
+export declare const MASK = "********";
+export declare function passwordKeys(fields: LayoutEl[]): string[];
+export declare function maskSecrets(fields: LayoutEl[], row: Record<string, unknown>): Record<string, unknown>;
+export declare function preserveSecrets(fields: LayoutEl[], incoming: Record<string, unknown>, existing: Record<string, unknown>): Record<string, unknown>;
+export declare function maskTree(fields: LayoutEl[], obj: Record<string, unknown>): Record<string, unknown>;
+export declare function preserveTree(fields: LayoutEl[], incoming: Record<string, unknown>, existing: Record<string, unknown>): Record<string, unknown>;

package/dist/field-masking.js

@@ -0,0 +1,60 @@
+import { fieldEntries, collectionGroups } from '@coffer-org/sdk/shelf';
+export const MASK = '********';
+export function passwordKeys(fields) {
+    return fieldEntries(fields)
+        .filter(([, f]) => f.kind === 'password')
+        .map(([k]) => k);
+}
+export function maskSecrets(fields, row) {
+    const out = { ...row };
+    for (const k of passwordKeys(fields))
+        if (out[k])
+            out[k] = MASK;
+    return out;
+}
+export function preserveSecrets(fields, incoming, existing) {
+    const out = { ...incoming };
+    for (const k of passwordKeys(fields)) {
+        if (out[k] !== MASK)
+            continue;
+        if (existing[k] !== undefined && existing[k] !== null)
+            out[k] = existing[k];
+        else
+            delete out[k];
+    }
+    return out;
+}
+export function maskTree(fields, obj) {
+    const out = maskSecrets(fields, obj);
+    for (const c of collectionGroups(fields)) {
+        const key = c.key;
+        const sub = c.group.fields;
+        const arr = out[key];
+        if (Array.isArray(arr)) {
+            out[key] = arr.map((item) => item && typeof item === 'object' ? maskTree(sub, item) : item);
+        }
+    }
+    return out;
+}
+export function preserveTree(fields, incoming, existing) {
+    const out = preserveSecrets(fields, incoming, existing);
+    for (const c of collectionGroups(fields)) {
+        const key = c.key;
+        const sub = c.group.fields;
+        const uniq = c.group.unique ?? [];
+        const inArr = out[key];
+        if (!Array.isArray(inArr))
+            continue;
+        const exArr = Array.isArray(existing[key]) ? existing[key] : [];
+        out[key] = inArr.map((item) => {
+            if (!item || typeof item !== 'object')
+                return item;
+            const rec = item;
+            const match = uniq.length > 0
+                ? exArr.find((e) => e && uniq.every((u) => e[u] === rec[u]))
+                : undefined;
+            return preserveTree(sub, rec, (match ?? {}));
+        });
+    }
+    return out;
+}

package/dist/index-signal.d.ts

@@ -0,0 +1,3 @@
+export declare function onRecordsChanged(cb: () => void): void;
+export declare function offRecordsChanged(): void;
+export declare function notifyRecordsChanged(): void;

package/dist/index-signal.js

@@ -0,0 +1,14 @@
+import { getLogger } from '@coffer-org/sdk/logger';
+const log = getLogger('rag-signal');
+let listener = null;
+export function onRecordsChanged(cb) {
+    if (listener)
+        log.warn('onRecordsChanged: overwriting an already-registered listener without offRecordsChanged');
+    listener = cb;
+}
+export function offRecordsChanged() {
+    listener = null;
+}
+export function notifyRecordsChanged() {
+    listener?.();
+}