@anteros/core 0.0.40 → 0.0.42

package/database/audit.ts

@@ -0,0 +1,143 @@
+import type { Db } from "mongodb";
+import { cfg } from "../server/config";
+import type { Tenant } from "../types/tenant";
+import { syncTtlRetention, type RetentionResult } from "./ttl";
+
+/**
+ * Collection holding the audit trail — one document per database operation
+ * (see `types/activity.d.ts` / `rest.audit`).
+ */
+const AUDIT_COLLECTION = '_audit_'
+
+/** Previous name of the audit collection — renamed once at boot by `migrateAuditCollection`. */
+const LEGACY_AUDIT_COLLECTION = '_activities_'
+
+/**
+ * TTL index used for retention. Created only when a `tenant.audit.retention`
+ * is configured — audit entries are kept forever by default.
+ */
+const AUDIT_TTL_INDEX = '_audit_ttl_'
+
+/**
+ * Query indexes, always ensured in the background (same convention as `_vars_`),
+ * whatever the tenant config — an audit collection is queried by date, by
+ * collection, by status and by trace, and grows fast enough that a COLLSCAN
+ * is never acceptable.
+ */
+const AUDIT_INDEXES: { name: string; key: Record<string, 1 | -1> }[] = [
+    /** Listing / date ranges (`getActivities` sorts on `ts` by default) */
+    { name: '_audit_ts_', key: { ts: -1 } },
+    /** `$match: { 'operation.collection': … }` + recent first */
+    { name: '_audit_collection_ts_', key: { 'operation.collection': 1, ts: -1 } },
+    /** `$match: { 'operation.status': 'error' }` + recent first */
+    { name: '_audit_status_ts_', key: { 'operation.status': 1, ts: -1 } },
+    /** Correlate every operation of a request (`trace.id`) */
+    { name: '_audit_trace_', key: { 'trace.id': 1 } },
+]
+
+/**
+ * One-off migration: `_activities_` → `_audit_`.
+ *
+ * Idempotent and never fatal:
+ * - legacy collection absent  → no-op (the common case after the migration)
+ * - target already exists     → no-op, the legacy collection is left untouched
+ *   (never merged nor dropped: losing audit entries silently is worse than a stale collection)
+ */
+async function migrateAuditCollection(db: Db): Promise<boolean> {
+    try {
+        const legacy = await db.listCollections({ name: LEGACY_AUDIT_COLLECTION }).toArray()
+        if (legacy.length === 0) return false
+
+        const target = await db.listCollections({ name: AUDIT_COLLECTION }).toArray()
+        if (target.length > 0) return false
+
+        await db.renameCollection(LEGACY_AUDIT_COLLECTION, AUDIT_COLLECTION, { dropTarget: false })
+        console.log(`🔁 Audit collection renamed: ${LEGACY_AUDIT_COLLECTION} → ${AUDIT_COLLECTION}`)
+        return true
+    } catch (err: any) {
+        console.error(
+            `Audit collection migration failed (${LEGACY_AUDIT_COLLECTION} → ${AUDIT_COLLECTION})`,
+            err?.message,
+        )
+        return false
+    }
+}
+
+/**
+ * Ensure the audit query indexes. `createIndex` is idempotent, so this runs on
+ * every boot and only fills in what is missing. Never fatal, never blocking
+ * (`background: true`).
+ *
+ * ⚠️ Indexes cost writes, and the audit collection is the most write-heavy one —
+ * four indexes are a deliberate trade-off (see the Audit documentation).
+ */
+async function ensureAuditIndexes(db: Db): Promise<string[]> {
+    const ensured: string[] = []
+    const col: any = db.collection(AUDIT_COLLECTION)
+
+    for (const index of AUDIT_INDEXES) {
+        try {
+            await col.createIndex(index.key, { name: index.name, background: true })
+            ensured.push(index.name)
+        } catch (err: any) {
+            console.error(`Audit index '${index.name}' failed`, err?.message)
+        }
+    }
+
+    return ensured
+}
+
+/**
+ * Declarative retention (MongoDB TTL on `ts`) — shared TTL helper:
+ * - duration string (`'90d'`, `'24h'`) → ensure the TTL index with that expiry
+ * - `false`                            → explicitly disabled: drop the TTL index
+ * - omitted                            → untouched (entries are kept forever)
+ */
+async function syncAuditRetention(db: Db, retention?: string | false): Promise<RetentionResult> {
+    return syncTtlRetention(db, {
+        collection: AUDIT_COLLECTION,
+        index: AUDIT_TTL_INDEX,
+        field: 'ts',
+        retention,
+        label: 'Audit',
+    })
+}
+
+/**
+ * Retention resolution — **the tenant always wins**, `server.audit.retention`
+ * is only the fallback:
+ *
+ * - `tenant.audit.retention` set (a duration, even `false`) → used as-is
+ * - otherwise                                        → `server.audit.retention`
+ * - neither                                          → `undefined` (kept forever)
+ */
+function resolveAuditRetention(tenant: Tenant): string | false | undefined {
+    if (tenant.audit?.retention !== undefined) return tenant.audit.retention
+    return cfg.server?.audit?.retention
+}
+
+/**
+ * Boot setup for a tenant's audit collection, in this order:
+ * migration → indexes → retention.
+ *
+ * The migration comes first on purpose: creating the indexes would create an
+ * empty `_audit_`, and the migration would then refuse to rename the legacy
+ * collection (leaving its entries stranded).
+ */
+async function setupAuditCollection(db: Db, retention?: string | false): Promise<void> {
+    await migrateAuditCollection(db)
+    await ensureAuditIndexes(db)
+    await syncAuditRetention(db, retention)
+}
+
+export {
+    AUDIT_COLLECTION,
+    LEGACY_AUDIT_COLLECTION,
+    AUDIT_TTL_INDEX,
+    AUDIT_INDEXES,
+    migrateAuditCollection,
+    ensureAuditIndexes,
+    syncAuditRetention,
+    resolveAuditRetention,
+    setupAuditCollection
+}

package/database/collection.ts

@@ -5,6 +5,7 @@ import fs from "fs/promises"
 import type { Collection } from "../types/collection"
 import type { IndexDescription } from "mongodb"
 import type { Field } from "../types/field"
+import { importDefinition } from "../lib/load"
 import { getTenant } from "./tenant"
 import { buildSchema } from "./schema"
 import type { FileCollection } from "../types/file"
@@ -38,7 +39,8 @@ async function syncCollections() {
             if (isDirectory) {
                 const glob = new Glob(path.join(COLLECTIONS_PATH, '**/*.model.ts'))
                 for await (let file of glob.scan('.')) {
-                    let collectionModule = await import(file)
+                    // A broken model file is skipped, never fatal for the others
+                    let collectionModule = await importDefinition(file, 'collection')
                     if (collectionModule?.default?._isCollection_) {
                         collections.push({
                             ...collectionModule?.default,
@@ -49,7 +51,7 @@ async function syncCollections() {
             }
         }
 
-        // construction des schema arktype pour les collections
+        // build the arktype schemas for the collections
         for (let collection of collections) {
             collection._schema_ = buildSchema(collection)
             collection._schemaPartial_ = buildSchema(collection, { partial: true })
@@ -102,7 +104,7 @@ async function initializeOnDatabase(collections: Collection[]) {
 
 
 
-                // creation d'index sur les champs
+                // create index on the fields
                 let specsFieldsIndexes: IndexDescription[] = []
                 for (let field of collection?.fields || []) {
                     field.indexOptions = field.indexOptions ?? {}
@@ -190,7 +192,7 @@ function getCollection(collectionName: string, tenantId: string): Collection | n
         return col
     }
 
-    // Chercher aussi dans les file collections
+    // Also look in the file collections
     let fileCol = cfg.fileCollections?.find(fc => fc.slug == collectionName && fc._tenant_ == tenantId)
     if (fileCol) {
         return {

package/database/deleteLog.ts

@@ -0,0 +1,98 @@
+import type { Db } from "mongodb";
+import { cfg } from "../server/config";
+import type { ReplicationMetaName } from "../types/replication";
+
+/** Collection holding the replication state (`_replication_`). */
+const MARKERS = "_replication_";
+
+/** Meta collection slug → the name `replication.exclude` knows it by. */
+const META_BY_SLUG: Record<string, ReplicationMetaName> = {
+    [MARKERS]: "replication",
+    _audit_: "audit",
+    _workflows_: "workflows",
+    _locks_: "locks",
+    _vars_: "vars",
+};
+
+/** Replication is armed: enabled, with at least one usable destination. */
+function replicationReady(tenantId: string): boolean {
+    const tenant: any = (cfg.tenants ?? []).find((candidate) => candidate.id === tenantId);
+    const replication = tenant?.replication;
+
+    if (!replication || replication.enabled === false) return false;
+    return (replication.destinations ?? []).some((destination: any) =>
+        destination?.enabled !== false && destination?.id && destination?.uri);
+}
+
+/** `replication.exclude` names this collection. */
+function isExcluded(tenantId: string, name: string): boolean {
+    const tenant: any = (cfg.tenants ?? []).find((candidate) => candidate.id === tenantId);
+    return (tenant?.replication?.exclude ?? []).includes(name);
+}
+
+/**
+ * Whether the framework replicates a **meta** collection (`_audit_`, `_vars_`…)
+ * for this tenant.
+ */
+function replicatesMetaCollection(tenantId: string, name: ReplicationMetaName): boolean {
+    return replicationReady(tenantId) && !isExcluded(tenantId, name);
+}
+
+/**
+ * The same question, by **collection slug** — used by the stores that delete on
+ * their own instead of going through the collection hooks (agent memory, and the
+ * variables / file paths mirror this).
+ *
+ * It answers exactly what `replicatedCollectionsFor()` would: a meta collection,
+ * an agent memory collection (`exclude: ['memory']` covers them all — the tenant
+ * names them), or a declared collection that opted in. A collection nobody
+ * replicates gets no tombstone: it would never be flushed and would only pile up
+ * in `_replication_`.
+ */
+function replicatesCollection(tenantId: string, slug: string): boolean {
+    const asMeta = META_BY_SLUG[slug];
+    if (asMeta) return replicatesMetaCollection(tenantId, asMeta);
+    if (!replicationReady(tenantId)) return false;
+
+    const isAgentMemory = (cfg.agentMemories ?? [])
+        .some((memory) => memory._tenant_ === tenantId && memory.collection === slug);
+    if (isAgentMemory) return !isExcluded(tenantId, "memory");
+
+    const declared = [...(cfg.collections ?? []), ...(cfg.fileCollections ?? [])]
+        .some((collection: any) =>
+            collection._tenant_ === tenantId
+            && collection.slug === slug
+            && collection.replication?.enabled);
+
+    return declared && !isExcluded(tenantId, slug);
+}
+
+/**
+ * Upsert delete tombstones consumed by the replication engine.
+ *
+ * Some collections bypass the collection hooks when deleting (variables,
+ * agent memory, file-collection documents) — those paths call this directly so
+ * their deletions still propagate to the destinations.
+ */
+async function writeDeleteMarkers(
+    db: Db,
+    tenantId: string,
+    collection: string,
+    docIds: string[],
+): Promise<void> {
+    if (!docIds.length) return;
+    const now = new Date();
+    const ops = docIds.map((docId) => ({
+        updateOne: {
+            filter: { _id: `rep.delete:${tenantId}:${collection}:${docId}` },
+            update: {
+                $set: { type: "delete", tenant: tenantId, collection, docId, deletedAt: now, updatedAt: now },
+                $setOnInsert: { synced: [] },
+            },
+            upsert: true,
+        },
+    }));
+    await db.collection(MARKERS).bulkWrite(ops as any, { ordered: false }).catch(() => { /* ignore */ });
+}
+
+export { writeDeleteMarkers, replicatesMetaCollection, replicatesCollection };

package/database/file.ts

@@ -4,6 +4,7 @@ import path from "path"
 import fs from "fs/promises"
 import type { FileCollection } from "../types/file"
 import { getTenant } from "./tenant"
+import { importDefinition } from "../lib/load"
 import type { IndexDescription } from "mongodb"
 
 async function syncFileCollections() {
@@ -18,7 +19,7 @@ async function syncFileCollections() {
             if (filesExist && (await fs.stat(FILES_PATH)).isDirectory()) {
                 const globFiles = new Glob(path.join(FILES_PATH, '**/*.file.ts'))
                 for await (let file of globFiles.scan('.')) {
-                    let fileModule = await import(file)
+                    let fileModule = await importDefinition(file, 'file collection')
                     if (fileModule?.default?._isFileCollection_) {
                         fileCollections.push({
                             ...fileModule?.default,
@@ -30,7 +31,7 @@ async function syncFileCollections() {
                 // Also scan *.model.ts in files/ with _isFileCollection_
                 const globModels = new Glob(path.join(FILES_PATH, '**/*.model.ts'))
                 for await (let file of globModels.scan('.')) {
-                    let fileModule = await import(file)
+                    let fileModule = await importDefinition(file, 'file collection')
                     if (fileModule?.default?._isFileCollection_) {
                         fileCollections.push({
                             ...fileModule?.default,
@@ -46,7 +47,7 @@ async function syncFileCollections() {
             if (collectionsExist && (await fs.stat(COLLECTIONS_PATH)).isDirectory()) {
                 const globFiles = new Glob(path.join(COLLECTIONS_PATH, '**/*.file.ts'))
                 for await (let file of globFiles.scan('.')) {
-                    let fileModule = await import(file)
+                    let fileModule = await importDefinition(file, 'file collection')
                     if (fileModule?.default?._isFileCollection_) {
                         fileCollections.push({
                             ...fileModule?.default,
@@ -75,14 +76,14 @@ async function initializeFileCollectionsOnDatabase(files: FileCollection[]) {
             const db = tenant?.database?.db
             if (!db) continue;
 
-            // Créer la collection si elle n'existe pas
+            // Create the collection if it does not exist
             const collections = await db.listCollections({ name: fileCollection.slug }).toArray()
             if (collections.length === 0) {
                 await db.createCollection(fileCollection.slug)
             }
 
-            // Créer les indexes par défaut (uniquement ceux qui manquent —
-            // ne jamais forcer la recréation d'un index déjà existant)
+            // Create the default indexes (only the missing ones —
+            // never force the re-creation of an already existing index)
             const col = db.collection(fileCollection.slug)
             const existingIndexes = await col.listIndexes().toArray()
             const existingNames = new Set(existingIndexes.map((idx) => idx.name))