@anteros/core 0.0.41 → 0.0.44

package/tests/replication-change-stream.test.ts

@@ -0,0 +1,240 @@
+/**
+ * Change-stream replication — the **default** mode (`replication.mode` unset).
+ *
+ * What separates it from the scan engine, and what these cases pin down:
+ *  - nothing has to be triggered: a write reaches the destination on its own;
+ *  - deletes come from the oplog, so a collection **without hooks** propagates them
+ *    too (the scan engine can only see applicative tombstones, which is why an
+ *    `_audit_` entry deleted out of band used to survive on the destination);
+ *  - the position is a resume token, not a date cursor;
+ *  - the stream owns a collection, so a one-shot run leaves it alone;
+ *  - `reset()` re-arms the stream, which cold-starts again.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import { MongoClient } from "mongodb";
+import { useRest } from "../database/rest";
+import { formatConfig, cfg } from "../server/config";
+import { syncTenants } from "../database/tenant";
+import { syncCollections } from "../database/collection";
+import { syncVars } from "../database/vars";
+import {
+    startReplication,
+    stopReplication,
+    replicateTenant,
+    getReplicationState,
+    resetReplication,
+} from "../database/replication";
+import { AUDIT_COLLECTION } from "../database/audit";
+
+const ORDERS_T = "cs-orders";
+const ORDERS_DIR = "packages/core/tests/fixtures/replication-meta";
+const ORDERS_SRC = "mongodb://localhost:27017/_CS_SRC";
+const ORDERS_DST = "mongodb://localhost:27017/_CS_DST";
+const DEST_ID = "backup";
+
+const VARS_T = "cs-vars";
+const VARS_DIR = "packages/core/tests/fixtures/vars-tenant";
+const VARS_SRC = "mongodb://localhost:27017/_CS_VARS_SRC";
+const VARS_DST = "mongodb://localhost:27017/_CS_VARS_DST";
+
+let rest: InstanceType<typeof useRest>;
+let restVars: InstanceType<typeof useRest>;
+let src: MongoClient;
+let dest: MongoClient;
+let varsDest: MongoClient;
+
+/** Poll `check` until it holds — a change stream is asynchronous by nature. */
+async function waitFor(check: () => Promise<boolean>, timeout = 8_000, step = 50): Promise<boolean> {
+    const deadline = Date.now() + timeout;
+    while (Date.now() < deadline) {
+        if (await check()) return true;
+        await Bun.sleep(step);
+    }
+    return false;
+}
+
+const ordersDest = () => dest.db().collection("orders");
+const auditDest = () => dest.db().collection(AUDIT_COLLECTION);
+const varsOnDest = (ns: string, key: string) => varsDest.db().collection("_vars_").findOne({ ns, key });
+
+beforeAll(async () => {
+    formatConfig({
+        server: { port: 4000 },
+        tenants: [
+            {
+                id: ORDERS_T,
+                dir: ORDERS_DIR,
+                database: { uri: ORDERS_SRC },
+                // No `mode`: change streams are the default. No `key`, no `batchSize`,
+                // no `schedule` either — the stream is the whole engine.
+                replication: { runOnBoot: false, destinations: [{ id: DEST_ID, uri: ORDERS_DST }] },
+            },
+            {
+                id: VARS_T,
+                dir: VARS_DIR,
+                database: { uri: VARS_SRC },
+                replication: { runOnBoot: false, destinations: [{ id: "backup", uri: VARS_DST }] },
+            },
+        ],
+    });
+
+    await syncTenants();
+    await syncCollections();
+    await syncVars();
+    await startReplication();
+
+    rest = new useRest({ tenant_id: ORDERS_T });
+    restVars = new useRest({ tenant_id: VARS_T });
+    src = new MongoClient(ORDERS_SRC, { serverSelectionTimeoutMS: 3000 });
+    dest = new MongoClient(ORDERS_DST, { serverSelectionTimeoutMS: 3000 });
+    varsDest = new MongoClient(VARS_DST, { serverSelectionTimeoutMS: 3000 });
+    await Promise.all([src.connect(), dest.connect(), varsDest.connect()]);
+});
+
+afterAll(async () => {
+    await stopReplication();
+    for (const uri of [ORDERS_SRC, ORDERS_DST, VARS_SRC, VARS_DST]) {
+        try {
+            const client = new MongoClient(uri);
+            await client.connect();
+            await client.db().dropDatabase();
+            await client.close();
+        } catch (_) {}
+    }
+    cfg.agents = [];
+    cfg.agentMemories = [];
+    try { await src.close(); } catch (_) {}
+    try { await dest.close(); } catch (_) {}
+    try { await varsDest.close(); } catch (_) {}
+});
+
+describe("replication — changeStream mode", () => {
+    it("replicates a write on its own, with no run to trigger", async () => {
+        const marker = `cs-insert-${crypto.randomUUID()}`;
+        await rest.insertOne("orders", { title: marker });
+
+        // No `replicateTenant()` call anywhere: the stream is the engine
+        expect(await waitFor(async () => Boolean(await ordersDest().findOne({ title: marker })))).toBe(true);
+    });
+
+    it("replicates an update in place", async () => {
+        const marker = `cs-update-${crypto.randomUUID()}`;
+        const doc: any = await rest.insertOne("orders", { title: marker });
+        expect(await waitFor(async () => Boolean(await ordersDest().findOne({ title: marker })))).toBe(true);
+
+        await rest.updateOne("orders", doc._id, { $set: { title: `${marker}-paid` } });
+        expect(await waitFor(async () => (await ordersDest().countDocuments({ title: `${marker}-paid` })) === 1)).toBe(true);
+    });
+
+    it("propagates a delete without writing a tombstone", async () => {
+        const marker = `cs-delete-${crypto.randomUUID()}`;
+        const doc: any = await rest.insertOne("orders", { title: marker });
+        expect(await waitFor(async () => Boolean(await ordersDest().findOne({ title: marker })))).toBe(true);
+
+        await rest.deleteOne("orders", doc._id);
+        expect(await waitFor(async () => (await ordersDest().countDocuments({ title: marker })) === 0)).toBe(true);
+
+        // The oplog carried it: the hook + tombstone path is not even installed here
+        const tombstones = await src.db().collection("_replication_").countDocuments({ type: "delete", collection: "orders" });
+        expect(tombstones).toBe(0);
+    });
+
+    it("propagates a delete on a framework collection no hook can see", async () => {
+        const marker = `cs-audit-${crypto.randomUUID()}`;
+        await rest.audit.addActivities([{
+            internal: true,
+            trace: { id: crypto.randomUUID() },
+            meta: {},
+            operation: {
+                tenant: ORDERS_T, action: marker, collection: "orders",
+                status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
+            },
+            ts: new Date(),
+        } as any]);
+        expect(await waitFor(async () => Boolean(await auditDest().findOne({ "operation.action": marker })))).toBe(true);
+
+        // Out of band: no applicative hook runs, no tombstone is written — the scan
+        // engine would leave this entry on the destination forever
+        await src.db().collection(AUDIT_COLLECTION).deleteOne({ "operation.action": marker } as any);
+
+        expect(await waitFor(async () => (await auditDest().countDocuments({ "operation.action": marker })) === 0)).toBe(true);
+    }, 20_000);
+
+    it("stores a resume token in `_replication_`, tagged with the mode", async () => {
+        const [state] = await getReplicationState(ORDERS_T, DEST_ID, "orders");
+        expect(state?.mode).toBe("changeStream");
+        expect(state?.resumeToken).toBeDefined();
+        expect(state?.status).toBe("watching");
+    });
+
+    it("leaves a streamed collection alone when a run is triggered", async () => {
+        const results = await replicateTenant(ORDERS_T);
+
+        // The stream owns it: a scan on top would let a stale read overwrite a fresh
+        // event, so the run reports nothing for that collection
+        expect(results.find((result) => result.collection === "orders")).toBeUndefined();
+    });
+
+    it("cold-starts again after `reset()` — the position is really cleared", async () => {
+        const marker = `cs-reset-${crypto.randomUUID()}`;
+        await rest.insertOne("orders", { title: marker });
+        expect(await waitFor(async () => Boolean(await ordersDest().findOne({ title: marker })))).toBe(true);
+
+        // Lose the copy, then forget the position: only a backfill can bring it back
+        await ordersDest().deleteMany({});
+        const reset = await resetReplication(ORDERS_T, { destination: DEST_ID, collection: "orders" });
+        expect(reset.state).toBeGreaterThanOrEqual(1);
+
+        expect(await waitFor(async () => (await ordersDest().countDocuments({ title: marker })) === 1, 20_000)).toBe(true);
+    }, 30_000);
+
+    it("applies the collection's document filter to the stream (`_vars_` namespaces)", async () => {
+        const marker = `cs-vars-${crypto.randomUUID()}`;
+        await restVars.vars.set("config", marker, "REPLICATED");
+        await restVars.vars.set("private", "secret", marker);
+
+        expect(await waitFor(async () => Boolean(await varsOnDest("config", marker)))).toBe(true);
+        // `private.var.ts` opts out: the filter is on the post-image, so the stream
+        // never even writes it
+        expect(await varsOnDest("private", "secret")).toBeNull();
+    });
+});
+
+describe("replication — changeStream shutdown", () => {
+    it("stops writing once replication is stopped", async () => {
+        await stopReplication();
+
+        const marker = `cs-after-stop-${crypto.randomUUID()}`;
+        await rest.insertOne("orders", { title: marker });
+        await Bun.sleep(1_500); // long enough for a stream that is still alive
+
+        expect(await ordersDest().countDocuments({ title: marker })).toBe(0);
+    });
+
+    it("does not wait forever on a collection another writer owns", async () => {
+        // A stream elsewhere already owns `orders`; this boot must not block on it
+        await rest.lock(`replication:${DEST_ID}:orders`, 60_000);
+
+        const startedAt = Date.now();
+        await startReplication();
+        const elapsed = Date.now() - startedAt;
+
+        expect(elapsed).toBeLessThan(25_000);
+
+        // …and the collections it *could* take are still watched: an entry written
+        // with no run reaches the destination while `orders` is skipped
+        const marker = `cs-partial-${crypto.randomUUID()}`;
+        await rest.audit.addActivities([{
+            internal: true,
+            trace: { id: crypto.randomUUID() },
+            meta: {},
+            operation: {
+                tenant: ORDERS_T, action: marker, collection: "orders",
+                status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
+            },
+            ts: new Date(),
+        } as any]);
+
+        expect(await waitFor(async () => (await auditDest().countDocuments({ "operation.action": marker })) === 1)).toBe(true);
+    }, 60_000);
+});

package/tests/replication-control.test.ts

@@ -28,6 +28,8 @@ beforeAll(async () => {
             dir: "src",
             database: { uri: SRC_DB },
             replication: {
+                // These cases exercise the scan engine (cursor, tombstones, seeding)
+                mode: "scan",
                 runOnBoot: false,
                 schedule: { interval: "1h" },
                 destinations: [{ id: DEST_ID, uri: DEST_DB }],
@@ -102,7 +104,7 @@ describe("replication seed", () => {
         const preMarker = `pre-${crypto.randomUUID()}`;
         await rest.insertOne("orders", { title: preMarker });
 
-        const res = await seedReplication(TEST_TENANT, "latest");
+        const res = await seedReplication(TEST_TENANT, "latest", { collection: "orders" });
         expect(res.seeded).toBe(1);
 
         await replicateTenant(TEST_TENANT);
@@ -118,7 +120,7 @@ describe("replication seed", () => {
 
     it("seeds from an explicit date", async () => {
         await resetReplication(TEST_TENANT);
-        const res = await seedReplication(TEST_TENANT, "2020-01-01T00:00:00Z");
+        const res = await seedReplication(TEST_TENANT, "2020-01-01T00:00:00Z", { collection: "orders" });
         expect(res.seeded).toBe(1);
 
         const state = await getReplicationState(TEST_TENANT, DEST_ID, "orders");

package/tests/replication-initial-sync.test.ts

@@ -21,6 +21,8 @@ beforeAll(async () => {
             dir: "src",
             database: { uri: SRC_DB },
             replication: {
+                // These cases exercise the scan engine's `initialSync` seeding
+                mode: "scan",
                 runOnBoot: false,
                 schedule: { interval: "1h" },
                 destinations: [{ id: DEST_ID, uri: DEST_DB }],

package/tests/replication-meta.test.ts

@@ -0,0 +1,246 @@
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import os from "node:os";
+import { MongoClient } from "mongodb";
+import { useRest } from "../database/rest";
+import { formatConfig, cfg } from "../server/config";
+import { syncTenants } from "../database/tenant";
+import { syncCollections } from "../database/collection";
+import { syncWorkflows } from "../lib/workflow";
+import { startReplication, stopReplication, replicateTenant } from "../database/replication";
+import { syncAgents } from "../lib/agents";
+import { MongoAgentMemory } from "../lib/agentMemory";
+import { AGENT_THREADS } from "./fixtures/replication-meta/agents/chat.agent";
+import type { ReplicationMetaName } from "../types/replication";
+
+const DIR = "packages/core/tests/fixtures/replication-meta";
+
+/** Everything replicated: declared collection + the six framework collections */
+const ALL_T = "rm-default";
+const ALL_T_DB = "mongodb://localhost:27017/_RM_DEFAULT_SRC";
+const ALL_T_DEST = "mongodb://localhost:27017/_RM_DEFAULT_DEST";
+
+/** Nothing but the declared collection */
+const OFF_T = "rm-alloff";
+const OFF_T_DB = "mongodb://localhost:27017/_RM_ALLOFF_SRC";
+const OFF_T_DEST = "mongodb://localhost:27017/_RM_ALLOFF_DEST";
+
+let rest: InstanceType<typeof useRest>;
+let dest: MongoClient;
+let destOff: MongoClient;
+
+const target = (slug: string) => dest.db().collection(slug);
+const targetOff = (slug: string) => destOff.db().collection(slug);
+
+beforeAll(async () => {
+    formatConfig({
+        server: { port: 4000 },
+        tenants: [
+            {
+                id: ALL_T,
+                dir: DIR,
+                database: { uri: ALL_T_DB },
+                // Opt-in retention, so the destination's copy is pruned the same way
+                audit: { retention: "30d" },
+                replication: {
+                    // These cases exercise the scan engine (meta collections, exclude)
+                    mode: "scan",
+                    runOnBoot: false,
+                    schedule: { interval: "1h" },
+                    destinations: [{ id: "backup", uri: ALL_T_DEST }],
+                },
+            },
+            {
+                id: OFF_T,
+                dir: DIR,
+                database: { uri: OFF_T_DB },
+                replication: {
+                    mode: "scan",
+                    runOnBoot: false,
+                    schedule: { interval: "1h" },
+                    exclude: ["audit", "workflows", "locks", "replication", "vars", "memory"],
+                    destinations: [{ id: "backup", uri: OFF_T_DEST }],
+                },
+            },
+        ],
+    });
+
+    await syncTenants();
+    await syncCollections();
+    await syncWorkflows();
+    await syncAgents(); // publishes the agent memory collections (`cfg.agentMemories`)
+    await startReplication();
+
+    rest = new useRest({ tenant_id: ALL_T });
+    dest = new MongoClient(ALL_T_DEST, { serverSelectionTimeoutMS: 3000 });
+    destOff = new MongoClient(OFF_T_DEST, { serverSelectionTimeoutMS: 3000 });
+    await dest.connect();
+    await destOff.connect();
+});
+
+afterAll(async () => {
+    await stopReplication();
+    for (const uri of [ALL_T_DB, ALL_T_DEST, OFF_T_DB, OFF_T_DEST]) {
+        try {
+            const client = new MongoClient(uri);
+            await client.connect();
+            await client.db().dropDatabase();
+            await client.close();
+        } catch (_) {}
+    }
+    cfg.agents = [];
+    cfg.agentMemories = [];
+    try { await dest.close(); } catch (_) {}
+    try { await destOff.close(); } catch (_) {}
+});
+
+describe("replication — framework collections", () => {
+    it("replicates them by default (audit, workflows, locks, replication, vars, memory)", async () => {
+        const marker = `meta-${crypto.randomUUID()}`;
+
+        await rest.audit.addActivities([{
+            internal: true,
+            trace: { id: crypto.randomUUID() },
+            meta: {},
+            operation: {
+                tenant: ALL_T, action: marker, collection: "orders",
+                status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
+            },
+            ts: new Date(),
+        } as any]);
+        await rest.workflow.run("run", { marker });
+        await rest.vars.set("config", marker, "META-1");
+        await rest.lock(`meta-${marker}`);
+        await rest.insertOne("orders", { title: marker });
+        // An agent conversation — the collection this tenant named for its memory
+        await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
+            "chat-1",
+            [{ role: "user", content: marker }] as any,
+            { rest, tenant: ALL_T, resourceId: "u1" },
+        );
+
+        // A first run picks the state document it writes, hence the second one
+        await replicateTenant(ALL_T);
+        await replicateTenant(ALL_T);
+
+        expect(await target("_audit_").findOne({ "operation.action": marker })).toBeDefined();
+        expect(await target("_workflows_").findOne({ workflowId: "run" })).toBeDefined();
+        expect(await target("_vars_").findOne({ ns: "config", key: marker })).toBeDefined();
+        expect(await target("_locks_").findOne({ name: `meta-${marker}` })).toBeDefined();
+        expect(await target("_replication_").findOne({ type: "state", tenant: ALL_T })).toBeDefined();
+        expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-1" as any })).toBeDefined();
+        expect(await target("orders").findOne({ title: marker })).toBeDefined();
+
+        await rest.unlock(`meta-${marker}`);
+    }, 30_000);
+
+    it("propagates a cleared thread to the destination (tombstone)", async () => {
+        // The thread replicated by the previous test is deleted at the source: no
+        // date cursor can see a deletion, so the store leaves a tombstone.
+        await new MongoAgentMemory({ collection: AGENT_THREADS }).clear("chat-1", { rest, tenant: ALL_T, resourceId: "u1" });
+
+        await replicateTenant(ALL_T);
+
+        expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-1" as any })).toBeNull();
+        // The tombstone is consumed and purged
+        expect(await rest.db.collection("_replication_").countDocuments({ type: "delete", collection: AGENT_THREADS })).toBe(0);
+    }, 30_000);
+
+    it("uses each collection's own date key (audit is read on `ts`)", async () => {
+        const state: any = await rest.db.collection("_replication_")
+            .findOne({ type: "state", tenant: ALL_T, collection: "_audit_" });
+        expect(state).toBeDefined();
+        expect(state.key).toBe("ts");
+        // The state says which process is doing the work (`reusePort` / forked workers)
+        expect(state.pid).toBe(process.pid);
+        expect(state.hostname).toBe(os.hostname());
+    });
+
+    it("mirrors the audit retention on the destination", async () => {
+        const ttl: any = (await target("_audit_").listIndexes().toArray())
+            .find((i: any) => i.name === "_audit_ttl_");
+        expect(ttl).toBeDefined();
+        expect(ttl.expireAfterSeconds).toBe(30 * 86400);
+    }, 30_000);
+
+    it("leaves out only what `exclude` names", async () => {
+        const tenant: any = cfg.tenants?.find((t) => t.id === ALL_T);
+        const marker = `noaudit-${crypto.randomUUID()}`;
+        tenant.replication.exclude = ["audit", "memory"];
+
+        await rest.audit.addActivities([{
+            internal: true, trace: { id: crypto.randomUUID() }, meta: {},
+            operation: {
+                tenant: ALL_T, action: marker, collection: "orders",
+                status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
+            },
+            ts: new Date(),
+        } as any]);
+        await rest.vars.set("config", marker, "still-replicated");
+        await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
+            "chat-2",
+            [{ role: "user", content: marker }] as any,
+            { rest, tenant: ALL_T, resourceId: "u1" },
+        );
+
+        await replicateTenant(ALL_T);
+
+        expect(await target("_audit_").findOne({ "operation.action": marker })).toBeNull();      // excluded
+        expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-2" as any })).toBeNull();     // excluded
+        expect(await target("_vars_").findOne({ ns: "config", key: marker })).toBeDefined();     // still there
+
+        delete tenant.replication.exclude;
+    }, 30_000);
+
+    it("replicates nothing from the framework when every name is excluded", async () => {
+        const offRest = new useRest({ tenant_id: OFF_T });
+        const marker = `alloff-${crypto.randomUUID()}`;
+
+        await offRest.audit.addActivities([{
+            internal: true, trace: { id: crypto.randomUUID() }, meta: {},
+            operation: {
+                tenant: OFF_T, action: marker, collection: "orders",
+                status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
+            },
+            ts: new Date(),
+        } as any]);
+        await offRest.vars.set("config", marker, "not-replicated");
+        await offRest.insertOne("orders", { title: marker });
+        await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
+            "chat-3",
+            [{ role: "user", content: marker }] as any,
+            { rest: offRest, tenant: OFF_T, resourceId: "u1" },
+        );
+
+        await replicateTenant(OFF_T);
+        await replicateTenant(OFF_T);
+
+        // The declared collection still replicates…
+        expect(await targetOff("orders").findOne({ title: marker })).toBeDefined();
+        // …but the framework collections do not exist on that destination
+        expect(await targetOff("_audit_").countDocuments({})).toBe(0);
+        expect(await targetOff("_vars_").countDocuments({})).toBe(0);
+        expect(await targetOff("_workflows_").countDocuments({})).toBe(0);
+        expect(await targetOff(AGENT_THREADS).countDocuments({})).toBe(0);
+    }, 30_000);
+
+    it("propagates a delete on a collection whose key is a number", async () => {
+        const doc: any = await rest.insertOne("counters", { seq: 100, title: "numeric-key" });
+        await replicateTenant(ALL_T);
+        expect(await target("counters").findOne({ title: "numeric-key" })).toBeDefined();
+
+        await rest.deleteOne("counters", doc._id);
+        await replicateTenant(ALL_T);
+        // The tombstone guard compares the key to a date by default: a numeric key
+        // used to silently skip the delete
+        expect(await target("counters").findOne({ title: "numeric-key" })).toBeNull();
+    }, 30_000);
+
+    it("keeps the `exclude` list typed", () => {
+        const names: ReplicationMetaName[] = ["audit", "workflows", "locks", "replication", "vars", "memory"];
+        expect(names.length).toBe(6);
+
+        // @ts-expect-error — a declared collection slug is not a framework collection
+        const wrong: ReplicationMetaName[] = ["orders"];
+        expect(wrong.length).toBe(1);
+    });
+});

package/tests/replication.test.ts

@@ -26,6 +26,8 @@ beforeAll(async () => {
             dir: "src",
             database: { uri: SRC_DB },
             replication: {
+                // These cases exercise the scan engine (cursor, tombstones, initial sync)
+                mode: "scan",
                 runOnBoot: false,
                 schedule: { interval: "1h" },
                 destinations: [{ id: DEST_ID, uri: DEST_DB }],