@anteros/core 0.0.41 → 0.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/database/audit.ts +9 -59
- package/database/collection.ts +3 -1
- package/database/deleteLog.ts +67 -3
- package/database/file.ts +4 -3
- package/database/mongodbadapter.ts +103 -42
- package/database/replication.ts +110 -32
- package/database/ttl.ts +68 -0
- package/database/workflow.ts +466 -240
- package/index.ts +6 -0
- package/lib/agent.ts +1327 -0
- package/lib/agentMemory.ts +586 -0
- package/lib/agents.ts +157 -0
- package/lib/ai.ts +0 -0
- package/lib/api.ts +246 -0
- package/lib/attachment.ts +351 -0
- package/lib/auditResult.ts +13 -0
- package/lib/define.ts +43 -1
- package/lib/endpoints.ts +80 -0
- package/lib/files.ts +2 -1
- package/lib/jsonSchema.ts +183 -0
- package/lib/lifecycle.ts +7 -1
- package/lib/load.ts +17 -0
- package/lib/mcp.ts +3 -2
- package/lib/messages.ts +299 -0
- package/lib/providers.ts +867 -0
- package/lib/routes.ts +14 -1
- package/lib/scriptWorker.ts +4 -1
- package/lib/scripts.ts +6 -1
- package/lib/sockets.ts +3 -1
- package/lib/workflow.ts +209 -37
- package/lib/zip.ts +109 -0
- package/package.json +1 -1
- package/server/access.ts +96 -0
- package/server/agents.ts +451 -0
- package/server/api.ts +14 -72
- package/server/boot.ts +11 -0
- package/server/hono.ts +8 -3
- package/server/mcp.ts +17 -63
- package/server/routes.ts +19 -24
- package/tests/agent-memory.test.ts +617 -0
- package/tests/agent-working-memory.test.ts +402 -0
- package/tests/agent.test.ts +1239 -0
- package/tests/agents-api.test.ts +593 -0
- package/tests/api.test.ts +51 -0
- package/tests/attachment.test.ts +241 -0
- package/tests/audit-operations.test.ts +1 -1
- package/tests/audit-redact.test.ts +29 -3
- package/tests/audit-run-action.test.ts +1 -1
- package/tests/endpoints.test.ts +80 -0
- package/tests/fixtures/agent-tenant/agents/support.agent.ts +9 -0
- package/tests/fixtures/agent-tenant/agents/weather.agent.ts +21 -0
- package/tests/fixtures/agents-api/agents/assistant.agent.ts +46 -0
- package/tests/fixtures/agents-api/agents/plain.agent.ts +23 -0
- package/tests/fixtures/agents-api/agents/private.agent.ts +15 -0
- package/tests/fixtures/agents-api/collections/notes.model.ts +26 -0
- package/tests/fixtures/agents-api/routes/probe.route.ts +21 -0
- package/tests/fixtures/agents-api/services/probe.service.ts +19 -0
- package/tests/fixtures/attachment/data.csv +2 -0
- package/tests/fixtures/fake-provider.ts +128 -0
- package/tests/fixtures/replication-meta/agents/chat.agent.ts +16 -0
- package/tests/fixtures/replication-meta/collections/counters.model.ts +13 -0
- package/tests/fixtures/replication-meta/collections/orders.model.ts +9 -0
- package/tests/fixtures/replication-meta/workflows/run.workflow.ts +8 -0
- package/tests/fixtures/vars-tenant/vars/private.var.ts +4 -1
- package/tests/fixtures/workflow-engine/workflows/failing.workflow.ts +34 -0
- package/tests/fixtures/workflow-engine/workflows/flaky.workflow.ts +23 -0
- package/tests/fixtures/workflow-engine/workflows/hang.workflow.ts +17 -0
- package/tests/fixtures/workflow-engine/workflows/simple.workflow.ts +19 -0
- package/tests/fixtures/workflow-engine/workflows/slow.workflow.ts +23 -0
- package/tests/fixtures/workflow-engine/workflows/versioned.workflow.ts +19 -0
- package/tests/fixtures/workflow-tenant/workflows/broken.workflow.ts +3 -0
- package/tests/fixtures/workflow-tenant/workflows/disabled.workflow.ts +8 -0
- package/tests/fixtures/workflow-tenant/workflows/invoice.workflow.ts +21 -0
- package/tests/fixtures/workflow-tenant/workflows/no-id.workflow.ts +7 -0
- package/tests/fixtures/workflow-tenant/workflows/order-duplicate.workflow.ts +13 -0
- package/tests/fixtures/workflow-tenant/workflows/order.workflow.ts +17 -0
- package/tests/hono.test.ts +6 -3
- package/tests/locks.test.ts +120 -0
- package/tests/mcp.test.ts +11 -11
- package/tests/messages.test.ts +258 -0
- package/tests/replication-control.test.ts +2 -2
- package/tests/replication-meta.test.ts +243 -0
- package/tests/upload.test.ts +44 -12
- package/tests/vars-api.test.ts +1 -1
- package/tests/vars-replication.test.ts +3 -2
- package/tests/workflow-engine.test.ts +340 -0
- package/tests/workflow-retention.test.ts +103 -0
- package/tests/workflow.test.ts +186 -0
- package/types/agent.d.ts +837 -0
- package/types/config.d.ts +16 -0
- package/types/file.d.ts +7 -7
- package/types/hook.d.ts +6 -0
- package/types/lifecycle.d.ts +6 -0
- package/types/mcp.d.ts +11 -1
- package/types/replication.d.ts +29 -0
- package/types/route.d.ts +7 -1
- package/types/scripts.d.ts +6 -0
- package/types/service.d.ts +7 -1
- package/types/tenant.d.ts +13 -0
- package/types/vars.d.ts +2 -2
- package/types/websocket.d.ts +6 -0
- package/types/workflow.d.ts +50 -3
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { describe, it, expect, afterAll } from "bun:test";
|
|
2
|
+
import { normalizeMessages } from "../lib/messages";
|
|
3
|
+
import { MAX_FILE_BYTES } from "../lib/attachment";
|
|
4
|
+
|
|
5
|
+
/** A tiny local file server — the `file: { url }` path downloads through `fetch`. */
|
|
6
|
+
const servers: Array<ReturnType<typeof Bun.serve>> = [];
|
|
7
|
+
function serve(body: Uint8Array | string, contentType = "application/pdf") {
|
|
8
|
+
const server = Bun.serve({
|
|
9
|
+
port: 0,
|
|
10
|
+
hostname: "127.0.0.1",
|
|
11
|
+
fetch: () => new Response(body as any, { headers: { "Content-Type": contentType } }),
|
|
12
|
+
});
|
|
13
|
+
servers.push(server);
|
|
14
|
+
return `http://127.0.0.1:${server.port}/report.pdf`;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
afterAll(() => {
|
|
18
|
+
for (const server of servers) server.stop(true);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const first = async (input: any) => (await normalizeMessages(input))[0] as any;
|
|
22
|
+
|
|
23
|
+
describe("normalizeMessages — the shapes it accepts", () => {
|
|
24
|
+
it("turns a string into one user turn", async () => {
|
|
25
|
+
expect(await normalizeMessages("hi")).toEqual([{ role: "user", content: "hi" }]);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("accepts a single message — with or without an array", async () => {
|
|
29
|
+
const message = { role: "user", content: "hi" } as const;
|
|
30
|
+
expect(await normalizeMessages(message)).toEqual([{ role: "user", content: "hi" }]);
|
|
31
|
+
expect(await normalizeMessages([message])).toEqual([{ role: "user", content: "hi" }]);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("keeps a whole conversation in order", async () => {
|
|
35
|
+
const messages = await normalizeMessages([
|
|
36
|
+
{ role: "user", content: "hi" },
|
|
37
|
+
{ role: "assistant", content: "hello" },
|
|
38
|
+
{ role: "user", content: "and?" },
|
|
39
|
+
]);
|
|
40
|
+
expect(messages.map((message) => message.role)).toEqual(["user", "assistant", "user"]);
|
|
41
|
+
expect((messages[1] as any).content).toBe("hello");
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it("keeps only the canonical fields — a stray key never reaches the wire", async () => {
|
|
45
|
+
const [message] = await normalizeMessages([{ role: "user", content: "hi", threadId: "t", bogus: 1 } as any]);
|
|
46
|
+
expect(message).toEqual({ role: "user", content: "hi" });
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe("normalizeMessages — content parts", () => {
|
|
51
|
+
it("keeps text, image and file parts, url or base64", async () => {
|
|
52
|
+
const [message] = await normalizeMessages([{
|
|
53
|
+
role: "user",
|
|
54
|
+
content: [
|
|
55
|
+
{ type: "text", text: "look" },
|
|
56
|
+
{ type: "image", url: "https://example.com/a.png" },
|
|
57
|
+
{ type: "image", data: "AAAA", mimeType: "image/png" },
|
|
58
|
+
{ type: "file", name: "a.pdf", data: "BBBB", mimeType: "application/pdf" },
|
|
59
|
+
],
|
|
60
|
+
}] as any);
|
|
61
|
+
|
|
62
|
+
expect((message as any).content).toEqual([
|
|
63
|
+
{ type: "text", text: "look" },
|
|
64
|
+
{ type: "image", url: "https://example.com/a.png" },
|
|
65
|
+
{ type: "image", mimeType: "image/png", data: "AAAA" },
|
|
66
|
+
{ type: "file", name: "a.pdf", mimeType: "application/pdf", data: "BBBB" },
|
|
67
|
+
]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("joins a system message written as parts", async () => {
|
|
71
|
+
const [message] = await normalizeMessages([
|
|
72
|
+
{ role: "system", content: [{ type: "text", text: "a" }, { type: "text", text: "b" }] },
|
|
73
|
+
] as any);
|
|
74
|
+
expect(message).toEqual({ role: "system", content: "a\nb" });
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("downloads a remote file and inlines it as base64", async () => {
|
|
78
|
+
const url = serve("%PDF-1.7 payload");
|
|
79
|
+
const [message] = await normalizeMessages([
|
|
80
|
+
{ role: "user", content: [{ type: "file", url, name: "invoice.pdf" }] },
|
|
81
|
+
] as any);
|
|
82
|
+
|
|
83
|
+
const part: any = (message as any).content[0];
|
|
84
|
+
expect(part.type).toBe("file");
|
|
85
|
+
expect(part.name).toBe("invoice.pdf");
|
|
86
|
+
// The URL is gone — the payload travels as base64, which every provider takes
|
|
87
|
+
expect(part.url).toBeUndefined();
|
|
88
|
+
expect(Buffer.from(part.data, "base64").toString()).toBe("%PDF-1.7 payload");
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it("leaves a remote image as a URL — the provider fetches it", async () => {
|
|
92
|
+
const message: any = await first([{ role: "user", content: [{ type: "image", url: "https://example.com/a.png" }] }]);
|
|
93
|
+
expect(message.content[0].url).toBe("https://example.com/a.png");
|
|
94
|
+
expect(message.content[0].data).toBeUndefined();
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("refuses a part without a payload, an unknown address, or an unsupported scheme", async () => {
|
|
98
|
+
const cases = [
|
|
99
|
+
[{ type: "image" }],
|
|
100
|
+
[{ type: "file", name: "a.pdf" }],
|
|
101
|
+
[{ type: "image", url: "ftp://example.com/a.png" }],
|
|
102
|
+
[{ type: "image", url: "./local.png" }],
|
|
103
|
+
];
|
|
104
|
+
for (const content of cases) {
|
|
105
|
+
const error: any = await normalizeMessages([{ role: "user", content }] as any).catch((err) => err);
|
|
106
|
+
expect(error.code).toBe("INVALID_INPUT");
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("refuses a file that is too large", async () => {
|
|
111
|
+
const url = serve(new Uint8Array(MAX_FILE_BYTES + 1));
|
|
112
|
+
const error: any = await normalizeMessages([
|
|
113
|
+
{ role: "user", content: [{ type: "file", url, name: "huge.pdf" }] },
|
|
114
|
+
] as any).catch((err) => err);
|
|
115
|
+
expect(error.code).toBe("AGENT_FILE_TOO_LARGE");
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
describe("normalizeMessages — tool parts", () => {
|
|
120
|
+
it("hoists `tool-call` parts onto the assistant message", async () => {
|
|
121
|
+
const [message] = await normalizeMessages([{
|
|
122
|
+
role: "assistant",
|
|
123
|
+
content: [{ type: "tool-call", id: "call_1", name: "forecast", arguments: { city: "Paris" } }],
|
|
124
|
+
}] as any);
|
|
125
|
+
|
|
126
|
+
expect(message).toEqual({
|
|
127
|
+
role: "assistant",
|
|
128
|
+
content: null,
|
|
129
|
+
toolCalls: [{ id: "call_1", name: "forecast", args: { city: "Paris" } }],
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it("merges inline calls with the ones declared as `toolCalls`", async () => {
|
|
134
|
+
const [message] = await normalizeMessages([{
|
|
135
|
+
role: "assistant",
|
|
136
|
+
content: [{ type: "text", text: "let me check" }, { type: "tool-call", id: "b", name: "two" }],
|
|
137
|
+
toolCalls: [{ id: "a", name: "one", args: {} }],
|
|
138
|
+
}] as any) as any[];
|
|
139
|
+
|
|
140
|
+
expect(message.content).toBe("let me check");
|
|
141
|
+
expect(message.toolCalls.map((call: any) => call.id)).toEqual(["a", "b"]);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("keeps a raw `argsText` when the caller had one", async () => {
|
|
145
|
+
const [message] = await normalizeMessages([{
|
|
146
|
+
role: "assistant",
|
|
147
|
+
content: [{ type: "tool-call", id: "c", name: "f", argsText: '{"city":"Lyon"}', arguments: { city: "Lyon" } }],
|
|
148
|
+
}] as any) as any[];
|
|
149
|
+
expect(message.toolCalls[0]).toEqual({ id: "c", name: "f", args: { city: "Lyon" }, argsText: '{"city":"Lyon"}' });
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it("turns a `tool-result` part into the `tool` message it means", async () => {
|
|
153
|
+
const messages = await normalizeMessages([{
|
|
154
|
+
role: "user",
|
|
155
|
+
content: [
|
|
156
|
+
{ type: "text", text: "and?" },
|
|
157
|
+
{ type: "tool-result", id: "call_1", name: "forecast", result: { celsius: 21 } },
|
|
158
|
+
{ type: "tool-result", id: "call_2", name: "boom", error: "timeout" },
|
|
159
|
+
],
|
|
160
|
+
}] as any) as any[];
|
|
161
|
+
|
|
162
|
+
expect(messages.map((message) => message.role)).toEqual(["user", "tool", "tool"]);
|
|
163
|
+
expect(messages[0].content).toEqual([{ type: "text", text: "and?" }]);
|
|
164
|
+
expect(messages[1]).toEqual({
|
|
165
|
+
role: "tool", toolCallId: "call_1", name: "forecast", content: '{"celsius":21}',
|
|
166
|
+
});
|
|
167
|
+
expect(messages[2]).toEqual({
|
|
168
|
+
role: "tool", toolCallId: "call_2", name: "boom", content: "Error: timeout", isError: true,
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("expands a `tool` message carrying several results", async () => {
|
|
173
|
+
const messages = await normalizeMessages([{
|
|
174
|
+
role: "tool",
|
|
175
|
+
content: [
|
|
176
|
+
{ type: "tool-result", id: "a", result: "one" },
|
|
177
|
+
{ type: "tool-result", id: "b", result: "two" },
|
|
178
|
+
],
|
|
179
|
+
}] as any) as any[];
|
|
180
|
+
|
|
181
|
+
expect(messages).toEqual([
|
|
182
|
+
{ role: "tool", toolCallId: "a", name: "tool", content: "one" },
|
|
183
|
+
{ role: "tool", toolCallId: "b", name: "tool", content: "two" },
|
|
184
|
+
]);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it("keeps a `tool` message written the classic way", async () => {
|
|
188
|
+
const [message] = await normalizeMessages([
|
|
189
|
+
{ role: "tool", toolCallId: "call_1", name: "forecast", content: "21°C" },
|
|
190
|
+
] as any);
|
|
191
|
+
expect(message).toEqual({ role: "tool", toolCallId: "call_1", name: "forecast", content: "21°C" });
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
describe("normalizeMessages — what it refuses", () => {
|
|
196
|
+
it("refuses an unknown role and a message without one", async () => {
|
|
197
|
+
for (const input of [[{ role: "robot", content: "hi" }], [{ content: "hi" }], [null]]) {
|
|
198
|
+
const error: any = await normalizeMessages(input as any).catch((err) => err);
|
|
199
|
+
expect(error.code).toBe("INVALID_INPUT");
|
|
200
|
+
expect(error.status).toBe(400);
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("refuses a part the role cannot carry", async () => {
|
|
205
|
+
const cases: any[] = [
|
|
206
|
+
[{ role: "user", content: [{ type: "tool-call", id: "a", name: "f" }] }],
|
|
207
|
+
[{ role: "assistant", content: [{ type: "image", url: "https://example.com/a.png" }] }],
|
|
208
|
+
[{ role: "system", content: [{ type: "image", url: "https://example.com/a.png" }] }],
|
|
209
|
+
[{ role: "tool", content: [{ type: "text", text: "no" }] }],
|
|
210
|
+
[{ role: "tool", content: "orphan" }], // no `toolCallId`
|
|
211
|
+
[{ role: "user", content: [{ type: "tool-result" }] }], // no id
|
|
212
|
+
];
|
|
213
|
+
for (const input of cases) {
|
|
214
|
+
const error: any = await normalizeMessages(input as any).catch((err) => err);
|
|
215
|
+
expect(error.code).toBe("INVALID_INPUT");
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it("refuses a message without content", async () => {
|
|
220
|
+
for (const role of ["user", "system", "tool"]) {
|
|
221
|
+
const error: any = await normalizeMessages([{ role }] as any).catch((err) => err);
|
|
222
|
+
expect(error.code).toBe("INVALID_INPUT");
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
it("refuses an empty content array rather than sending an empty turn", async () => {
|
|
227
|
+
for (const input of [
|
|
228
|
+
[{ role: "user", content: [] }],
|
|
229
|
+
[{ role: "tool", content: [] }],
|
|
230
|
+
[{ role: "developer", content: [] }],
|
|
231
|
+
]) {
|
|
232
|
+
const error: any = await normalizeMessages(input as any).catch((err) => err);
|
|
233
|
+
expect(error.code).toBe("INVALID_INPUT");
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// An assistant turn with no text is legitimate — a tool call carries the turn
|
|
237
|
+
expect(await normalizeMessages([{ role: "assistant", content: [] }] as any))
|
|
238
|
+
.toEqual([{ role: "assistant", content: null }]);
|
|
239
|
+
});
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
describe("normalizeMessages — developer role and names", () => {
|
|
243
|
+
it("keeps the `developer` role and a message name", async () => {
|
|
244
|
+
const messages = await normalizeMessages([
|
|
245
|
+
{ role: "developer", content: "answer concisely", name: "policy" },
|
|
246
|
+
{ role: "user", content: "hi", name: "ada" },
|
|
247
|
+
] as any);
|
|
248
|
+
|
|
249
|
+
expect(messages[0]).toEqual({ role: "developer", content: "answer concisely", name: "policy" });
|
|
250
|
+
expect(messages[1]).toEqual({ role: "user", content: "hi", name: "ada" });
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it("keeps the raw thinking blocks of a replayed assistant turn", async () => {
|
|
254
|
+
const thinking = [{ type: "thinking", thinking: "…", signature: "sig" }];
|
|
255
|
+
const [message] = await normalizeMessages([{ role: "assistant", content: "done", thinking }] as any) as any[];
|
|
256
|
+
expect(message.thinking).toEqual(thinking);
|
|
257
|
+
});
|
|
258
|
+
});
|
|
@@ -102,7 +102,7 @@ describe("replication seed", () => {
|
|
|
102
102
|
const preMarker = `pre-${crypto.randomUUID()}`;
|
|
103
103
|
await rest.insertOne("orders", { title: preMarker });
|
|
104
104
|
|
|
105
|
-
const res = await seedReplication(TEST_TENANT, "latest");
|
|
105
|
+
const res = await seedReplication(TEST_TENANT, "latest", { collection: "orders" });
|
|
106
106
|
expect(res.seeded).toBe(1);
|
|
107
107
|
|
|
108
108
|
await replicateTenant(TEST_TENANT);
|
|
@@ -118,7 +118,7 @@ describe("replication seed", () => {
|
|
|
118
118
|
|
|
119
119
|
it("seeds from an explicit date", async () => {
|
|
120
120
|
await resetReplication(TEST_TENANT);
|
|
121
|
-
const res = await seedReplication(TEST_TENANT, "2020-01-01T00:00:00Z");
|
|
121
|
+
const res = await seedReplication(TEST_TENANT, "2020-01-01T00:00:00Z", { collection: "orders" });
|
|
122
122
|
expect(res.seeded).toBe(1);
|
|
123
123
|
|
|
124
124
|
const state = await getReplicationState(TEST_TENANT, DEST_ID, "orders");
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from "bun:test";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import { MongoClient } from "mongodb";
|
|
4
|
+
import { useRest } from "../database/rest";
|
|
5
|
+
import { formatConfig, cfg } from "../server/config";
|
|
6
|
+
import { syncTenants } from "../database/tenant";
|
|
7
|
+
import { syncCollections } from "../database/collection";
|
|
8
|
+
import { syncWorkflows } from "../lib/workflow";
|
|
9
|
+
import { startReplication, stopReplication, replicateTenant } from "../database/replication";
|
|
10
|
+
import { syncAgents } from "../lib/agents";
|
|
11
|
+
import { MongoAgentMemory } from "../lib/agentMemory";
|
|
12
|
+
import { AGENT_THREADS } from "./fixtures/replication-meta/agents/chat.agent";
|
|
13
|
+
import type { ReplicationMetaName } from "../types/replication";
|
|
14
|
+
|
|
15
|
+
const DIR = "packages/core/tests/fixtures/replication-meta";
|
|
16
|
+
|
|
17
|
+
/** Everything replicated: declared collection + the six framework collections */
|
|
18
|
+
const ALL_T = "rm-default";
|
|
19
|
+
const ALL_T_DB = "mongodb://localhost:27017/_RM_DEFAULT_SRC";
|
|
20
|
+
const ALL_T_DEST = "mongodb://localhost:27017/_RM_DEFAULT_DEST";
|
|
21
|
+
|
|
22
|
+
/** Nothing but the declared collection */
|
|
23
|
+
const OFF_T = "rm-alloff";
|
|
24
|
+
const OFF_T_DB = "mongodb://localhost:27017/_RM_ALLOFF_SRC";
|
|
25
|
+
const OFF_T_DEST = "mongodb://localhost:27017/_RM_ALLOFF_DEST";
|
|
26
|
+
|
|
27
|
+
let rest: InstanceType<typeof useRest>;
|
|
28
|
+
let dest: MongoClient;
|
|
29
|
+
let destOff: MongoClient;
|
|
30
|
+
|
|
31
|
+
const target = (slug: string) => dest.db().collection(slug);
|
|
32
|
+
const targetOff = (slug: string) => destOff.db().collection(slug);
|
|
33
|
+
|
|
34
|
+
beforeAll(async () => {
|
|
35
|
+
formatConfig({
|
|
36
|
+
server: { port: 4000 },
|
|
37
|
+
tenants: [
|
|
38
|
+
{
|
|
39
|
+
id: ALL_T,
|
|
40
|
+
dir: DIR,
|
|
41
|
+
database: { uri: ALL_T_DB },
|
|
42
|
+
// Opt-in retention, so the destination's copy is pruned the same way
|
|
43
|
+
audit: { retention: "30d" },
|
|
44
|
+
replication: {
|
|
45
|
+
runOnBoot: false,
|
|
46
|
+
schedule: { interval: "1h" },
|
|
47
|
+
destinations: [{ id: "backup", uri: ALL_T_DEST }],
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
id: OFF_T,
|
|
52
|
+
dir: DIR,
|
|
53
|
+
database: { uri: OFF_T_DB },
|
|
54
|
+
replication: {
|
|
55
|
+
runOnBoot: false,
|
|
56
|
+
schedule: { interval: "1h" },
|
|
57
|
+
exclude: ["audit", "workflows", "locks", "replication", "vars", "memory"],
|
|
58
|
+
destinations: [{ id: "backup", uri: OFF_T_DEST }],
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
await syncTenants();
|
|
65
|
+
await syncCollections();
|
|
66
|
+
await syncWorkflows();
|
|
67
|
+
await syncAgents(); // publishes the agent memory collections (`cfg.agentMemories`)
|
|
68
|
+
await startReplication();
|
|
69
|
+
|
|
70
|
+
rest = new useRest({ tenant_id: ALL_T });
|
|
71
|
+
dest = new MongoClient(ALL_T_DEST, { serverSelectionTimeoutMS: 3000 });
|
|
72
|
+
destOff = new MongoClient(OFF_T_DEST, { serverSelectionTimeoutMS: 3000 });
|
|
73
|
+
await dest.connect();
|
|
74
|
+
await destOff.connect();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
afterAll(async () => {
|
|
78
|
+
await stopReplication();
|
|
79
|
+
for (const uri of [ALL_T_DB, ALL_T_DEST, OFF_T_DB, OFF_T_DEST]) {
|
|
80
|
+
try {
|
|
81
|
+
const client = new MongoClient(uri);
|
|
82
|
+
await client.connect();
|
|
83
|
+
await client.db().dropDatabase();
|
|
84
|
+
await client.close();
|
|
85
|
+
} catch (_) {}
|
|
86
|
+
}
|
|
87
|
+
cfg.agents = [];
|
|
88
|
+
cfg.agentMemories = [];
|
|
89
|
+
try { await dest.close(); } catch (_) {}
|
|
90
|
+
try { await destOff.close(); } catch (_) {}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe("replication — framework collections", () => {
|
|
94
|
+
it("replicates them by default (audit, workflows, locks, replication, vars, memory)", async () => {
|
|
95
|
+
const marker = `meta-${crypto.randomUUID()}`;
|
|
96
|
+
|
|
97
|
+
await rest.audit.addActivities([{
|
|
98
|
+
internal: true,
|
|
99
|
+
trace: { id: crypto.randomUUID() },
|
|
100
|
+
meta: {},
|
|
101
|
+
operation: {
|
|
102
|
+
tenant: ALL_T, action: marker, collection: "orders",
|
|
103
|
+
status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
|
|
104
|
+
},
|
|
105
|
+
ts: new Date(),
|
|
106
|
+
} as any]);
|
|
107
|
+
await rest.workflow.run("run", { marker });
|
|
108
|
+
await rest.vars.set("config", marker, "META-1");
|
|
109
|
+
await rest.lock(`meta-${marker}`);
|
|
110
|
+
await rest.insertOne("orders", { title: marker });
|
|
111
|
+
// An agent conversation — the collection this tenant named for its memory
|
|
112
|
+
await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
|
|
113
|
+
"chat-1",
|
|
114
|
+
[{ role: "user", content: marker }] as any,
|
|
115
|
+
{ rest, tenant: ALL_T, resourceId: "u1" },
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
// A first run picks the state document it writes, hence the second one
|
|
119
|
+
await replicateTenant(ALL_T);
|
|
120
|
+
await replicateTenant(ALL_T);
|
|
121
|
+
|
|
122
|
+
expect(await target("_audit_").findOne({ "operation.action": marker })).toBeDefined();
|
|
123
|
+
expect(await target("_workflows_").findOne({ workflowId: "run" })).toBeDefined();
|
|
124
|
+
expect(await target("_vars_").findOne({ ns: "config", key: marker })).toBeDefined();
|
|
125
|
+
expect(await target("_locks_").findOne({ name: `meta-${marker}` })).toBeDefined();
|
|
126
|
+
expect(await target("_replication_").findOne({ type: "state", tenant: ALL_T })).toBeDefined();
|
|
127
|
+
expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-1" as any })).toBeDefined();
|
|
128
|
+
expect(await target("orders").findOne({ title: marker })).toBeDefined();
|
|
129
|
+
|
|
130
|
+
await rest.unlock(`meta-${marker}`);
|
|
131
|
+
}, 30_000);
|
|
132
|
+
|
|
133
|
+
it("propagates a cleared thread to the destination (tombstone)", async () => {
|
|
134
|
+
// The thread replicated by the previous test is deleted at the source: no
|
|
135
|
+
// date cursor can see a deletion, so the store leaves a tombstone.
|
|
136
|
+
await new MongoAgentMemory({ collection: AGENT_THREADS }).clear("chat-1", { rest, tenant: ALL_T, resourceId: "u1" });
|
|
137
|
+
|
|
138
|
+
await replicateTenant(ALL_T);
|
|
139
|
+
|
|
140
|
+
expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-1" as any })).toBeNull();
|
|
141
|
+
// The tombstone is consumed and purged
|
|
142
|
+
expect(await rest.db.collection("_replication_").countDocuments({ type: "delete", collection: AGENT_THREADS })).toBe(0);
|
|
143
|
+
}, 30_000);
|
|
144
|
+
|
|
145
|
+
it("uses each collection's own date key (audit is read on `ts`)", async () => {
|
|
146
|
+
const state: any = await rest.db.collection("_replication_")
|
|
147
|
+
.findOne({ type: "state", tenant: ALL_T, collection: "_audit_" });
|
|
148
|
+
expect(state).toBeDefined();
|
|
149
|
+
expect(state.key).toBe("ts");
|
|
150
|
+
// The state says which process is doing the work (`reusePort` / forked workers)
|
|
151
|
+
expect(state.pid).toBe(process.pid);
|
|
152
|
+
expect(state.hostname).toBe(os.hostname());
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("mirrors the audit retention on the destination", async () => {
|
|
156
|
+
const ttl: any = (await target("_audit_").listIndexes().toArray())
|
|
157
|
+
.find((i: any) => i.name === "_audit_ttl_");
|
|
158
|
+
expect(ttl).toBeDefined();
|
|
159
|
+
expect(ttl.expireAfterSeconds).toBe(30 * 86400);
|
|
160
|
+
}, 30_000);
|
|
161
|
+
|
|
162
|
+
it("leaves out only what `exclude` names", async () => {
|
|
163
|
+
const tenant: any = cfg.tenants?.find((t) => t.id === ALL_T);
|
|
164
|
+
const marker = `noaudit-${crypto.randomUUID()}`;
|
|
165
|
+
tenant.replication.exclude = ["audit", "memory"];
|
|
166
|
+
|
|
167
|
+
await rest.audit.addActivities([{
|
|
168
|
+
internal: true, trace: { id: crypto.randomUUID() }, meta: {},
|
|
169
|
+
operation: {
|
|
170
|
+
tenant: ALL_T, action: marker, collection: "orders",
|
|
171
|
+
status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
|
|
172
|
+
},
|
|
173
|
+
ts: new Date(),
|
|
174
|
+
} as any]);
|
|
175
|
+
await rest.vars.set("config", marker, "still-replicated");
|
|
176
|
+
await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
|
|
177
|
+
"chat-2",
|
|
178
|
+
[{ role: "user", content: marker }] as any,
|
|
179
|
+
{ rest, tenant: ALL_T, resourceId: "u1" },
|
|
180
|
+
);
|
|
181
|
+
|
|
182
|
+
await replicateTenant(ALL_T);
|
|
183
|
+
|
|
184
|
+
expect(await target("_audit_").findOne({ "operation.action": marker })).toBeNull(); // excluded
|
|
185
|
+
expect(await target(AGENT_THREADS).findOne({ _id: "u1:chat-2" as any })).toBeNull(); // excluded
|
|
186
|
+
expect(await target("_vars_").findOne({ ns: "config", key: marker })).toBeDefined(); // still there
|
|
187
|
+
|
|
188
|
+
delete tenant.replication.exclude;
|
|
189
|
+
}, 30_000);
|
|
190
|
+
|
|
191
|
+
it("replicates nothing from the framework when every name is excluded", async () => {
|
|
192
|
+
const offRest = new useRest({ tenant_id: OFF_T });
|
|
193
|
+
const marker = `alloff-${crypto.randomUUID()}`;
|
|
194
|
+
|
|
195
|
+
await offRest.audit.addActivities([{
|
|
196
|
+
internal: true, trace: { id: crypto.randomUUID() }, meta: {},
|
|
197
|
+
operation: {
|
|
198
|
+
tenant: OFF_T, action: marker, collection: "orders",
|
|
199
|
+
status: "success", input: null, result: null, error: null, duration: 0, transaction: false,
|
|
200
|
+
},
|
|
201
|
+
ts: new Date(),
|
|
202
|
+
} as any]);
|
|
203
|
+
await offRest.vars.set("config", marker, "not-replicated");
|
|
204
|
+
await offRest.insertOne("orders", { title: marker });
|
|
205
|
+
await new MongoAgentMemory({ collection: AGENT_THREADS }).save(
|
|
206
|
+
"chat-3",
|
|
207
|
+
[{ role: "user", content: marker }] as any,
|
|
208
|
+
{ rest: offRest, tenant: OFF_T, resourceId: "u1" },
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
await replicateTenant(OFF_T);
|
|
212
|
+
await replicateTenant(OFF_T);
|
|
213
|
+
|
|
214
|
+
// The declared collection still replicates…
|
|
215
|
+
expect(await targetOff("orders").findOne({ title: marker })).toBeDefined();
|
|
216
|
+
// …but the framework collections do not exist on that destination
|
|
217
|
+
expect(await targetOff("_audit_").countDocuments({})).toBe(0);
|
|
218
|
+
expect(await targetOff("_vars_").countDocuments({})).toBe(0);
|
|
219
|
+
expect(await targetOff("_workflows_").countDocuments({})).toBe(0);
|
|
220
|
+
expect(await targetOff(AGENT_THREADS).countDocuments({})).toBe(0);
|
|
221
|
+
}, 30_000);
|
|
222
|
+
|
|
223
|
+
it("propagates a delete on a collection whose key is a number", async () => {
|
|
224
|
+
const doc: any = await rest.insertOne("counters", { seq: 100, title: "numeric-key" });
|
|
225
|
+
await replicateTenant(ALL_T);
|
|
226
|
+
expect(await target("counters").findOne({ title: "numeric-key" })).toBeDefined();
|
|
227
|
+
|
|
228
|
+
await rest.deleteOne("counters", doc._id);
|
|
229
|
+
await replicateTenant(ALL_T);
|
|
230
|
+
// The tombstone guard compares the key to a date by default: a numeric key
|
|
231
|
+
// used to silently skip the delete
|
|
232
|
+
expect(await target("counters").findOne({ title: "numeric-key" })).toBeNull();
|
|
233
|
+
}, 30_000);
|
|
234
|
+
|
|
235
|
+
it("keeps the `exclude` list typed", () => {
|
|
236
|
+
const names: ReplicationMetaName[] = ["audit", "workflows", "locks", "replication", "vars", "memory"];
|
|
237
|
+
expect(names.length).toBe(6);
|
|
238
|
+
|
|
239
|
+
// @ts-expect-error — a declared collection slug is not a framework collection
|
|
240
|
+
const wrong: ReplicationMetaName[] = ["orders"];
|
|
241
|
+
expect(wrong.length).toBe(1);
|
|
242
|
+
});
|
|
243
|
+
});
|
package/tests/upload.test.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { formatConfig, cfg } from "../server/config";
|
|
|
5
5
|
import { syncTenants } from "../database/tenant";
|
|
6
6
|
import { getFileCollection } from "../database/file";
|
|
7
7
|
import { useRest } from "../database/rest";
|
|
8
|
+
import { createApi } from "../lib/api";
|
|
8
9
|
|
|
9
10
|
const TENANT = "up-test";
|
|
10
11
|
const DB = "mongodb://localhost:27017/_DB_UP_TEST";
|
|
@@ -75,7 +76,7 @@ describe("upload — audit trail", () => {
|
|
|
75
76
|
form.append("file", pngFile("audited.png"));
|
|
76
77
|
form.append("label", "audited");
|
|
77
78
|
|
|
78
|
-
const res = await fetch(`${url}/
|
|
79
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
79
80
|
expect(res.status).toBe(200);
|
|
80
81
|
const body: any = await res.json();
|
|
81
82
|
|
|
@@ -101,10 +102,10 @@ describe("upload — audit trail", () => {
|
|
|
101
102
|
const form = new FormData();
|
|
102
103
|
form.append("file", pngFile("deleted.png"));
|
|
103
104
|
const uploaded: any = await (
|
|
104
|
-
await fetch(`${url}/
|
|
105
|
+
await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form })
|
|
105
106
|
).json();
|
|
106
107
|
|
|
107
|
-
const res = await fetch(`${url}/
|
|
108
|
+
const res = await fetch(`${url}/api/${TENANT}/files/${SLUG}/${uploaded._id}`, { method: "DELETE" });
|
|
108
109
|
expect(res.status).toBe(200);
|
|
109
110
|
|
|
110
111
|
const logged: any = await auditEntry("deleteFile");
|
|
@@ -122,7 +123,7 @@ describe("upload — single file (unchanged contract)", () => {
|
|
|
122
123
|
form.append("file", pngFile("one.png"));
|
|
123
124
|
form.append("label", "single");
|
|
124
125
|
|
|
125
|
-
const res = await fetch(`${url}/
|
|
126
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
126
127
|
expect(res.status).toBe(200);
|
|
127
128
|
|
|
128
129
|
const body: any = await res.json();
|
|
@@ -130,7 +131,7 @@ describe("upload — single file (unchanged contract)", () => {
|
|
|
130
131
|
expect(body._id).toBeString();
|
|
131
132
|
expect(body._file.name).toBe("one.png");
|
|
132
133
|
expect(body._file.mimetype).toBe("image/png");
|
|
133
|
-
expect(body._file.url).toContain(`/
|
|
134
|
+
expect(body._file.url).toContain(`/api/${TENANT}/files/${SLUG}/`);
|
|
134
135
|
expect(body.createdAt).toBeString();
|
|
135
136
|
expect(body.updatedAt).toBeString();
|
|
136
137
|
|
|
@@ -148,7 +149,7 @@ describe("upload — single file (unchanged contract)", () => {
|
|
|
148
149
|
const form = new FormData();
|
|
149
150
|
form.append("upload", pngFile("alt.png"));
|
|
150
151
|
|
|
151
|
-
const res = await fetch(`${url}/
|
|
152
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
152
153
|
expect(res.status).toBe(200);
|
|
153
154
|
const body: any = await res.json();
|
|
154
155
|
expect(Array.isArray(body)).toBe(false);
|
|
@@ -164,7 +165,7 @@ describe("upload — multiple files", () => {
|
|
|
164
165
|
form.append("file", pngFile("c.png"));
|
|
165
166
|
form.append("label", "batch");
|
|
166
167
|
|
|
167
|
-
const res = await fetch(`${url}/
|
|
168
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
168
169
|
expect(res.status).toBe(200);
|
|
169
170
|
|
|
170
171
|
const body: any = await res.json();
|
|
@@ -178,7 +179,7 @@ describe("upload — multiple files", () => {
|
|
|
178
179
|
for (const id of ids) {
|
|
179
180
|
const doc: any = await rest.findOne(SLUG, id);
|
|
180
181
|
expect(doc.label).toBe("batch");
|
|
181
|
-
expect(doc._file.url).toContain(`/
|
|
182
|
+
expect(doc._file.url).toContain(`/api/${TENANT}/files/${SLUG}/`);
|
|
182
183
|
// Uploads carry timestamps (required by the replication cursor)
|
|
183
184
|
expect(doc.createdAt).toBeString();
|
|
184
185
|
expect(doc.updatedAt).toBeString();
|
|
@@ -189,7 +190,7 @@ describe("upload — multiple files", () => {
|
|
|
189
190
|
const form = new FormData();
|
|
190
191
|
form.append("label", "nofile");
|
|
191
192
|
|
|
192
|
-
const res = await fetch(`${url}/
|
|
193
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
193
194
|
expect(res.status).toBe(400);
|
|
194
195
|
const body: any = await res.json();
|
|
195
196
|
expect(body.code).toBe("FILE_REQUIRED");
|
|
@@ -200,7 +201,7 @@ describe("upload — multiple files", () => {
|
|
|
200
201
|
form.append("file", pngFile("ok.png"));
|
|
201
202
|
form.append("file", new File([Buffer.from("nope")], "bad.exe", { type: "application/x-msdownload" }));
|
|
202
203
|
|
|
203
|
-
const res = await fetch(`${url}/
|
|
204
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
204
205
|
expect(res.status).toBe(400);
|
|
205
206
|
const body: any = await res.json();
|
|
206
207
|
expect(body.code).toBe("MIMETYPE_NOT_ALLOWED");
|
|
@@ -214,7 +215,7 @@ describe("upload — custom field types (toBson, like collections)", () => {
|
|
|
214
215
|
form.append("label", "dated");
|
|
215
216
|
form.append("takenAt", "2024-05-01T10:00:00.000Z");
|
|
216
217
|
|
|
217
|
-
const res = await fetch(`${url}/
|
|
218
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
218
219
|
expect(res.status).toBe(200);
|
|
219
220
|
const body: any = await res.json();
|
|
220
221
|
|
|
@@ -227,7 +228,7 @@ describe("upload — custom field types (toBson, like collections)", () => {
|
|
|
227
228
|
const form = new FormData();
|
|
228
229
|
form.append("file", pngFile("2024-01-01")); // ← a file literally named like a date
|
|
229
230
|
|
|
230
|
-
const res = await fetch(`${url}/
|
|
231
|
+
const res = await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form });
|
|
231
232
|
expect(res.status).toBe(200);
|
|
232
233
|
const body: any = await res.json();
|
|
233
234
|
expect(body._file.name).toBe("2024-01-01");
|
|
@@ -237,3 +238,34 @@ describe("upload — custom field types (toBson, like collections)", () => {
|
|
|
237
238
|
expect(typeof raw._file.name).toBe("string");
|
|
238
239
|
});
|
|
239
240
|
});
|
|
241
|
+
|
|
242
|
+
describe("api.files — the in-process facade", () => {
|
|
243
|
+
it("builds a file URL, with the transformations the route understands", () => {
|
|
244
|
+
const api = createApi(rest);
|
|
245
|
+
|
|
246
|
+
expect(api.files.url(SLUG, "f1.png")).toBe(`/api/${TENANT}/files/${SLUG}/f1.png`);
|
|
247
|
+
expect(api.files.url(SLUG, "f1.png", { width: 200, format: "webp" }))
|
|
248
|
+
.toBe(`/api/${TENANT}/files/${SLUG}/f1.png?w=200&format=webp`);
|
|
249
|
+
expect(api.files.url(SLUG, "f1.png", { width: 200, height: 100, format: "avif", quality: 70 }))
|
|
250
|
+
.toBe(`/api/${TENANT}/files/${SLUG}/f1.png?w=200&h=100&format=avif&q=70`);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it("deletes the binary and the document — the same path as the route", async () => {
|
|
254
|
+
const form = new FormData();
|
|
255
|
+
form.append("file", pngFile("to-delete.png"));
|
|
256
|
+
const uploaded: any = await (
|
|
257
|
+
await fetch(`${url}/api/${TENANT}/upload/${SLUG}`, { method: "POST", body: form })
|
|
258
|
+
).json();
|
|
259
|
+
|
|
260
|
+
const fs = await import("node:fs/promises");
|
|
261
|
+
const storage = getFileCollection(SLUG, TENANT)?.storage as any;
|
|
262
|
+
const dir = `${process.cwd()}/${storage?.path ?? "storage"}/${TENANT}/${SLUG}`;
|
|
263
|
+
expect((await fs.readdir(dir)).some((name) => name.startsWith(uploaded._id))).toBe(true);
|
|
264
|
+
|
|
265
|
+
const result = await createApi(rest).files.delete(SLUG, uploaded._id);
|
|
266
|
+
|
|
267
|
+
expect(result).toEqual({ message: "File deleted", ok: true });
|
|
268
|
+
expect(await rest.findOne(SLUG, uploaded._id)).toBeNull();
|
|
269
|
+
expect((await fs.readdir(dir)).some((name) => name.startsWith(uploaded._id))).toBe(false);
|
|
270
|
+
});
|
|
271
|
+
});
|