@anteros/core 0.0.40 → 0.0.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/database/audit.ts +143 -0
  2. package/database/collection.ts +6 -4
  3. package/database/deleteLog.ts +98 -0
  4. package/database/file.ts +7 -6
  5. package/database/mongodbadapter.ts +420 -185
  6. package/database/replication.ts +858 -0
  7. package/database/schema.ts +140 -125
  8. package/database/tenant.ts +4 -0
  9. package/database/ttl.ts +68 -0
  10. package/database/vars.ts +405 -0
  11. package/database/workflow.ts +466 -240
  12. package/index.ts +16 -0
  13. package/lib/agent.ts +1327 -0
  14. package/lib/agentMemory.ts +586 -0
  15. package/lib/agents.ts +157 -0
  16. package/lib/ai.ts +0 -0
  17. package/lib/api.ts +246 -0
  18. package/lib/attachment.ts +351 -0
  19. package/lib/audit.ts +271 -0
  20. package/lib/auditResult.ts +133 -0
  21. package/lib/define.ts +62 -1
  22. package/lib/endpoints.ts +80 -0
  23. package/lib/files.ts +66 -12
  24. package/lib/jsonSchema.ts +183 -0
  25. package/lib/lifecycle.ts +147 -0
  26. package/lib/load.ts +17 -0
  27. package/lib/mcp.ts +3 -2
  28. package/lib/messages.ts +299 -0
  29. package/lib/providers.ts +867 -0
  30. package/lib/redact.ts +117 -0
  31. package/lib/routes.ts +14 -1
  32. package/lib/scriptWorker.ts +4 -1
  33. package/lib/scripts.ts +6 -1
  34. package/lib/sockets.ts +3 -1
  35. package/lib/workflow.ts +209 -37
  36. package/lib/zip.ts +109 -0
  37. package/package.json +1 -1
  38. package/server/access.ts +96 -0
  39. package/server/agents.ts +451 -0
  40. package/server/api.ts +121 -95
  41. package/server/boot.ts +196 -53
  42. package/server/flags.ts +136 -0
  43. package/server/hono.ts +9 -4
  44. package/server/mcp.ts +17 -63
  45. package/server/routes.ts +19 -24
  46. package/tests/agent-memory.test.ts +617 -0
  47. package/tests/agent-working-memory.test.ts +402 -0
  48. package/tests/agent.test.ts +1239 -0
  49. package/tests/agents-api.test.ts +593 -0
  50. package/tests/api.test.ts +51 -0
  51. package/tests/attachment.test.ts +241 -0
  52. package/tests/audit-file.test.ts +296 -0
  53. package/tests/audit-operations.test.ts +184 -0
  54. package/tests/audit-redact.test.ts +259 -0
  55. package/tests/audit-results.test.ts +188 -0
  56. package/tests/audit-retention-config.test.ts +76 -0
  57. package/tests/audit-run-action.test.ts +120 -0
  58. package/tests/audit.test.ts +320 -0
  59. package/tests/db-cache.test.ts +7 -7
  60. package/tests/endpoints.test.ts +80 -0
  61. package/tests/file-collection-replication.test.ts +98 -0
  62. package/tests/fixtures/agent-tenant/agents/support.agent.ts +9 -0
  63. package/tests/fixtures/agent-tenant/agents/weather.agent.ts +21 -0
  64. package/tests/fixtures/agents-api/agents/assistant.agent.ts +46 -0
  65. package/tests/fixtures/agents-api/agents/plain.agent.ts +23 -0
  66. package/tests/fixtures/agents-api/agents/private.agent.ts +15 -0
  67. package/tests/fixtures/agents-api/collections/notes.model.ts +26 -0
  68. package/tests/fixtures/agents-api/routes/probe.route.ts +21 -0
  69. package/tests/fixtures/agents-api/services/probe.service.ts +19 -0
  70. package/tests/fixtures/attachment/data.csv +2 -0
  71. package/tests/fixtures/fake-provider.ts +128 -0
  72. package/tests/fixtures/lc-disabled/lifecycle.ts +11 -0
  73. package/tests/fixtures/lc-hang/lifecycle.ts +13 -0
  74. package/tests/fixtures/lc-normal/lifecycle.ts +23 -0
  75. package/tests/fixtures/lc-throw/lifecycle.ts +6 -0
  76. package/tests/fixtures/repl-only/app.ts +26 -0
  77. package/tests/fixtures/repl-only/tenant/collections/orders.model.ts +14 -0
  78. package/tests/fixtures/repl-only/tenant/lifecycle.ts +14 -0
  79. package/tests/fixtures/repl-only/tenant/scripts/probe.run.ts +8 -0
  80. package/tests/fixtures/replication-meta/agents/chat.agent.ts +16 -0
  81. package/tests/fixtures/replication-meta/collections/counters.model.ts +13 -0
  82. package/tests/fixtures/replication-meta/collections/orders.model.ts +9 -0
  83. package/tests/fixtures/replication-meta/workflows/run.workflow.ts +8 -0
  84. package/tests/fixtures/vars-tenant/vars/config.var.ts +23 -0
  85. package/tests/fixtures/vars-tenant/vars/private.var.ts +9 -0
  86. package/tests/fixtures/workflow-engine/workflows/failing.workflow.ts +34 -0
  87. package/tests/fixtures/workflow-engine/workflows/flaky.workflow.ts +23 -0
  88. package/tests/fixtures/workflow-engine/workflows/hang.workflow.ts +17 -0
  89. package/tests/fixtures/workflow-engine/workflows/simple.workflow.ts +19 -0
  90. package/tests/fixtures/workflow-engine/workflows/slow.workflow.ts +23 -0
  91. package/tests/fixtures/workflow-engine/workflows/versioned.workflow.ts +19 -0
  92. package/tests/fixtures/workflow-tenant/workflows/broken.workflow.ts +3 -0
  93. package/tests/fixtures/workflow-tenant/workflows/disabled.workflow.ts +8 -0
  94. package/tests/fixtures/workflow-tenant/workflows/invoice.workflow.ts +21 -0
  95. package/tests/fixtures/workflow-tenant/workflows/no-id.workflow.ts +7 -0
  96. package/tests/fixtures/workflow-tenant/workflows/order-duplicate.workflow.ts +13 -0
  97. package/tests/fixtures/workflow-tenant/workflows/order.workflow.ts +17 -0
  98. package/tests/hono.test.ts +6 -3
  99. package/tests/lifecycle.test.ts +96 -0
  100. package/tests/locks.test.ts +120 -0
  101. package/tests/mcp.test.ts +11 -11
  102. package/tests/messages.test.ts +258 -0
  103. package/tests/replication-control.test.ts +128 -0
  104. package/tests/replication-initial-sync.test.ts +86 -0
  105. package/tests/replication-meta.test.ts +243 -0
  106. package/tests/replication-only.test.ts +358 -0
  107. package/tests/replication.test.ts +188 -0
  108. package/tests/upload.test.ts +141 -8
  109. package/tests/vars-api.test.ts +123 -0
  110. package/tests/vars-replication.test.ts +96 -0
  111. package/tests/vars.test.ts +216 -0
  112. package/tests/workflow-engine.test.ts +340 -0
  113. package/tests/workflow-retention.test.ts +103 -0
  114. package/tests/workflow.test.ts +186 -0
  115. package/types/agent.d.ts +837 -0
  116. package/types/collection.d.ts +3 -0
  117. package/types/config.d.ts +80 -0
  118. package/types/file.d.ts +11 -8
  119. package/types/hook.d.ts +6 -0
  120. package/types/lifecycle.d.ts +77 -0
  121. package/types/mcp.d.ts +11 -1
  122. package/types/replication.d.ts +228 -0
  123. package/types/route.d.ts +7 -1
  124. package/types/scripts.d.ts +6 -0
  125. package/types/service.d.ts +7 -1
  126. package/types/tenant.d.ts +36 -0
  127. package/types/vars.d.ts +147 -0
  128. package/types/websocket.d.ts +6 -0
  129. package/types/workflow.d.ts +50 -3
  130. package/utils/func.ts +67 -52
@@ -0,0 +1,351 @@
1
+ import path from "path";
2
+ import { AppError } from "./error";
3
+ import { openZip, type ZipArchive } from "./zip";
4
+ import type { AgentAttachment, AgentMediaPart, AgentTextPart } from "../types/agent";
5
+
6
+ /**
7
+ * Attachments — images and documents handed to a model with a run.
8
+ *
9
+ * An LLM only understands three things: **text**, **images** and — for some
10
+ * models — **documents** (PDF). So every file is resolved to one of those:
11
+ *
12
+ * | Input | What the model receives |
13
+ * | --- | --- |
14
+ * | `.md` `.txt` `.json` `.csv` `.tsv` `.yaml` `.xml` `.html` `.log`, code… | the **text**, under a header naming the file |
15
+ * | `.png` `.jpg` `.jpeg` `.webp` `.gif` | an **image** part (base64) — native vision |
16
+ * | `.pdf` | a **file** part (base64) — native document input, where the model supports it |
17
+ * | `.docx` | the **text** extracted from `word/document.xml` (no dependency: the file is a ZIP) |
18
+ * | `.xlsx` | the **rows**, one CSV block per sheet, from the workbook parts |
19
+ * | `.doc` (legacy), anything else | refused — `AGENT_FILE_UNSUPPORTED` |
20
+ *
21
+ * The extraction of `.docx` / `.xlsx` is a deliberate simplification: it reads
22
+ * the text content, not the layout, the styles or the embedded objects. Good
23
+ * enough for a model; never a replacement for a document renderer.
24
+ */
25
+
26
+ /** Text formats read as-is — the cheapest and most faithful thing to do. */
27
+ const TEXT_EXTENSIONS = new Set([
28
+ "md", "markdown", "mdx", "txt", "text", "log",
29
+ "json", "jsonc", "ndjson", "csv", "tsv",
30
+ "yaml", "yml", "toml", "ini", "env",
31
+ "xml", "html", "htm", "svg",
32
+ "ts", "tsx", "js", "jsx", "mjs", "cjs", "py", "rb", "go", "rs", "java", "php",
33
+ "sql", "sh", "bash", "zsh", "css", "scss", "vue", "svelte", "graphql", "gql",
34
+ ]);
35
+
36
+ /** Image formats both providers accept natively. */
37
+ const IMAGE_MIME: Record<string, string> = {
38
+ png: "image/png",
39
+ jpg: "image/jpeg",
40
+ jpeg: "image/jpeg",
41
+ webp: "image/webp",
42
+ gif: "image/gif",
43
+ };
44
+
45
+ /** Formats the model can receive as a document part. */
46
+ const DOCUMENT_MIME: Record<string, string> = {
47
+ pdf: "application/pdf",
48
+ };
49
+
50
+ /** Extensions worth naming when we refuse a file. */
51
+ const OFFICE_BINARY: Record<string, string> = {
52
+ doc: "the legacy .doc format is not readable — convert it to .docx or PDF",
53
+ xls: "the legacy .xls format is not readable — convert it to .xlsx or CSV",
54
+ ppt: "the legacy .ppt format is not readable — convert it to .pptx or PDF",
55
+ pptx: "PowerPoint is not extracted — convert it to PDF",
56
+ };
57
+
58
+ /** One file never exceeds this — a model is not a file server. */
59
+ const MAX_FILE_BYTES = 10 * 1024 * 1024;
60
+ /** Inlined text is capped too: a 40 MB CSV must not become a 40 MB prompt. */
61
+ const MAX_TEXT_CHARS = 200_000;
62
+
63
+ function extensionOf(name: string): string {
64
+ return path.extname(name).replace(/^\./, "").toLowerCase();
65
+ }
66
+
67
+ function unsupported(name: string): AppError {
68
+ const extension = extensionOf(name);
69
+ const hint = OFFICE_BINARY[extension];
70
+ return new AppError(
71
+ hint
72
+ ? `Cannot read '${name}': ${hint}`
73
+ : `Cannot read '${name}': supported attachments are images (png, jpg, webp, gif), PDF, `
74
+ + `docx, xlsx and text formats (md, txt, json, csv, yaml, code…)`,
75
+ { status: 400, code: "AGENT_FILE_UNSUPPORTED" },
76
+ );
77
+ }
78
+
79
+ function tooLarge(name: string, size: number): AppError {
80
+ return new AppError(
81
+ `'${name}' is ${Math.round(size / 1024 / 1024)}MB — the limit is ${MAX_FILE_BYTES / 1024 / 1024}MB`,
82
+ { status: 413, code: "AGENT_FILE_TOO_LARGE" },
83
+ );
84
+ }
85
+
86
+ function toBase64(bytes: Uint8Array): string {
87
+ return Buffer.from(bytes).toString("base64");
88
+ }
89
+
90
+ /** Decode the XML entities an Office part can contain. */
91
+ function decodeEntities(text: string): string {
92
+ return text
93
+ .replace(/&lt;/g, "<")
94
+ .replace(/&gt;/g, ">")
95
+ .replace(/&quot;/g, '"')
96
+ .replace(/&apos;/g, "'")
97
+ .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
98
+ .replace(/&amp;/g, "&");
99
+ }
100
+
101
+ function stripTags(xml: string): string {
102
+ return decodeEntities(xml.replace(/<[^>]*>/g, ""));
103
+ }
104
+
105
+ // ─── Word ────────────────────────────────────────────────────────────────
106
+
107
+ /** `word/document.xml` → plain text (paragraphs, tabs and breaks preserved). */
108
+ function docxText(zip: ZipArchive): string {
109
+ const document = zip.read("word/document.xml");
110
+ if (!document) throw new AppError("Not a readable .docx (no word/document.xml)", {
111
+ status: 400, code: "AGENT_FILE_INVALID",
112
+ });
113
+
114
+ const xml = new TextDecoder().decode(document);
115
+ const text = xml
116
+ .replace(/<w:tab\b[^>]*\/>/g, "\t")
117
+ .replace(/<w:br\b[^>]*\/>/g, "\n")
118
+ .replace(/<\/w:p>/g, "\n")
119
+ .replace(/<[^>]*>/g, "");
120
+
121
+ return decodeEntities(text)
122
+ .replace(/[ \t]+\n/g, "\n")
123
+ .replace(/\n{3,}/g, "\n\n")
124
+ .trim();
125
+ }
126
+
127
+ // ─── Excel ───────────────────────────────────────────────────────────────
128
+
129
+ /** `xl/sharedStrings.xml` → the strings the cells refer to by index. */
130
+ function xlsxSharedStrings(zip: ZipArchive): string[] {
131
+ const shared = zip.read("xl/sharedStrings.xml");
132
+ if (!shared) return [];
133
+
134
+ const xml = new TextDecoder().decode(shared);
135
+ return [...xml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g)].map((match) =>
136
+ [...(match[1] ?? "").matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
137
+ .map((run) => decodeEntities(run[1] ?? ""))
138
+ .join(""));
139
+ }
140
+
141
+ /** `B7` → 1 (0-based column index), so gaps in a row are preserved. */
142
+ function columnOf(reference: string): number {
143
+ const letters = /^([A-Z]+)/.exec(reference)?.[1] ?? "";
144
+ let index = 0;
145
+ for (const letter of letters) index = index * 26 + (letter.charCodeAt(0) - 64);
146
+ return Math.max(0, index - 1);
147
+ }
148
+
149
+ function csvCell(value: string): string {
150
+ return /[",\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value;
151
+ }
152
+
153
+ /** One worksheet → CSV rows (values, not formulas — a model reads values). */
154
+ function xlsxSheet(xml: string, shared: string[]): string[] {
155
+ const rows: string[] = [];
156
+
157
+ for (const rowMatch of xml.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/g)) {
158
+ const cells: string[] = [];
159
+ for (const cellMatch of (rowMatch[1] ?? "").matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
160
+ const attributes = cellMatch[1] ?? "";
161
+ const body = cellMatch[2] ?? "";
162
+ const type = /\bt="([^"]+)"/.exec(attributes)?.[1];
163
+ const reference = /\br="([^"]+)"/.exec(attributes)?.[1] ?? "";
164
+ const position = reference ? columnOf(reference) : cells.length;
165
+
166
+ let value = "";
167
+ if (type === "s") {
168
+ const index = Number(stripTags(/\<v\>([\s\S]*?)\<\/v\>/.exec(body)?.[1] ?? ""));
169
+ value = shared[index] ?? "";
170
+ } else if (type === "inlineStr") {
171
+ value = [...body.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
172
+ .map((run) => decodeEntities(run[1] ?? "")).join("");
173
+ } else {
174
+ value = decodeEntities(/\<v\>([\s\S]*?)\<\/v\>/.exec(body)?.[1] ?? "");
175
+ }
176
+
177
+ while (cells.length < position) cells.push("");
178
+ cells[position] = value;
179
+ }
180
+ rows.push(cells.map(csvCell).join(","));
181
+ }
182
+
183
+ return rows;
184
+ }
185
+
186
+ /** Every worksheet, in tab order, as `### <sheet name>` + CSV rows. */
187
+ function xlsxText(zip: ZipArchive): string {
188
+ const sheets = zip.names()
189
+ .filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name))
190
+ .sort((a, b) => Number(/(\d+)\.xml$/.exec(a)?.[1]) - Number(/(\d+)\.xml$/.exec(b)?.[1]));
191
+
192
+ if (!sheets.length) throw new AppError("Not a readable .xlsx (no worksheet)", {
193
+ status: 400, code: "AGENT_FILE_INVALID",
194
+ });
195
+
196
+ const shared = xlsxSharedStrings(zip);
197
+
198
+ return sheets.map((sheet, index) => {
199
+ const xml = new TextDecoder().decode(zip.read(sheet) ?? new Uint8Array());
200
+ return `### Sheet ${index + 1}\n\n${xlsxSheet(xml, shared).join("\n")}`;
201
+ }).join("\n\n");
202
+ }
203
+
204
+ // ─── Resolution ──────────────────────────────────────────────────────────
205
+
206
+ /** What a caller can attach — a path (server-side), a `Blob`/`File`, raw bytes, or base64. */
207
+ export type NormalizedAttachment = {
208
+ name: string;
209
+ mimeType: string;
210
+ bytes: Uint8Array;
211
+ };
212
+
213
+ /**
214
+ * Read what the caller attached. A string is a **path** on the server (read
215
+ * now, at call time) — the SDK sends base64 instead, since a browser has no
216
+ * filesystem.
217
+ */
218
+ async function materialize(attachment: AgentAttachment, baseDir: string): Promise<NormalizedAttachment> {
219
+ if (typeof attachment === "string") {
220
+ const file = path.resolve(baseDir, attachment);
221
+ if (!(await Bun.file(file).exists())) {
222
+ throw new AppError(`Attachment '${attachment}' not found`, { status: 400, code: "AGENT_FILE_NOT_FOUND" });
223
+ }
224
+ const bytes = new Uint8Array(await Bun.file(file).arrayBuffer());
225
+ return { name: path.basename(file), mimeType: "", bytes };
226
+ }
227
+
228
+ if (attachment instanceof Uint8Array) {
229
+ return { name: "attachment", mimeType: "", bytes: attachment };
230
+ }
231
+
232
+ if (typeof Blob !== "undefined" && attachment instanceof Blob) {
233
+ const name = (attachment as File).name ?? "attachment";
234
+ return {
235
+ name,
236
+ mimeType: (attachment as File).type ?? "",
237
+ bytes: new Uint8Array(await attachment.arrayBuffer()),
238
+ };
239
+ }
240
+
241
+ const explicit = attachment as { name?: string; mimeType?: string; data: string | Uint8Array; encoding?: "base64" | "utf8" };
242
+ const name = explicit.name ?? "attachment";
243
+ if (explicit.data instanceof Uint8Array) {
244
+ return { name, mimeType: explicit.mimeType ?? "", bytes: explicit.data };
245
+ }
246
+
247
+ const encoding = explicit.encoding ?? "base64";
248
+ return {
249
+ name,
250
+ mimeType: explicit.mimeType ?? "",
251
+ bytes: encoding === "utf8"
252
+ ? new TextEncoder().encode(explicit.data)
253
+ : new Uint8Array(Buffer.from(explicit.data, "base64")),
254
+ };
255
+ }
256
+
257
+ /**
258
+ * Turn attachments into **content parts** the providers understand.
259
+ *
260
+ * @returns the parts to append to the current user turn (`[]` when there is
261
+ * nothing to attach). Throws `AGENT_FILE_*` when a file cannot be sent — a
262
+ * silently dropped attachment is worse than a loud failure.
263
+ */
264
+ async function resolveAttachments(
265
+ attachments: AgentAttachment | AgentAttachment[] | undefined,
266
+ options: { baseDir?: string } = {},
267
+ ): Promise<AgentMediaPart[]> {
268
+ if (!attachments) return [];
269
+ const list = Array.isArray(attachments) ? attachments : [attachments];
270
+ if (!list.length) return [];
271
+
272
+ const baseDir = options.baseDir ?? process.cwd();
273
+ const parts: AgentMediaPart[] = [];
274
+
275
+ for (const attachment of list) {
276
+ const file = await materialize(attachment, baseDir);
277
+ if (file.bytes.byteLength > MAX_FILE_BYTES) throw tooLarge(file.name, file.bytes.byteLength);
278
+
279
+ const extension = extensionOf(file.name);
280
+ const declared = file.mimeType.toLowerCase();
281
+ const mimeType = declared || IMAGE_MIME[extension] || DOCUMENT_MIME[extension] || "";
282
+
283
+ // 1. Images — native vision on both protocols
284
+ const imageMime = IMAGE_MIME[extension] ?? (declared.startsWith("image/") ? declared : "");
285
+ if (imageMime) {
286
+ parts.push({ type: "image", data: toBase64(file.bytes), mimeType: imageMime });
287
+ continue;
288
+ }
289
+
290
+ // 2/3. PDF — a document part (the provider decides what it does with it)
291
+ const documentMime = DOCUMENT_MIME[extension] ?? (declared === "application/pdf" ? declared : "");
292
+ if (documentMime) {
293
+ parts.push({ type: "file", name: file.name, mimeType: documentMime, data: toBase64(file.bytes) });
294
+ continue;
295
+ }
296
+
297
+ // 4. Office documents — extracted to text, no dependency
298
+ if (extension === "docx" || extension === "xlsx") {
299
+ const text = extension === "docx"
300
+ ? docxText(openZip(file.bytes))
301
+ : xlsxText(openZip(file.bytes));
302
+ parts.push({ type: "text", text: textPart(file.name, text) });
303
+ continue;
304
+ }
305
+
306
+ // 5. Text formats — read as-is
307
+ if (TEXT_EXTENSIONS.has(extension) || declared.startsWith("text/") || declared === "application/json") {
308
+ const text = new TextDecoder().decode(file.bytes);
309
+ parts.push({ type: "text", text: textPart(file.name, text) });
310
+ continue;
311
+ }
312
+
313
+ throw unsupported(file.name);
314
+ }
315
+
316
+ return parts;
317
+ }
318
+
319
+ /** A file's text under a header naming it — the model must know where it comes from. */
320
+ function textPart(name: string, text: string): string {
321
+ const trimmed = text.trim();
322
+ if (trimmed.length <= MAX_TEXT_CHARS) return `--- ${name} ---\n${trimmed}`;
323
+ return `--- ${name} (truncated) ---\n${trimmed.slice(0, MAX_TEXT_CHARS)}`;
324
+ }
325
+
326
+ /**
327
+ * What a **memory** keeps of an attachment. Conversations are text: storing a
328
+ * 5 MB image in every turn would fill the store for nothing, and the model
329
+ * already read it once. The reply and the question stay verbatim.
330
+ */
331
+ function attachmentNote(part: AgentMediaPart): string {
332
+ if (part.type === "image") return "[image]";
333
+ if (part.type === "file") return `[file ${part.name ?? "document"}]`;
334
+ const header = /^--- (.+?) ---/.exec(part.text)?.[1];
335
+ return header ? `[file ${header}]` : "";
336
+ }
337
+
338
+ /** A part as it should be *stored* — the heavy payloads replaced by a note. */
339
+ function forMemory(part: AgentMediaPart): AgentTextPart {
340
+ if (part.type === "text") return part;
341
+ return { type: "text", text: attachmentNote(part) };
342
+ }
343
+
344
+ export {
345
+ MAX_FILE_BYTES,
346
+ MAX_TEXT_CHARS,
347
+ docxText,
348
+ forMemory,
349
+ resolveAttachments,
350
+ xlsxText,
351
+ };
package/lib/audit.ts ADDED
@@ -0,0 +1,271 @@
1
+ import fs from "fs/promises";
2
+ import path from "path";
3
+ import { cfg } from "../server/config";
4
+ import type { Tenant } from "../types/tenant";
5
+ import { parseDuration } from "../utils/func";
6
+ import { resolveAuditRetention } from "../database/audit";
7
+
8
+ /**
9
+ * Local, append-only backup of the audit trail — one JSONL file per day,
10
+ * rotated on size and pruned by age. Meant for operators who want a copy of
11
+ * `_audit_` on the same machine as the logs, independent from MongoDB.
12
+ */
13
+ export type AuditFileOptions = {
14
+ /** Target directory — default: `<server.logging.dir>/audit` (`.logs/audit`) */
15
+ dir?: string;
16
+ /**
17
+ * Delete files older than this duration (`'30d'`).
18
+ * `false` → never prune. Omitted → mirrors `audit.retention` when set, else keeps files forever.
19
+ */
20
+ retention?: string | false;
21
+ /** Rotate when the current file exceeds this size in bytes (default: 50MB) */
22
+ maxSize?: number;
23
+ };
24
+
25
+ /** `true` → defaults, `string` → directory shorthand, object → full options */
26
+ export type AuditFileConfig = boolean | string | AuditFileOptions;
27
+
28
+ type ResolvedAuditFile = {
29
+ dir: string;
30
+ retentionMs: number | null;
31
+ maxSize: number;
32
+ };
33
+
34
+ type Sink = ResolvedAuditFile & {
35
+ /** Current file ('' until opened) */
36
+ file: string;
37
+ /** `YYYY-MM-DD` of the current file */
38
+ day: string;
39
+ size: number;
40
+ pending: string[];
41
+ writing: boolean;
42
+ prunedAt: number;
43
+ inFlight: Promise<void>;
44
+ };
45
+
46
+ const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50MB
47
+ const PRUNE_INTERVAL_MS = 5 * 60 * 1000; // prune at most every 5 minutes
48
+ const EXT = '.jsonl';
49
+
50
+ const sinks = new Map<string, Sink>();
51
+
52
+ /**
53
+ * Resolve the file backup settings — **the tenant always wins**,
54
+ * `server.audit.file` is only the fallback. Returns `null` when disabled.
55
+ */
56
+ function resolveAuditFile(tenant: Tenant): ResolvedAuditFile | null {
57
+ const raw: AuditFileConfig | undefined = tenant.audit?.file !== undefined
58
+ ? tenant.audit.file
59
+ : cfg.server?.audit?.file
60
+
61
+ if (raw === undefined || raw === false) return null
62
+
63
+ const options: AuditFileOptions = raw === true
64
+ ? {}
65
+ : typeof raw === 'string' ? { dir: raw } : raw
66
+
67
+ const dir = options.dir ?? path.join(cfg.server?.logging?.dir ?? '.logs', 'audit')
68
+
69
+ // Retention: explicit value wins (including `false`), otherwise the local
70
+ // copy mirrors the database retention so both follow the same policy.
71
+ const retention = options.retention !== undefined ? options.retention : resolveAuditRetention(tenant)
72
+ let retentionMs: number | null = null
73
+ if (retention !== undefined && retention !== false) {
74
+ const parsed = parseDuration(retention)
75
+ if (parsed === null) {
76
+ console.error(`Invalid audit file retention '${retention}' — expected a duration like '30d' or '24h'`)
77
+ } else {
78
+ retentionMs = parsed
79
+ }
80
+ }
81
+
82
+ return { dir, retentionMs, maxSize: options.maxSize ?? DEFAULT_MAX_SIZE }
83
+ }
84
+
85
+ function prefix(tenantId: string): string {
86
+ return `audit-${tenantId}`
87
+ }
88
+
89
+ function escapeRegex(value: string): string {
90
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
91
+ }
92
+
93
+ /** `JSON.stringify` that never throws (an activity must never break a request) */
94
+ function safeStringify(value: unknown): string {
95
+ try {
96
+ return JSON.stringify(value)
97
+ } catch {
98
+ return JSON.stringify({ error: 'unserializable activity' })
99
+ }
100
+ }
101
+
102
+ function getSink(key: string, resolved: ResolvedAuditFile): Sink {
103
+ let sink = sinks.get(key)
104
+ if (!sink) {
105
+ sink = {
106
+ ...resolved,
107
+ file: '',
108
+ day: '',
109
+ size: 0,
110
+ pending: [],
111
+ writing: false,
112
+ prunedAt: 0,
113
+ inFlight: Promise.resolve(),
114
+ }
115
+ sinks.set(key, sink)
116
+ } else {
117
+ // Options may change between boots / tests — always latest wins
118
+ sink.dir = resolved.dir
119
+ sink.retentionMs = resolved.retentionMs
120
+ sink.maxSize = resolved.maxSize
121
+ }
122
+ return sink
123
+ }
124
+
125
+ /**
126
+ * Pick the file `day` should be appended to: `audit-<tenant>-<day>.jsonl`,
127
+ * then `-1`, `-2`… when the size cap is reached.
128
+ *
129
+ * Files of the day are scanned in rotation order and the first one with room is
130
+ * reused — so a restart mid-day keeps appending to the current file, and the cap
131
+ * is never exceeded (a single line larger than the cap gets its own file).
132
+ */
133
+ async function openFile(sink: Sink, tenantId: string, day: string, bytes: number): Promise<void> {
134
+ await fs.mkdir(sink.dir, { recursive: true })
135
+
136
+ const base = `${prefix(tenantId)}-${day}`
137
+ const re = new RegExp(`^${escapeRegex(base)}(?:-(\\d+))?${escapeRegex(EXT)}$`)
138
+
139
+ const candidates: { index: number; name: string; size: number }[] = []
140
+ try {
141
+ for (const entry of await fs.readdir(sink.dir)) {
142
+ const match = entry.match(re)
143
+ if (!match) continue
144
+ try {
145
+ candidates.push({
146
+ index: Number(match[1] ?? 0),
147
+ name: entry,
148
+ size: (await fs.stat(path.join(sink.dir, entry))).size,
149
+ })
150
+ } catch { /* vanished between readdir and stat */ }
151
+ }
152
+ } catch { /* unreadable dir — treated as empty */ }
153
+
154
+ candidates.sort((a, b) => a.index - b.index)
155
+
156
+ const target = candidates.find((candidate) => candidate.size + bytes <= sink.maxSize)
157
+ if (target) {
158
+ sink.file = path.join(sink.dir, target.name)
159
+ sink.day = day
160
+ sink.size = target.size
161
+ return
162
+ }
163
+
164
+ const next = candidates.length ? candidates[candidates.length - 1]!.index + 1 : 0
165
+ sink.file = path.join(sink.dir, next === 0 ? `${base}${EXT}` : `${base}-${next}${EXT}`)
166
+ sink.day = day
167
+ sink.size = 0
168
+ }
169
+
170
+ /** Delete this tenant's audit files older than `retentionMs` (by mtime). Returns the removed names. */
171
+ async function pruneAuditFiles(dir: string, tenantId: string, retentionMs: number): Promise<string[]> {
172
+ const removed: string[] = []
173
+ try {
174
+ const re = new RegExp(`^${escapeRegex(prefix(tenantId))}-.*${escapeRegex(EXT)}$`)
175
+ const cutoff = Date.now() - retentionMs
176
+
177
+ for (const entry of await fs.readdir(dir)) {
178
+ if (!re.test(entry)) continue
179
+ const full = path.join(dir, entry)
180
+ try {
181
+ const stat = await fs.stat(full)
182
+ if (stat.mtimeMs < cutoff) {
183
+ await fs.unlink(full)
184
+ removed.push(entry)
185
+ }
186
+ } catch { /* already gone */ }
187
+ }
188
+ } catch { /* dir missing — nothing to prune */ }
189
+
190
+ return removed
191
+ }
192
+
193
+ async function maybePrune(sink: Sink, tenantId: string): Promise<void> {
194
+ if (sink.retentionMs === null) return
195
+ if (Date.now() - sink.prunedAt < PRUNE_INTERVAL_MS) return
196
+ sink.prunedAt = Date.now()
197
+ const removed = await pruneAuditFiles(sink.dir, tenantId, sink.retentionMs)
198
+ if (removed.length) {
199
+ console.log(`🧹 Audit files pruned (${tenantId}): ${removed.length} file(s)`)
200
+ }
201
+ }
202
+
203
+ /**
204
+ * Serialized per tenant: concurrent callers coalesce their lines into a single
205
+ * append, order is preserved, and lines are never interleaved.
206
+ */
207
+ async function drain(key: string, tenantId: string): Promise<void> {
208
+ const sink = sinks.get(key)
209
+ if (!sink || sink.writing) return
210
+ sink.writing = true
211
+
212
+ try {
213
+ while (sink.pending.length) {
214
+ const lines = sink.pending.splice(0)
215
+ const batch = lines.join('\n') + '\n'
216
+ const bytes = Buffer.byteLength(batch)
217
+ const day = new Date().toISOString().slice(0, 10)
218
+
219
+ try {
220
+ if (!sink.file || sink.day !== day || sink.size + bytes > sink.maxSize) {
221
+ await openFile(sink, tenantId, day, bytes)
222
+ }
223
+ await fs.appendFile(sink.file, batch)
224
+ } catch (err) {
225
+ // Keep the lines for the next call instead of losing them
226
+ sink.pending.unshift(...lines)
227
+ throw err
228
+ }
229
+
230
+ sink.size += bytes
231
+ await maybePrune(sink, tenantId)
232
+ }
233
+ } catch (err: any) {
234
+ console.error(`Audit file write failed (${tenantId})`, err?.message)
235
+ } finally {
236
+ sink.writing = false
237
+ }
238
+ }
239
+
240
+ /**
241
+ * Append activities to the tenant's local audit file — opt-in, fire-and-forget:
242
+ * it never blocks the caller and never throws.
243
+ */
244
+ function writeAuditFile(tenant: Tenant | null | undefined, activities: unknown[]): void {
245
+ if (!tenant || !activities?.length) return
246
+
247
+ const resolved = resolveAuditFile(tenant)
248
+ if (!resolved) return
249
+
250
+ const key = `${tenant.id}:${resolved.dir}`
251
+ const sink = getSink(key, resolved)
252
+ for (const activity of activities) sink.pending.push(safeStringify(activity))
253
+
254
+ sink.inFlight = drain(key, tenant.id).catch(() => {})
255
+ }
256
+
257
+ /** Wait until every pending line is on disk (tests, graceful shutdown). */
258
+ async function flushAuditFile(): Promise<void> {
259
+ for (let i = 0; i < 20; i++) {
260
+ const current = [...sinks.values()]
261
+ await Promise.all(current.map((sink) => sink.inFlight.catch(() => {})))
262
+ if (!current.some((sink) => sink.writing || sink.pending.length)) return
263
+ }
264
+ }
265
+
266
+ export {
267
+ resolveAuditFile,
268
+ writeAuditFile,
269
+ flushAuditFile,
270
+ pruneAuditFiles
271
+ }