@anteros/core 0.0.40 → 0.0.42

package/tests/attachment.test.ts

@@ -0,0 +1,241 @@
+import { describe, it, expect } from "bun:test";
+import { deflateRawSync } from "node:zlib";
+import { openZip } from "../lib/zip";
+import { docxText, resolveAttachments, xlsxText, MAX_FILE_BYTES } from "../lib/attachment";
+import { openaiMessages, anthropicMessages } from "../lib/providers";
+
+/**
+ * Attachments — a file becomes text, an image or a document part, and the zip
+ * reader behind `.docx` / `.xlsx` is exercised against **real** archives built by
+ * `makeZip` (the same bytes Office writes: local headers + central directory).
+ */
+
+/** Minimal ZIP writer (deflate or stored) — enough to build a .docx/.xlsx fixture. */
+function makeZip(entries: Record<string, string>, options: { store?: boolean } = {}): Uint8Array {
+    const encoder = new TextEncoder();
+    const locals: Uint8Array[] = [];
+    const central: Uint8Array[] = [];
+    let offset = 0;
+
+    const crcTable = (() => {
+        const table = new Uint32Array(256);
+        for (let i = 0; i < 256; i++) {
+            let c = i;
+            for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+            table[i] = c >>> 0;
+        }
+        return table;
+    })();
+
+    const crc32 = (bytes: Uint8Array): number => {
+        let crc = 0xffffffff;
+        for (const byte of bytes) crc = crcTable[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
+        return (crc ^ 0xffffffff) >>> 0;
+    };
+
+    for (const [name, content] of Object.entries(entries)) {
+        const raw = encoder.encode(content);
+        const method = options.store ? 0 : 8;
+        const data = options.store ? raw : new Uint8Array(deflateRawSync(raw));
+        const nameBytes = encoder.encode(name);
+        const crc = crc32(raw);
+
+        const local = new Uint8Array(30 + nameBytes.length + data.length);
+        const localView = new DataView(local.buffer);
+        localView.setUint32(0, 0x04034b50, true);
+        localView.setUint16(4, 20, true);           // version
+        localView.setUint16(8, method, true);
+        localView.setUint32(14, crc, true);
+        localView.setUint32(18, data.length, true); // compressed
+        localView.setUint32(22, raw.length, true);  // uncompressed
+        localView.setUint16(26, nameBytes.length, true);
+        local.set(nameBytes, 30);
+        local.set(data, 30 + nameBytes.length);
+        locals.push(local);
+
+        const entry = new Uint8Array(46 + nameBytes.length);
+        const entryView = new DataView(entry.buffer);
+        entryView.setUint32(0, 0x02014b50, true);
+        entryView.setUint16(4, 20, true);           // version made by
+        entryView.setUint16(6, 20, true);           // version needed
+        entryView.setUint16(10, method, true);
+        entryView.setUint32(16, crc, true);
+        entryView.setUint32(20, data.length, true);
+        entryView.setUint32(24, raw.length, true);
+        entryView.setUint16(28, nameBytes.length, true);
+        entryView.setUint32(42, offset, true);      // local header offset
+        entry.set(nameBytes, 46);
+        central.push(entry);
+
+        offset += local.length;
+    }
+
+    const centralSize = central.reduce((total, entry) => total + entry.length, 0);
+    const end = new Uint8Array(22);
+    const endView = new DataView(end.buffer);
+    endView.setUint32(0, 0x06054b50, true);
+    endView.setUint16(8, central.length, true);
+    endView.setUint16(10, central.length, true);
+    endView.setUint32(12, centralSize, true);
+    endView.setUint32(16, offset, true);
+
+    const total = offset + centralSize + 22;
+    const zip = new Uint8Array(total);
+    let cursor = 0;
+    for (const chunk of [...locals, ...central, end]) {
+        zip.set(chunk, cursor);
+        cursor += chunk.length;
+    }
+    return zip;
+}
+
+const DOCX = makeZip({
+    "[Content_Types].xml": "<Types/>",
+    "word/document.xml":
+        '<w:body><w:p><w:r><w:t>Invoice INV-2026-001</w:t></w:r></w:p>'
+        + "<w:p><w:r><w:t>Total: 1 200 &amp; 50 EUR</w:t></w:r></w:p>"
+        + "<w:p><w:r><w:tab/><w:t>Paid</w:t></w:r></w:p></w:body>",
+});
+
+const XLSX = makeZip({
+    "xl/sharedStrings.xml":
+        "<sst><si><t>Product</t></si><si><r><t>Widget</t></r><r><t> A</t></r></si>"
+        + "<si><t>Invoice INV-2026-001</t></si></sst>",
+    "xl/worksheets/sheet1.xml":
+        "<worksheet><sheetData>"
+        + '<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>'
+        + '<row r="2"><c r="A2" t="s"><v>2</v></c><c r="B2"><v>99.5</v></c><c r="C2" t="inlineStr"><is><t>x,y</t></is></c></row>'
+        + "</sheetData></worksheet>",
+});
+
+const qr = await resolveAttachments({ name: "photo.png", mimeType: "image/png", data: Buffer.from("PNG!").toString("base64") });
+
+describe("openZip", () => {
+    it("lists and reads the entries of a deflated archive", () => {
+        const zip = openZip(XLSX);
+        expect(zip.names().sort()).toEqual(["xl/sharedStrings.xml", "xl/worksheets/sheet1.xml"]);
+        expect(new TextDecoder().decode(zip.read("xl/sharedStrings.xml")!)).toContain("Invoice");
+        expect(zip.read("nope.xml")).toBeNull();
+    });
+
+    it("reads stored entries too, and refuses a non-archive", () => {
+        const zip = openZip(makeZip({ "a.txt": "hello" }, { store: true }));
+        expect(new TextDecoder().decode(zip.read("a.txt")!)).toBe("hello");
+
+        expect(() => openZip(new TextEncoder().encode("not a zip at all"))).toThrow(/central directory/);
+    });
+});
+
+describe("office extraction", () => {
+    it("extracts the text of a .docx, one line per paragraph", () => {
+        const text = docxText(openZip(DOCX));
+        expect(text).toContain("Invoice INV-2026-001");
+        expect(text).toContain("Total: 1 200 & 50 EUR"); // entities decoded
+        expect(text.split("\n").filter(Boolean)).toHaveLength(3);
+    });
+
+    it("extracts the rows of a .xlsx as CSV, shared strings resolved", () => {
+        const text = xlsxText(openZip(XLSX));
+        expect(text).toContain("### Sheet 1");
+        expect(text).toContain("Product,Widget A");           // concatenated runs
+        expect(text).toContain("Invoice INV-2026-001,99.5");  // shared string + number
+        expect(text).toContain('"x,y"');                      // a value with a comma is quoted
+    });
+});
+
+describe("resolveAttachments", () => {
+    it("turns text formats into a text part naming the file", async () => {
+        const parts = await resolveAttachments(
+            { name: "data.csv", data: Buffer.from("a,b\n1,2").toString("base64") },
+        );
+        expect(parts).toHaveLength(1);
+        expect(parts[0]).toEqual({ type: "text", text: "--- data.csv ---\na,b\n1,2" });
+    });
+
+    it("accepts UTF-8 payloads without base64", async () => {
+        const parts = await resolveAttachments({ name: "notes.md", data: "# Hi", encoding: "utf8" });
+        expect((parts[0] as any).text).toBe("--- notes.md ---\n# Hi");
+    });
+
+    it("keeps an image as an image part", () => {
+        expect(qr[0]).toEqual({ type: "image", data: Buffer.from("PNG!").toString("base64"), mimeType: "image/png" });
+    });
+
+    it("keeps a PDF as a document part", async () => {
+        const parts = await resolveAttachments({ name: "invoice.pdf", data: Buffer.from("%PDF-1.7").toString("base64") });
+        expect(parts[0]).toEqual({
+            type: "file",
+            name: "invoice.pdf",
+            mimeType: "application/pdf",
+            data: Buffer.from("%PDF-1.7").toString("base64"),
+        });
+    });
+
+    it("extracts .docx and .xlsx to text", async () => {
+        const [docx, xlsx] = await resolveAttachments([
+            { name: "invoice.docx", data: Buffer.from(DOCX).toString("base64") },
+            { name: "sales.xlsx", data: Buffer.from(XLSX).toString("base64") },
+        ]);
+
+        expect((docx as any).text).toContain("--- invoice.docx ---");
+        expect((docx as any).text).toContain("Invoice INV-2026-001");
+        expect((xlsx as any).text).toContain("Product,Widget A");
+    });
+
+    it("reads a path on the server — relative to the base dir", async () => {
+        const parts = await resolveAttachments("./fixtures/attachment/data.csv", {
+            baseDir: "packages/core/tests",
+        });
+        expect((parts[0] as any).text).toBe("--- data.csv ---\nref,total\nA-1,99");
+    });
+
+    it("refuses what a model cannot read, and says why", async () => {
+        await expect(resolveAttachments(
+            { name: "legacy.doc", data: Buffer.from("D0CF11E0").toString("base64") },
+        )).rejects.toMatchObject({ code: "AGENT_FILE_UNSUPPORTED" });
+
+        await expect(resolveAttachments(
+            { name: "archive.tar.gz", data: Buffer.from("...").toString("base64") },
+        )).rejects.toThrow(/supported attachments/);
+
+        await expect(resolveAttachments("./fixtures/attachment/missing.pdf", {
+            baseDir: "packages/core/tests",
+        })).rejects.toMatchObject({ code: "AGENT_FILE_NOT_FOUND" });
+    });
+
+    it("refuses a file over the size limit", async () => {
+        const oversized = { name: "big.pdf", data: new Uint8Array(MAX_FILE_BYTES + 1) };
+        await expect(resolveAttachments(oversized)).rejects.toMatchObject({ code: "AGENT_FILE_TOO_LARGE" });
+    });
+});
+
+describe("attachments through a provider", () => {
+    it("maps an image and a PDF to both protocols", () => {
+        const messages = [{
+            role: "user" as const,
+            content: [
+                { type: "text" as const, text: "What is this?" },
+                { type: "image" as const, data: "AAAA", mimeType: "image/png" },
+                { type: "file" as const, name: "invoice.pdf", mimeType: "application/pdf", data: "BBBB" },
+            ],
+        }];
+
+        const openai = openaiMessages(undefined, messages as any)[0];
+        expect(openai.content[0]).toEqual({ type: "text", text: "What is this?" });
+        expect(openai.content[1].image_url.url).toBe("data:image/png;base64,AAAA");
+        expect(openai.content[2]).toEqual({
+            type: "file",
+            file: { filename: "invoice.pdf", file_data: "data:application/pdf;base64,BBBB" },
+        });
+
+        const anthropic = anthropicMessages(messages as any).messages[0];
+        expect(anthropic.content[1]).toEqual({
+            type: "image",
+            source: { type: "base64", media_type: "image/png", data: "AAAA" },
+        });
+        expect(anthropic.content[2]).toEqual({
+            type: "document",
+            source: { type: "base64", media_type: "application/pdf", data: "BBBB" },
+        });
+    });
+});

package/tests/audit-file.test.ts

@@ -0,0 +1,296 @@
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { useRest } from "../database/rest";
+import { formatConfig, cfg } from "../server/config";
+import { syncTenants, getTenant } from "../database/tenant";
+import { resolveAuditFile, writeAuditFile, flushAuditFile, pruneAuditFiles } from "../lib/audit";
+import type { AuditFileOptions } from "../lib/audit";
+
+const DIR = path.join(import.meta.dir, ".tmp-audit-file");
+const TODAY = new Date().toISOString().slice(0, 10);
+
+/** Minimal activity document (same shape as `ActivityInput`). */
+function activity(action: string, collection = "items") {
+    return {
+        internal: true,
+        trace: { id: crypto.randomUUID() },
+        meta: {},
+        operation: {
+            tenant: "test",
+            action,
+            collection,
+            status: "success",
+            input: null,
+            result: null,
+            error: null,
+            duration: 0,
+            transaction: false,
+        },
+        ts: new Date(),
+    } as any;
+}
+
+function readLines(file: string): any[] {
+    return fs.readFileSync(file, "utf-8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
+}
+
+function filesOf(dir: string, tenantId: string): string[] {
+    try {
+        return fs.readdirSync(dir).filter((f) => f.startsWith(`audit-${tenantId}-`)).sort();
+    } catch {
+        return [];
+    }
+}
+
+beforeAll(() => {
+    try { fs.rmSync(DIR, { recursive: true, force: true }); } catch {}
+});
+
+afterAll(() => {
+    // `cfg.server` is shared by every test file in the same process: never leak
+    // the audit sink into the other suites (they would write files too)
+    delete (cfg.server as any).audit;
+    try { fs.rmSync(DIR, { recursive: true, force: true }); } catch {};
+});
+
+// ─── Disabled by default ────────────────────────────────────────────────
+
+describe("audit file — disabled by default", () => {
+    const TENANT = "af-off";
+    let rest: InstanceType<typeof useRest>;
+
+    beforeAll(async () => {
+        formatConfig({
+            server: { port: 4000 },
+            tenants: [{ id: TENANT, dir: "src", database: { uri: "mongodb://localhost:27017/_AUDIT_FILE_OFF" } }],
+        });
+        await syncTenants();
+        rest = new useRest({ internal: true, tenant_id: TENANT });
+    });
+
+    it("writes nothing when no config is set anywhere", async () => {
+        await rest.audit.addActivities([activity("off")]);
+        await flushAuditFile();
+
+        expect(filesOf(DIR, TENANT)).toEqual([]);
+        expect(fs.existsSync(path.join(DIR, TENANT))).toBe(false);
+    });
+
+    it("resolves to null", () => {
+        expect(resolveAuditFile({ id: TENANT, dir: "src", database: { uri: "" } } as any)).toBeNull();
+    });
+});
+
+// ─── Configured (tenant level and server default) ───────────────────────
+
+describe("audit file — writes and rotation", () => {
+    const TENANT = "af-tenant";
+    const ROT = "af-rot";
+    const OPTOUT = "af-optout";
+    const DIR_TENANT = path.join(DIR, "tenant");
+    const DIR_ROT = path.join(DIR, "rot");
+    const DIR_SERVER = path.join(DIR, "server");
+
+    let rest: InstanceType<typeof useRest>;
+    let rot: InstanceType<typeof useRest>;
+    let optout: InstanceType<typeof useRest>;
+
+    beforeAll(async () => {
+        formatConfig({
+            server: {
+                port: 4000,
+                // Server-wide default: everything goes to <DIR>/server unless a tenant says otherwise
+                audit: { file: { dir: DIR_SERVER } },
+            },
+            tenants: [
+                {
+                    id: TENANT,
+                    dir: "src",
+                    database: { uri: "mongodb://localhost:27017/_AUDIT_FILE_TENANT" },
+                    // Tenant wins: own dir + 7d retention (default size cap)
+                    audit: { file: { dir: DIR_TENANT, retention: "7d" } },
+                },
+                {
+                    id: ROT,
+                    dir: "src",
+                    database: { uri: "mongodb://localhost:27017/_AUDIT_FILE_ROT" },
+                    // Cap is set by the rotation test itself, once the real line size is known
+                    audit: { file: { dir: DIR_ROT } },
+                },
+                {
+                    id: OPTOUT,
+                    dir: "src",
+                    database: { uri: "mongodb://localhost:27017/_AUDIT_FILE_OPTOUT" },
+                    audit: { file: false }, // opt out of the server default
+                },
+            ],
+        });
+        await syncTenants();
+        rest = new useRest({ internal: true, tenant_id: TENANT });
+        rot = new useRest({ internal: true, tenant_id: ROT });
+        optout = new useRest({ internal: true, tenant_id: OPTOUT });
+    });
+
+    it("appends one JSONL line per activity, in a dated file", async () => {
+        await rest.audit.addActivities([activity("first"), activity("second")]);
+        await flushAuditFile();
+
+        const files = filesOf(DIR_TENANT, TENANT);
+        expect(files).toEqual([`audit-${TENANT}-${TODAY}.jsonl`]);
+
+        const lines = readLines(path.join(DIR_TENANT, files[0]!));
+        expect(lines.length).toBe(2);
+        expect(lines.map((l) => l.operation.action)).toEqual(["first", "second"]);
+        expect(typeof lines[0]!.ts).toBe("string"); // Date serialized as ISO
+    });
+
+    it("appends (never truncates) on the next call", async () => {
+        await rest.audit.addActivities([activity("third")]);
+        await flushAuditFile();
+
+        const lines = readLines(path.join(DIR_TENANT, `audit-${TENANT}-${TODAY}.jsonl`));
+        expect(lines.map((l) => l.operation.action)).toEqual(["first", "second", "third"]);
+    });
+
+    it("rotates when the size cap is reached", async () => {
+        // First activity reserves the plain file of the day
+        await rot.audit.addActivities([activity("rot-0")]);
+        await flushAuditFile();
+
+        const plain = path.join(DIR_ROT, `audit-${ROT}-${TODAY}.jsonl`);
+        const lineBytes = fs.statSync(plain).size;
+        // Cap sized so exactly two lines fit per file (line size is known now)
+        const cap = lineBytes * 2 + 5;
+        ((getTenant(ROT)!.audit!.file) as AuditFileOptions).maxSize = cap;
+
+        for (let i = 1; i < 4; i++) {
+            await rot.audit.addActivities([activity(`rot-${i}`)]);
+            await flushAuditFile();
+        }
+
+        // Chronological split: the plain file holds the two first lines,
+        // the `-1` rotation the next two
+        const files = filesOf(DIR_ROT, ROT);
+        const rotated = `audit-${ROT}-${TODAY}-1.jsonl`;
+        const name = `audit-${ROT}-${TODAY}.jsonl`;
+
+        expect(files.length).toBe(2);
+        expect(files).toContain(name);
+        expect(files).toContain(rotated);
+
+        const actions = (file: string) =>
+            readLines(path.join(DIR_ROT, file)).map((l) => l.operation.action);
+        expect(actions(name)).toEqual(["rot-0", "rot-1"]);
+        expect(actions(rotated)).toEqual(["rot-2", "rot-3"]);
+
+        // every file holds valid JSONL, stays under the cap, and no line is lost
+        for (const file of files) {
+            const raw = fs.readFileSync(path.join(DIR_ROT, file), "utf-8");
+            expect(raw.endsWith("\n")).toBe(true);
+            expect(raw.length).toBeLessThanOrEqual(cap);
+        }
+        expect([...actions(name), ...actions(rotated)])
+            .toEqual(["rot-0", "rot-1", "rot-2", "rot-3"]);
+    });
+
+    it("carries the `_id` assigned by MongoDB", async () => {
+        const files = filesOf(DIR_TENANT, TENANT);
+        const line = readLines(path.join(DIR_TENANT, files[0]!))[0]!;
+        expect(line._id).toBeString();
+        expect(line._id.length).toBe(24);
+    });
+
+    it("uses the server default when the tenant declares nothing", async () => {
+        const inherited = "af-inherit";
+        // Same server config, another tenant — resolved from `server.audit.file`
+        await rest.audit.addActivities([activity("own")]);
+        writeAuditFile({ id: inherited, dir: "src", database: { uri: "" } } as any, [activity("srv")]);
+        await flushAuditFile();
+
+        const files = filesOf(DIR_SERVER, inherited);
+        expect(files.length).toBe(1);
+        expect(readLines(path.join(DIR_SERVER, files[0]!))[0].operation.action).toBe("srv");
+    });
+
+    it("lets a tenant opt out of the server default with `false`", async () => {
+        await optout.audit.addActivities([activity("blocked")]);
+        await flushAuditFile();
+
+        expect(filesOf(DIR_SERVER, OPTOUT)).toEqual([]);
+        expect(filesOf(DIR_TENANT, OPTOUT)).toEqual([]);
+    });
+
+    it("prunes files older than the retention", async () => {
+        const stale = path.join(DIR_TENANT, `audit-${TENANT}-2020-01-01.jsonl`);
+        fs.writeFileSync(stale, "{}\n");
+        const old = new Date(Date.now() - 60 * 86400_000);
+        fs.utimesSync(stale, old, old);
+
+        const fresh = path.join(DIR_TENANT, `audit-${TENANT}-${TODAY}.jsonl`);
+        const before = filesOf(DIR_TENANT, TENANT);
+
+        const removed = await pruneAuditFiles(DIR_TENANT, TENANT, 7 * 86400_000);
+
+        expect(removed).toContain(`audit-${TENANT}-2020-01-01.jsonl`);
+        expect(fs.existsSync(stale)).toBe(false);
+        expect(fs.existsSync(fresh)).toBe(true);
+        expect(filesOf(DIR_TENANT, TENANT).length).toBe(before.length - 1);
+    });
+
+    it("only touches its own tenant's files", async () => {
+        const stale = path.join(DIR_TENANT, `audit-${OPTOUT}-2020-01-01.jsonl`);
+        fs.writeFileSync(stale, "{}\n");
+        const old = new Date(Date.now() - 60 * 86400_000);
+        fs.utimesSync(stale, old, old);
+
+        await pruneAuditFiles(DIR_TENANT, TENANT, 7 * 86400_000);
+
+        expect(fs.existsSync(stale)).toBe(true);
+        fs.rmSync(stale, { force: true });
+    });
+
+    it("mirrors the database retention when no file retention is set", () => {
+        const inherited = resolveAuditFile({
+            id: "x", dir: "src", database: { uri: "" }, audit: { retention: "30d", file: true },
+        } as any);
+        expect(inherited!.retentionMs).toBe(30 * 86400_000);
+        expect(inherited!.dir).toBe(path.join(".logs", "audit"));
+    });
+});
+
+// ─── Automatic pruning on first write ───────────────────────────────────
+
+describe("audit file — automatic pruning", () => {
+    const TENANT = "af-prune";
+    const DIR_PRUNE = path.join(DIR, "prune");
+    let rest: InstanceType<typeof useRest>;
+
+    beforeAll(async () => {
+        formatConfig({
+            server: { port: 4000 },
+            tenants: [{
+                id: TENANT,
+                dir: "src",
+                database: { uri: "mongodb://localhost:27017/_AUDIT_FILE_PRUNE" },
+                audit: { file: { dir: DIR_PRUNE, retention: "1d" } },
+            }],
+        });
+        await syncTenants();
+        rest = new useRest({ internal: true, tenant_id: TENANT });
+    });
+
+    it("deletes stale files on the first write, without waiting", async () => {
+        fs.mkdirSync(DIR_PRUNE, { recursive: true });
+        const stale = path.join(DIR_PRUNE, `audit-${TENANT}-2019-12-31.jsonl`);
+        fs.writeFileSync(stale, "{}\n");
+        const old = new Date(Date.now() - 30 * 86400_000);
+        fs.utimesSync(stale, old, old);
+
+        await rest.audit.addActivities([activity("trigger-prune")]);
+        await flushAuditFile();
+
+        expect(fs.existsSync(stale)).toBe(false);
+        expect(filesOf(DIR_PRUNE, TENANT).some((f) => f.includes(TODAY))).toBe(true);
+    });
+});