@tumbaland/backend-core 1.38.0 → 1.40.0

package/src/audit/context.test.ts

@@ -0,0 +1,91 @@
+import type { Request, Response } from 'express';
+
+jest.mock('./reads', () => ({ recordRead: jest.fn() }));
+
+import { recordRead } from './reads';
+import { beginAudit, currentAuditContext, withAuditContext } from './context';
+import { PERSONAL_TENANT } from '../apiKeys/types';
+
+const ORIGINAL_ENV = process.env;
+
+const req = (over: Partial<Request> = {}): Request =>
+  ({ method: 'GET', query: {}, body: {}, baseUrl: '', path: '/', ...over }) as unknown as Request;
+
+const res = () => ({ on: jest.fn(), locals: {} }) as unknown as Response;
+const user = { id: 'u1', email: 'tom@example.com', name: 'Tom' };
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  process.env = { ...ORIGINAL_ENV };
+});
+
+afterEach(() => {
+  process.env = ORIGINAL_ENV;
+});
+
+describe('beginAudit', () => {
+  it('puts the actor in scope for everything after it', () => {
+    let seen: string | undefined;
+    beginAudit(req({ user }), res(), () => {
+      seen = currentAuditContext()?.actor.label;
+    });
+
+    expect(seen).toBe('Tom');
+  });
+
+  it('leaves no context behind once the request is done', () => {
+    beginAudit(req({ user }), res(), () => undefined);
+    // A leaked context would attribute the next request — or a background job —
+    // to whoever happened to come before it.
+    expect(currentAuditContext()).toBeUndefined();
+  });
+
+  it('continues without a context when nobody is authenticated', () => {
+    const next = jest.fn();
+    beginAudit(req(), res(), next);
+
+    expect(next).toHaveBeenCalled();
+    expect(recordRead).not.toHaveBeenCalled();
+  });
+
+  it('arms the read trail for the same request', () => {
+    beginAudit(req({ user }), res(), () => undefined);
+    expect(recordRead).toHaveBeenCalled();
+  });
+});
+
+describe('naming the service', () => {
+  it('uses SERVICE_NAME, as metrics and health already do', () => {
+    process.env.SERVICE_NAME = 'album-service';
+    let seen: string | undefined;
+
+    beginAudit(req({ user }), res(), () => {
+      seen = currentAuditContext()?.service;
+    });
+
+    expect(seen).toBe('album-service');
+  });
+
+  it('falls back to the package name, so local development is still labelled', () => {
+    // Nothing sets SERVICE_NAME in dev, and every row landing under one
+    // meaningless label makes the UI's area filter look broken.
+    delete process.env.SERVICE_NAME;
+    process.env.npm_package_name = 'finance-service';
+    let seen: string | undefined;
+
+    beginAudit(req({ user }), res(), () => {
+      seen = currentAuditContext()?.service;
+    });
+
+    expect(seen).toBe('finance-service');
+  });
+});
+
+describe('withAuditContext', () => {
+  it('runs work as an explicit actor, for jobs and tests', () => {
+    const actor = { kind: 'person' as const, userId: 'u1', label: 'Tom', tenant: PERSONAL_TENANT };
+    const seen = withAuditContext({ actor, service: 'test' }, () => currentAuditContext());
+
+    expect(seen?.actor.label).toBe('Tom');
+  });
+});

package/src/audit/context.ts

@@ -0,0 +1,83 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import type { NextFunction, Request, RequestHandler, Response } from 'express';
+import { actorOf, type Actor } from './actor';
+import { recordRead } from './reads';
+
+/**
+ * The actor, reachable from wherever the write actually happens.
+ *
+ * Attribution has to come from the request and be recorded at the data layer,
+ * and those are far apart: a Mongoose hook knows exactly which fields changed
+ * and nothing about who asked. Threading an actor through every service and
+ * controller signature to bridge that would touch every function between the
+ * two and be silently wrong the first time someone forgot.
+ *
+ * `AsyncLocalStorage` carries it instead, so the hook reads the actor without
+ * anything in between knowing it exists. A write outside a request — a script,
+ * a migration — simply finds no actor, which is the truth about it.
+ */
+export interface AuditContext {
+  actor: Actor;
+  service: string;
+}
+
+const storage = new AsyncLocalStorage<AuditContext>();
+
+export const currentAuditContext = (): AuditContext | undefined => storage.getStore();
+
+/**
+ * Run the rest of a request with its actor in scope.
+ *
+ * Mounted after authentication, since there is no actor before it.
+ */
+export const auditContext =
+  (service: string): RequestHandler =>
+  (req: Request, _res: Response, next: NextFunction): void => {
+    const actor = actorOf(req);
+    if (!actor) {
+      next();
+      return;
+    }
+
+    storage.run({ actor, service }, next);
+  };
+
+/** Run something with an explicit actor — for jobs and tests. */
+export const withAuditContext = <T>(context: AuditContext, fn: () => T): T =>
+  storage.run(context, fn);
+
+/**
+ * The service this process is, for a trail that spans several of them.
+ *
+ * `SERVICE_NAME` is already how metrics, health and the internal-service clients
+ * identify a process, so auditing uses the same one rather than introducing a
+ * second name for the same thing.
+ *
+ * The fallback to the package name matters more here than elsewhere: the trail
+ * is filtered by service in the UI, and in local development nothing sets
+ * `SERVICE_NAME`, so every row would land under one meaningless label and the
+ * filter would look broken. Every service's package is named after it.
+ */
+const serviceName = (): string =>
+  process.env.SERVICE_NAME || process.env.npm_package_name || 'unknown-service';
+
+/**
+ * Put a request's actor in scope, and arm the read trail, for everything after.
+ *
+ * Called from `authenticateAgent` rather than mounted as its own middleware. A
+ * separate mount only works on a router that authenticates with
+ * `router.use(...)`; routers that authenticate per route — album's photos,
+ * finance's tickers — have no actor yet when a router-level middleware runs, so
+ * the context would be silently empty for exactly the endpoints most worth
+ * auditing. Doing it where authentication happens makes the two inseparable.
+ */
+export function beginAudit(req: Request, res: Response, next: NextFunction): void {
+  const actor = actorOf(req);
+  if (!actor) {
+    next();
+    return;
+  }
+
+  recordRead(req, res, actor, serviceName());
+  storage.run({ actor, service: serviceName() }, next);
+}

package/src/audit/index.ts

@@ -0,0 +1,11 @@
+export { actorOf, describeActor } from './actor';
+export type { Actor, ActorKind } from './actor';
+export { AuditEvent } from './AuditEvent';
+export type { IAuditEvent, ToolCall } from './AuditEvent';
+export { auditContext, beginAudit, currentAuditContext, withAuditContext } from './context';
+export type { AuditContext } from './context';
+export { auditPlugin } from './plugin';
+export type { AuditPluginOptions } from './plugin';
+export { auditReads, countRead, resourceOf } from './reads';
+export { listAuditEvents } from './service';
+export type { AuditQuery, AuditEntry } from './service';

package/src/audit/plugin.test.ts

@@ -0,0 +1,258 @@
+/**
+ * Integration tests against a real MongoDB, not mocks.
+ *
+ * The whole value of this plugin is that it sees what actually changed, and a
+ * mocked Mongoose cannot tell you that: hooks that never fire, a `findOneAndUpdate`
+ * that returns whatever the mock was told to, a diff computed against a fixture
+ * rather than a document. Every bug worth catching here — a hook on the wrong
+ * event, an update whose before-image was read after the write, a delete whose
+ * snapshot came back empty — passes a mocked test and fails a real one.
+ */
+import mongoose, { Schema } from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+
+jest.mock('../logging/logger', () => ({
+  __esModule: true,
+  default: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn(), http: jest.fn() }
+}));
+
+import { AuditEvent } from './AuditEvent';
+import { auditPlugin } from './plugin';
+import { withAuditContext } from './context';
+import type { Actor } from './actor';
+
+let mongo: MongoMemoryServer;
+
+interface INote {
+  title: string;
+  hours?: number;
+  secret?: string;
+  createdBy?: string;
+  updatedBy?: string;
+}
+
+const NoteSchema = new Schema<INote>(
+  { title: String, hours: Number, secret: String },
+  { timestamps: true }
+);
+NoteSchema.plugin(auditPlugin, { resource: 'note', redact: ['secret'] });
+const Note = mongoose.model<INote>('AuditTestNote', NoteSchema);
+
+const CLAUDE: Actor = {
+  kind: 'assistant',
+  userId: 'u1',
+  label: 'Claude',
+  credentialId: 'client-1',
+  tenant: 'g1'
+};
+
+/**
+ * A Mongoose query builds lazily and only runs when it is awaited, so the await
+ * has to happen *inside* the context. Awaiting outside it — which reads
+ * identically at the call site — starts the query with no actor in scope and
+ * silently records nothing.
+ */
+const asClaude = <T>(fn: () => Promise<T>): Promise<T> =>
+  withAuditContext({ actor: CLAUDE, service: 'test-service' }, async () => await fn());
+
+const asPerson = <T>(fn: () => Promise<T>): Promise<T> =>
+  withAuditContext(
+    { actor: { kind: 'person', userId: 'u1', label: 'Tom', tenant: 'g1' }, service: 'test-service' },
+    async () => await fn()
+  );
+
+/** Recording is fire-and-forget, so a test has to let the write land. */
+const settle = () => new Promise((resolve) => setTimeout(resolve, 60));
+
+const events = () => AuditEvent.find().sort({ _id: 1 }).lean();
+
+beforeAll(async () => {
+  mongo = await MongoMemoryServer.create();
+  await mongoose.connect(mongo.getUri());
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongo.stop();
+});
+
+beforeEach(async () => {
+  await Promise.all([Note.deleteMany({}), AuditEvent.deleteMany({})]);
+});
+
+describe('attribution on the record', () => {
+  it('stamps who created it and who last touched it', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    expect(note.createdBy).toBe('Claude');
+    expect(note.updatedBy).toBe('Claude');
+  });
+
+  it('leaves createdBy alone on a later edit, and moves updatedBy', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    await asPerson(() => Note.findOneAndUpdate({ _id: note._id }, { title: 'Late dinner' }, { new: true }));
+
+    const after = await Note.findById(note._id);
+    expect(after?.createdBy).toBe('Claude');
+    expect(after?.updatedBy).toBe('Tom');
+  });
+
+  it('writes nothing at all outside a request', async () => {
+    // A backfill script is not a person, and a row claiming it was would be
+    // worse than no row.
+    const note = await new Note({ title: 'From a migration' }).save();
+    await settle();
+
+    expect(note.createdBy).toBeUndefined();
+    expect(await events()).toHaveLength(0);
+  });
+});
+
+describe('the trail', () => {
+  it('records a save of an existing document as an update, not a second create', async () => {
+    // Anything that loads a document, changes a field and saves it — revoking
+    // an API key, counting a reveal — would otherwise appear as the thing being
+    // created again, with a fresh snapshot and no sign of what changed.
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+
+    await asClaude(async () => {
+      const loaded = await Note.findById(note._id);
+      loaded!.hours = 2;
+      return loaded!.save();
+    });
+    await settle();
+
+    const actions = (await events()).map((e) => e.action);
+    expect(actions).toEqual(['create', 'update']);
+
+    const update = (await events()).find((e) => e.action === 'update');
+    expect(update?.changes).toEqual({ hours: { from: 3, to: 2 } });
+    expect(update?.snapshot).toBeUndefined();
+  });
+
+  it('records nothing when a save changed nothing', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    await asClaude(async () => {
+      const loaded = await Note.findById(note._id);
+      return loaded!.save();
+    });
+    await settle();
+
+    expect((await events()).filter((e) => e.action === 'update')).toHaveLength(0);
+  });
+
+  it('records a create with the document that was written', async () => {
+    await asClaude(() => new Note({ title: 'Dinner', hours: 2 }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(event).toMatchObject({
+      action: 'create',
+      resource: 'note',
+      actorKind: 'assistant',
+      actorLabel: 'Claude',
+      actorCredentialId: 'client-1',
+      tenant: 'g1',
+      service: 'test-service'
+    });
+    expect(event.snapshot).toMatchObject({ title: 'Dinner', hours: 2 });
+  });
+
+  it('records an update as before and after, field by field', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+    await settle();
+
+    const update = (await events()).find((e) => e.action === 'update');
+    // The point of the whole exercise: "Claude edited an entry" is barely worth
+    // storing; "Claude changed hours from 3 to 2" is what someone came to see.
+    expect(update?.changes).toEqual({ hours: { from: 3, to: 2 } });
+  });
+
+  it('never persists its own before-image into the document', async () => {
+    // The obvious way to stash state on a query is `query.set()`, and that
+    // writes into the update payload — so the before-image would be saved onto
+    // the very document it was taken of, growing a copy of each record inside
+    // itself on every edit.
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+
+    const raw = await mongoose.connection.db!.collection('audittestnotes').findOne({ _id: note._id });
+    expect(Object.keys(raw!).filter((key) => key.startsWith('__audit'))).toEqual([]);
+  });
+
+  it('records nothing for an update that changed nothing', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 3 }, { new: true }));
+    await settle();
+
+    expect((await events()).filter((e) => e.action === 'update')).toHaveLength(0);
+  });
+
+  it('reports only the fields the update touched', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+    await settle();
+
+    const update = (await events()).find((e) => e.action === 'update');
+    expect(Object.keys(update!.changes!)).toEqual(['hours']);
+  });
+
+  it('keeps what a delete removed, since nothing else does', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 2 }).save());
+    await asClaude(() => Note.findOneAndDelete({ _id: note._id }));
+    await settle();
+
+    const deleted = (await events()).find((e) => e.action === 'delete');
+    expect(deleted?.snapshot).toMatchObject({ title: 'Dinner', hours: 2 });
+    expect(deleted?.resourceId).toBe(String(note._id));
+  });
+
+  it('records a bulk delete one row per document', async () => {
+    // A day's save deletes the entries it replaces, and "the editor removed
+    // three things" is exactly what someone would come here to find.
+    await asClaude(async () => {
+      await new Note({ title: 'One' }).save();
+      await new Note({ title: 'Two' }).save();
+      await Note.deleteMany({});
+    });
+    await settle();
+
+    const deletes = (await events()).filter((e) => e.action === 'delete');
+    expect(deletes).toHaveLength(2);
+    expect(deletes.map((d) => (d.snapshot as { title: string }).title).sort()).toEqual(['One', 'Two']);
+  });
+
+  it('leaves redacted fields out of what it keeps', async () => {
+    await asClaude(() => new Note({ title: 'Dinner', secret: 'do not store' }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(JSON.stringify(event)).not.toContain('do not store');
+  });
+
+  it('omits bookkeeping fields nobody would want to read', async () => {
+    await asClaude(() => new Note({ title: 'Dinner' }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(Object.keys(event.snapshot!)).not.toContain('updatedAt');
+    expect(Object.keys(event.snapshot!)).not.toContain('__v');
+  });
+});
+
+describe('recording never breaks the thing it records', () => {
+  it('still saves the document when the trail cannot be written', async () => {
+    const create = jest.spyOn(AuditEvent, 'create').mockRejectedValue(new Error('audit db down'));
+
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+    await settle();
+
+    // Losing a row of history is a cost worth paying; losing the entry someone
+    // was logging is not.
+    expect(await Note.findById(note._id)).not.toBeNull();
+    create.mockRestore();
+  });
+});