@tumbaland/backend-core 1.38.0 → 1.39.0

package/src/audit/context.ts

@@ -0,0 +1,83 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import type { NextFunction, Request, RequestHandler, Response } from 'express';
+import { actorOf, type Actor } from './actor';
+import { recordRead } from './reads';
+
+/**
+ * The actor, reachable from wherever the write actually happens.
+ *
+ * Attribution has to come from the request and be recorded at the data layer,
+ * and those are far apart: a Mongoose hook knows exactly which fields changed
+ * and nothing about who asked. Threading an actor through every service and
+ * controller signature to bridge that would touch every function between the
+ * two and be silently wrong the first time someone forgot.
+ *
+ * `AsyncLocalStorage` carries it instead, so the hook reads the actor without
+ * anything in between knowing it exists. A write outside a request — a script,
+ * a migration — simply finds no actor, which is the truth about it.
+ */
+export interface AuditContext {
+  actor: Actor;
+  service: string;
+}
+
+const storage = new AsyncLocalStorage<AuditContext>();
+
+export const currentAuditContext = (): AuditContext | undefined => storage.getStore();
+
+/**
+ * Run the rest of a request with its actor in scope.
+ *
+ * Mounted after authentication, since there is no actor before it.
+ */
+export const auditContext =
+  (service: string): RequestHandler =>
+  (req: Request, _res: Response, next: NextFunction): void => {
+    const actor = actorOf(req);
+    if (!actor) {
+      next();
+      return;
+    }
+
+    storage.run({ actor, service }, next);
+  };
+
+/** Run something with an explicit actor — for jobs and tests. */
+export const withAuditContext = <T>(context: AuditContext, fn: () => T): T =>
+  storage.run(context, fn);
+
+/**
+ * The service this process is, for a trail that spans several of them.
+ *
+ * `SERVICE_NAME` is already how metrics, health and the internal-service clients
+ * identify a process, so auditing uses the same one rather than introducing a
+ * second name for the same thing.
+ *
+ * The fallback to the package name matters more here than elsewhere: the trail
+ * is filtered by service in the UI, and in local development nothing sets
+ * `SERVICE_NAME`, so every row would land under one meaningless label and the
+ * filter would look broken. Every service's package is named after it.
+ */
+const serviceName = (): string =>
+  process.env.SERVICE_NAME || process.env.npm_package_name || 'unknown-service';
+
+/**
+ * Put a request's actor in scope, and arm the read trail, for everything after.
+ *
+ * Called from `authenticateAgent` rather than mounted as its own middleware. A
+ * separate mount only works on a router that authenticates with
+ * `router.use(...)`; routers that authenticate per route — album's photos,
+ * finance's tickers — have no actor yet when a router-level middleware runs, so
+ * the context would be silently empty for exactly the endpoints most worth
+ * auditing. Doing it where authentication happens makes the two inseparable.
+ */
+export function beginAudit(req: Request, res: Response, next: NextFunction): void {
+  const actor = actorOf(req);
+  if (!actor) {
+    next();
+    return;
+  }
+
+  recordRead(req, res, actor, serviceName());
+  storage.run({ actor, service: serviceName() }, next);
+}

package/src/audit/index.ts

@@ -0,0 +1,11 @@
+export { actorOf, describeActor } from './actor';
+export type { Actor, ActorKind } from './actor';
+export { AuditEvent } from './AuditEvent';
+export type { IAuditEvent } from './AuditEvent';
+export { auditContext, beginAudit, currentAuditContext, withAuditContext } from './context';
+export type { AuditContext } from './context';
+export { auditPlugin } from './plugin';
+export type { AuditPluginOptions } from './plugin';
+export { auditReads, countRead, resourceOf } from './reads';
+export { listAuditEvents } from './service';
+export type { AuditQuery, AuditEntry } from './service';

package/src/audit/plugin.test.ts

@@ -0,0 +1,258 @@
+/**
+ * Integration tests against a real MongoDB, not mocks.
+ *
+ * The whole value of this plugin is that it sees what actually changed, and a
+ * mocked Mongoose cannot tell you that: hooks that never fire, a `findOneAndUpdate`
+ * that returns whatever the mock was told to, a diff computed against a fixture
+ * rather than a document. Every bug worth catching here — a hook on the wrong
+ * event, an update whose before-image was read after the write, a delete whose
+ * snapshot came back empty — passes a mocked test and fails a real one.
+ */
+import mongoose, { Schema } from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+
+jest.mock('../logging/logger', () => ({
+  __esModule: true,
+  default: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn(), http: jest.fn() }
+}));
+
+import { AuditEvent } from './AuditEvent';
+import { auditPlugin } from './plugin';
+import { withAuditContext } from './context';
+import type { Actor } from './actor';
+
+let mongo: MongoMemoryServer;
+
+interface INote {
+  title: string;
+  hours?: number;
+  secret?: string;
+  createdBy?: string;
+  updatedBy?: string;
+}
+
+const NoteSchema = new Schema<INote>(
+  { title: String, hours: Number, secret: String },
+  { timestamps: true }
+);
+NoteSchema.plugin(auditPlugin, { resource: 'note', redact: ['secret'] });
+const Note = mongoose.model<INote>('AuditTestNote', NoteSchema);
+
+const CLAUDE: Actor = {
+  kind: 'assistant',
+  userId: 'u1',
+  label: 'Claude',
+  credentialId: 'client-1',
+  tenant: 'g1'
+};
+
+/**
+ * A Mongoose query builds lazily and only runs when it is awaited, so the await
+ * has to happen *inside* the context. Awaiting outside it — which reads
+ * identically at the call site — starts the query with no actor in scope and
+ * silently records nothing.
+ */
+const asClaude = <T>(fn: () => Promise<T>): Promise<T> =>
+  withAuditContext({ actor: CLAUDE, service: 'test-service' }, async () => await fn());
+
+const asPerson = <T>(fn: () => Promise<T>): Promise<T> =>
+  withAuditContext(
+    { actor: { kind: 'person', userId: 'u1', label: 'Tom', tenant: 'g1' }, service: 'test-service' },
+    async () => await fn()
+  );
+
+/** Recording is fire-and-forget, so a test has to let the write land. */
+const settle = () => new Promise((resolve) => setTimeout(resolve, 60));
+
+const events = () => AuditEvent.find().sort({ _id: 1 }).lean();
+
+beforeAll(async () => {
+  mongo = await MongoMemoryServer.create();
+  await mongoose.connect(mongo.getUri());
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongo.stop();
+});
+
+beforeEach(async () => {
+  await Promise.all([Note.deleteMany({}), AuditEvent.deleteMany({})]);
+});
+
+describe('attribution on the record', () => {
+  it('stamps who created it and who last touched it', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    expect(note.createdBy).toBe('Claude');
+    expect(note.updatedBy).toBe('Claude');
+  });
+
+  it('leaves createdBy alone on a later edit, and moves updatedBy', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    await asPerson(() => Note.findOneAndUpdate({ _id: note._id }, { title: 'Late dinner' }, { new: true }));
+
+    const after = await Note.findById(note._id);
+    expect(after?.createdBy).toBe('Claude');
+    expect(after?.updatedBy).toBe('Tom');
+  });
+
+  it('writes nothing at all outside a request', async () => {
+    // A backfill script is not a person, and a row claiming it was would be
+    // worse than no row.
+    const note = await new Note({ title: 'From a migration' }).save();
+    await settle();
+
+    expect(note.createdBy).toBeUndefined();
+    expect(await events()).toHaveLength(0);
+  });
+});
+
+describe('the trail', () => {
+  it('records a save of an existing document as an update, not a second create', async () => {
+    // Anything that loads a document, changes a field and saves it — revoking
+    // an API key, counting a reveal — would otherwise appear as the thing being
+    // created again, with a fresh snapshot and no sign of what changed.
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+
+    await asClaude(async () => {
+      const loaded = await Note.findById(note._id);
+      loaded!.hours = 2;
+      return loaded!.save();
+    });
+    await settle();
+
+    const actions = (await events()).map((e) => e.action);
+    expect(actions).toEqual(['create', 'update']);
+
+    const update = (await events()).find((e) => e.action === 'update');
+    expect(update?.changes).toEqual({ hours: { from: 3, to: 2 } });
+    expect(update?.snapshot).toBeUndefined();
+  });
+
+  it('records nothing when a save changed nothing', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+
+    await asClaude(async () => {
+      const loaded = await Note.findById(note._id);
+      return loaded!.save();
+    });
+    await settle();
+
+    expect((await events()).filter((e) => e.action === 'update')).toHaveLength(0);
+  });
+
+  it('records a create with the document that was written', async () => {
+    await asClaude(() => new Note({ title: 'Dinner', hours: 2 }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(event).toMatchObject({
+      action: 'create',
+      resource: 'note',
+      actorKind: 'assistant',
+      actorLabel: 'Claude',
+      actorCredentialId: 'client-1',
+      tenant: 'g1',
+      service: 'test-service'
+    });
+    expect(event.snapshot).toMatchObject({ title: 'Dinner', hours: 2 });
+  });
+
+  it('records an update as before and after, field by field', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+    await settle();
+
+    const update = (await events()).find((e) => e.action === 'update');
+    // The point of the whole exercise: "Claude edited an entry" is barely worth
+    // storing; "Claude changed hours from 3 to 2" is what someone came to see.
+    expect(update?.changes).toEqual({ hours: { from: 3, to: 2 } });
+  });
+
+  it('never persists its own before-image into the document', async () => {
+    // The obvious way to stash state on a query is `query.set()`, and that
+    // writes into the update payload — so the before-image would be saved onto
+    // the very document it was taken of, growing a copy of each record inside
+    // itself on every edit.
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+
+    const raw = await mongoose.connection.db!.collection('audittestnotes').findOne({ _id: note._id });
+    expect(Object.keys(raw!).filter((key) => key.startsWith('__audit'))).toEqual([]);
+  });
+
+  it('records nothing for an update that changed nothing', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 3 }, { new: true }));
+    await settle();
+
+    expect((await events()).filter((e) => e.action === 'update')).toHaveLength(0);
+  });
+
+  it('reports only the fields the update touched', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 3 }).save());
+    await asClaude(() => Note.findOneAndUpdate({ _id: note._id }, { hours: 2 }, { new: true }));
+    await settle();
+
+    const update = (await events()).find((e) => e.action === 'update');
+    expect(Object.keys(update!.changes!)).toEqual(['hours']);
+  });
+
+  it('keeps what a delete removed, since nothing else does', async () => {
+    const note = await asClaude(() => new Note({ title: 'Dinner', hours: 2 }).save());
+    await asClaude(() => Note.findOneAndDelete({ _id: note._id }));
+    await settle();
+
+    const deleted = (await events()).find((e) => e.action === 'delete');
+    expect(deleted?.snapshot).toMatchObject({ title: 'Dinner', hours: 2 });
+    expect(deleted?.resourceId).toBe(String(note._id));
+  });
+
+  it('records a bulk delete one row per document', async () => {
+    // A day's save deletes the entries it replaces, and "the editor removed
+    // three things" is exactly what someone would come here to find.
+    await asClaude(async () => {
+      await new Note({ title: 'One' }).save();
+      await new Note({ title: 'Two' }).save();
+      await Note.deleteMany({});
+    });
+    await settle();
+
+    const deletes = (await events()).filter((e) => e.action === 'delete');
+    expect(deletes).toHaveLength(2);
+    expect(deletes.map((d) => (d.snapshot as { title: string }).title).sort()).toEqual(['One', 'Two']);
+  });
+
+  it('leaves redacted fields out of what it keeps', async () => {
+    await asClaude(() => new Note({ title: 'Dinner', secret: 'do not store' }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(JSON.stringify(event)).not.toContain('do not store');
+  });
+
+  it('omits bookkeeping fields nobody would want to read', async () => {
+    await asClaude(() => new Note({ title: 'Dinner' }).save());
+    await settle();
+
+    const [event] = await events();
+    expect(Object.keys(event.snapshot!)).not.toContain('updatedAt');
+    expect(Object.keys(event.snapshot!)).not.toContain('__v');
+  });
+});
+
+describe('recording never breaks the thing it records', () => {
+  it('still saves the document when the trail cannot be written', async () => {
+    const create = jest.spyOn(AuditEvent, 'create').mockRejectedValue(new Error('audit db down'));
+
+    const note = await asClaude(() => new Note({ title: 'Dinner' }).save());
+    await settle();
+
+    // Losing a row of history is a cost worth paying; losing the entry someone
+    // was logging is not.
+    expect(await Note.findById(note._id)).not.toBeNull();
+    create.mockRestore();
+  });
+});

package/src/audit/plugin.ts

@@ -0,0 +1,254 @@
+import type { Model, Schema } from 'mongoose';
+import logger from '../logging/logger';
+import { AuditEvent } from './AuditEvent';
+import { currentAuditContext } from './context';
+
+/** Bookkeeping fields nobody wants to read in a change list. */
+const NOISE = new Set(['updatedAt', 'createdAt', '__v', 'createdBy', 'updatedBy']);
+
+/**
+ * Where a hook stashes what it read before the write.
+ *
+ * A symbol on the query object, not `query.set()` — that method writes into the
+ * *update payload*, so stashing there would persist the before-image into the
+ * document it was taken of. Per-query, so concurrent writes cannot read each
+ * other's.
+ */
+const BEFORE = Symbol('auditBefore');
+
+/** Comparable form, so a Date and an ObjectId do not read as changed every time. */
+const plain = (value: unknown): unknown => {
+  if (value === null || value === undefined) return value;
+  if (value instanceof Date) return value.toISOString();
+  if (typeof value === 'object' && 'toHexString' in (value as object)) return String(value);
+  return value;
+};
+
+const changesBetween = (
+  before: Record<string, unknown>,
+  after: Record<string, unknown>,
+  fields: string[]
+): Record<string, { from: unknown; to: unknown }> | undefined => {
+  const changes: Record<string, { from: unknown; to: unknown }> = {};
+
+  for (const field of fields) {
+    if (NOISE.has(field)) continue;
+    const from = plain(before?.[field]);
+    const to = plain(after?.[field]);
+    if (JSON.stringify(from) !== JSON.stringify(to)) changes[field] = { from, to };
+  }
+
+  return Object.keys(changes).length > 0 ? changes : undefined;
+};
+
+/**
+ * Never let recording a thing fail the thing.
+ *
+ * An audit trail is worth having and is not worth refusing a write over: a
+ * failure here means someone loses a row of history, and throwing would mean
+ * they lose the entry they were logging. So it is fire-and-forget with the
+ * failure written to the ordinary log, where an operator sees it.
+ */
+const record = (event: Record<string, unknown>): void => {
+  void AuditEvent.create(event).catch((error) => {
+    logger.warn('Could not record an audit event', {
+      error: (error as Error)?.message,
+      action: event.action,
+      resource: event.resource
+    });
+  });
+};
+
+export interface AuditPluginOptions {
+  /** what these documents are called in a trail — 'activity', 'photo' */
+  resource: string;
+  /** fields never worth recording the content of */
+  redact?: string[];
+}
+
+/**
+ * Attribute and record every write to a collection.
+ *
+ * Applied to a model rather than called from its controllers, for two reasons.
+ * The data layer is the only place that knows which fields actually changed —
+ * a controller sees the patch that was requested, not the difference it made,
+ * and those differ whenever a field was already the value being set. And a
+ * plugin cannot be forgotten: a new endpoint that writes through the model is
+ * audited without anyone remembering to add a line.
+ *
+ * Writes made outside a request — a migration, a backfill script — find no
+ * actor and are recorded as nothing, which is honest. A row claiming a script
+ * was a person would be worse than no row.
+ */
+export function auditPlugin(schema: Schema, options: AuditPluginOptions): void {
+  const { resource, redact = [] } = options;
+  const hide = new Set(redact);
+
+  const visible = (doc: Record<string, unknown>): Record<string, unknown> =>
+    Object.fromEntries(
+      Object.entries(doc)
+        .filter(([key]) => !NOISE.has(key) && !hide.has(key))
+        .map(([key, value]) => [key, plain(value)])
+    );
+
+  // Who last touched this, on the record itself. Denormalized on purpose: a day
+  // view showing "logged by Claude" should not have to query a second
+  // collection for every row it renders.
+  schema.add({
+    createdBy: { type: String },
+    updatedBy: { type: String }
+  });
+
+  schema.pre('save', async function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const label = context.actor.label;
+    if (this.isNew) this.set('createdBy', label);
+    this.set('updatedBy', label);
+
+    // `isNew` is false by the time the post hook runs, so it is carried here.
+    // Recording every save as a create was the original mistake: any code that
+    // loads a document, changes a field and saves it — revoking an API key,
+    // counting a reveal — would appear in the trail as the thing being created
+    // again, with a fresh snapshot and no sign of what actually changed.
+    this.$locals.auditWasNew = this.isNew;
+    if (this.isNew) return;
+
+    this.$locals.auditPaths = this.modifiedPaths().filter((path) => !NOISE.has(path));
+    this.$locals.auditBefore = await (this.constructor as Model<unknown>)
+      .findById(this._id)
+      .lean();
+  });
+
+  schema.post('save', function (doc) {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const common = {
+      at: new Date(),
+      service: context.service,
+      resource,
+      resourceId: String(doc._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant
+    };
+
+    if (doc.$locals.auditWasNew) {
+      record({ ...common, action: 'create', snapshot: visible(doc.toObject()) });
+      return;
+    }
+
+    const before = (doc.$locals.auditBefore ?? {}) as Record<string, unknown>;
+    const paths = (doc.$locals.auditPaths ?? []) as string[];
+    const changes = changesBetween(before, doc.toObject(), paths);
+    if (!changes) return;
+
+    record({ ...common, action: 'update', changes });
+  });
+
+  // The document as it stood, fetched before the write so there is something to
+  // compare against. Held on the query, which is per-operation, so concurrent
+  // updates cannot read each other's.
+  schema.pre('findOneAndUpdate', async function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    (this as unknown as Record<symbol, unknown>)[BEFORE] = await this.model
+      .findOne(this.getQuery())
+      .lean();
+
+    // This one genuinely belongs in the update: it is a field on the document.
+    this.set('updatedBy', context.actor.label);
+  });
+
+  schema.post('findOneAndUpdate', function (doc) {
+    const context = currentAuditContext();
+    if (!context || !doc) return;
+
+    const before = ((this as unknown as Record<symbol, unknown>)[BEFORE] ?? {}) as Record<
+      string,
+      unknown
+    >;
+    const after = doc.toObject ? doc.toObject() : (doc as Record<string, unknown>);
+    // Only the fields the update touched: comparing everything would report a
+    // change on any field a concurrent write happened to move.
+    const update = (this.getUpdate() ?? {}) as Record<string, unknown>;
+    const touched = Object.keys((update.$set as object) ?? update);
+    const changes = changesBetween(before, after, touched);
+    if (!changes) return;
+
+    record({
+      at: new Date(),
+      service: context.service,
+      action: 'update',
+      resource,
+      resourceId: String((doc as { _id: unknown })._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant,
+      changes
+    });
+  });
+
+  schema.post('findOneAndDelete', function (doc) {
+    const context = currentAuditContext();
+    if (!context || !doc) return;
+
+    record({
+      at: new Date(),
+      service: context.service,
+      action: 'delete',
+      resource,
+      resourceId: String((doc as { _id: unknown })._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant,
+      // The whole document, because a delete is the one action whose record is
+      // the only remaining copy of what was there.
+      snapshot: visible(doc.toObject ? doc.toObject() : (doc as Record<string, unknown>))
+    });
+  });
+
+  // Bulk deletes are read back first, one row each. A day's save deletes the
+  // entries it is replacing, and "the day editor removed three things" is
+  // exactly the change someone would come here to find.
+  schema.pre('deleteMany', async function () {
+    if (!currentAuditContext()) return;
+    (this as unknown as Record<symbol, unknown>)[BEFORE] = await this.model
+      .find(this.getQuery())
+      .lean();
+  });
+
+  schema.post('deleteMany', function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const doomed = ((this as unknown as Record<symbol, unknown>)[BEFORE] ?? []) as Record<
+      string,
+      unknown
+    >[];
+    for (const doc of doomed) {
+      record({
+        at: new Date(),
+        service: context.service,
+        action: 'delete',
+        resource,
+        resourceId: String(doc._id),
+        actorKind: context.actor.kind,
+        userId: context.actor.userId,
+        actorLabel: context.actor.label,
+        actorCredentialId: context.actor.credentialId,
+        tenant: context.actor.tenant,
+        snapshot: visible(doc)
+      });
+    }
+  });
+}