@tumbaland/backend-core 1.38.0 → 1.40.0

package/src/audit/plugin.ts

@@ -0,0 +1,254 @@
+import type { Model, Schema } from 'mongoose';
+import logger from '../logging/logger';
+import { AuditEvent } from './AuditEvent';
+import { currentAuditContext } from './context';
+
+/** Bookkeeping fields nobody wants to read in a change list. */
+const NOISE = new Set(['updatedAt', 'createdAt', '__v', 'createdBy', 'updatedBy']);
+
+/**
+ * Where a hook stashes what it read before the write.
+ *
+ * A symbol on the query object, not `query.set()` — that method writes into the
+ * *update payload*, so stashing there would persist the before-image into the
+ * document it was taken of. Per-query, so concurrent writes cannot read each
+ * other's.
+ */
+const BEFORE = Symbol('auditBefore');
+
+/** Comparable form, so a Date and an ObjectId do not read as changed every time. */
+const plain = (value: unknown): unknown => {
+  if (value === null || value === undefined) return value;
+  if (value instanceof Date) return value.toISOString();
+  if (typeof value === 'object' && 'toHexString' in (value as object)) return String(value);
+  return value;
+};
+
+const changesBetween = (
+  before: Record<string, unknown>,
+  after: Record<string, unknown>,
+  fields: string[]
+): Record<string, { from: unknown; to: unknown }> | undefined => {
+  const changes: Record<string, { from: unknown; to: unknown }> = {};
+
+  for (const field of fields) {
+    if (NOISE.has(field)) continue;
+    const from = plain(before?.[field]);
+    const to = plain(after?.[field]);
+    if (JSON.stringify(from) !== JSON.stringify(to)) changes[field] = { from, to };
+  }
+
+  return Object.keys(changes).length > 0 ? changes : undefined;
+};
+
+/**
+ * Never let recording a thing fail the thing.
+ *
+ * An audit trail is worth having and is not worth refusing a write over: a
+ * failure here means someone loses a row of history, and throwing would mean
+ * they lose the entry they were logging. So it is fire-and-forget with the
+ * failure written to the ordinary log, where an operator sees it.
+ */
+const record = (event: Record<string, unknown>): void => {
+  void AuditEvent.create(event).catch((error) => {
+    logger.warn('Could not record an audit event', {
+      error: (error as Error)?.message,
+      action: event.action,
+      resource: event.resource
+    });
+  });
+};
+
+export interface AuditPluginOptions {
+  /** what these documents are called in a trail — 'activity', 'photo' */
+  resource: string;
+  /** fields never worth recording the content of */
+  redact?: string[];
+}
+
+/**
+ * Attribute and record every write to a collection.
+ *
+ * Applied to a model rather than called from its controllers, for two reasons.
+ * The data layer is the only place that knows which fields actually changed —
+ * a controller sees the patch that was requested, not the difference it made,
+ * and those differ whenever a field was already the value being set. And a
+ * plugin cannot be forgotten: a new endpoint that writes through the model is
+ * audited without anyone remembering to add a line.
+ *
+ * Writes made outside a request — a migration, a backfill script — find no
+ * actor and are recorded as nothing, which is honest. A row claiming a script
+ * was a person would be worse than no row.
+ */
+export function auditPlugin(schema: Schema, options: AuditPluginOptions): void {
+  const { resource, redact = [] } = options;
+  const hide = new Set(redact);
+
+  const visible = (doc: Record<string, unknown>): Record<string, unknown> =>
+    Object.fromEntries(
+      Object.entries(doc)
+        .filter(([key]) => !NOISE.has(key) && !hide.has(key))
+        .map(([key, value]) => [key, plain(value)])
+    );
+
+  // Who last touched this, on the record itself. Denormalized on purpose: a day
+  // view showing "logged by Claude" should not have to query a second
+  // collection for every row it renders.
+  schema.add({
+    createdBy: { type: String },
+    updatedBy: { type: String }
+  });
+
+  schema.pre('save', async function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const label = context.actor.label;
+    if (this.isNew) this.set('createdBy', label);
+    this.set('updatedBy', label);
+
+    // `isNew` is false by the time the post hook runs, so it is carried here.
+    // Recording every save as a create was the original mistake: any code that
+    // loads a document, changes a field and saves it — revoking an API key,
+    // counting a reveal — would appear in the trail as the thing being created
+    // again, with a fresh snapshot and no sign of what actually changed.
+    this.$locals.auditWasNew = this.isNew;
+    if (this.isNew) return;
+
+    this.$locals.auditPaths = this.modifiedPaths().filter((path) => !NOISE.has(path));
+    this.$locals.auditBefore = await (this.constructor as Model<unknown>)
+      .findById(this._id)
+      .lean();
+  });
+
+  schema.post('save', function (doc) {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const common = {
+      at: new Date(),
+      service: context.service,
+      resource,
+      resourceId: String(doc._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant
+    };
+
+    if (doc.$locals.auditWasNew) {
+      record({ ...common, action: 'create', snapshot: visible(doc.toObject()) });
+      return;
+    }
+
+    const before = (doc.$locals.auditBefore ?? {}) as Record<string, unknown>;
+    const paths = (doc.$locals.auditPaths ?? []) as string[];
+    const changes = changesBetween(before, doc.toObject(), paths);
+    if (!changes) return;
+
+    record({ ...common, action: 'update', changes });
+  });
+
+  // The document as it stood, fetched before the write so there is something to
+  // compare against. Held on the query, which is per-operation, so concurrent
+  // updates cannot read each other's.
+  schema.pre('findOneAndUpdate', async function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    (this as unknown as Record<symbol, unknown>)[BEFORE] = await this.model
+      .findOne(this.getQuery())
+      .lean();
+
+    // This one genuinely belongs in the update: it is a field on the document.
+    this.set('updatedBy', context.actor.label);
+  });
+
+  schema.post('findOneAndUpdate', function (doc) {
+    const context = currentAuditContext();
+    if (!context || !doc) return;
+
+    const before = ((this as unknown as Record<symbol, unknown>)[BEFORE] ?? {}) as Record<
+      string,
+      unknown
+    >;
+    const after = doc.toObject ? doc.toObject() : (doc as Record<string, unknown>);
+    // Only the fields the update touched: comparing everything would report a
+    // change on any field a concurrent write happened to move.
+    const update = (this.getUpdate() ?? {}) as Record<string, unknown>;
+    const touched = Object.keys((update.$set as object) ?? update);
+    const changes = changesBetween(before, after, touched);
+    if (!changes) return;
+
+    record({
+      at: new Date(),
+      service: context.service,
+      action: 'update',
+      resource,
+      resourceId: String((doc as { _id: unknown })._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant,
+      changes
+    });
+  });
+
+  schema.post('findOneAndDelete', function (doc) {
+    const context = currentAuditContext();
+    if (!context || !doc) return;
+
+    record({
+      at: new Date(),
+      service: context.service,
+      action: 'delete',
+      resource,
+      resourceId: String((doc as { _id: unknown })._id),
+      actorKind: context.actor.kind,
+      userId: context.actor.userId,
+      actorLabel: context.actor.label,
+      actorCredentialId: context.actor.credentialId,
+      tenant: context.actor.tenant,
+      // The whole document, because a delete is the one action whose record is
+      // the only remaining copy of what was there.
+      snapshot: visible(doc.toObject ? doc.toObject() : (doc as Record<string, unknown>))
+    });
+  });
+
+  // Bulk deletes are read back first, one row each. A day's save deletes the
+  // entries it is replacing, and "the day editor removed three things" is
+  // exactly the change someone would come here to find.
+  schema.pre('deleteMany', async function () {
+    if (!currentAuditContext()) return;
+    (this as unknown as Record<symbol, unknown>)[BEFORE] = await this.model
+      .find(this.getQuery())
+      .lean();
+  });
+
+  schema.post('deleteMany', function () {
+    const context = currentAuditContext();
+    if (!context) return;
+
+    const doomed = ((this as unknown as Record<symbol, unknown>)[BEFORE] ?? []) as Record<
+      string,
+      unknown
+    >[];
+    for (const doc of doomed) {
+      record({
+        at: new Date(),
+        service: context.service,
+        action: 'delete',
+        resource,
+        resourceId: String(doc._id),
+        actorKind: context.actor.kind,
+        userId: context.actor.userId,
+        actorLabel: context.actor.label,
+        actorCredentialId: context.actor.credentialId,
+        tenant: context.actor.tenant,
+        snapshot: visible(doc)
+      });
+    }
+  });
+}

package/src/audit/reads.test.ts

@@ -0,0 +1,164 @@
+import type { Request, Response } from 'express';
+
+jest.mock('./AuditEvent', () => ({ AuditEvent: { create: jest.fn().mockResolvedValue({}) } }));
+jest.mock('../logging/logger', () => ({
+  __esModule: true,
+  default: { warn: jest.fn(), info: jest.fn(), error: jest.fn(), debug: jest.fn() }
+}));
+
+import { AuditEvent } from './AuditEvent';
+import { auditReads, countRead, resourceOf } from './reads';
+import { PERSONAL_TENANT } from '../apiKeys/types';
+
+const created = AuditEvent.create as jest.Mock;
+
+/** A response that runs its finish listeners when told to. */
+function mockRes(statusCode = 200) {
+  const listeners: (() => void)[] = [];
+  const res = {
+    statusCode,
+    locals: {} as Record<string, unknown>,
+    on: (event: string, fn: () => void) => {
+      if (event === 'finish') listeners.push(fn);
+      return res;
+    },
+    finish: () => listeners.forEach((fn) => fn())
+  };
+  return res as unknown as Response & { finish: () => void };
+}
+
+const req = (over: Partial<Request> = {}): Request =>
+  ({ method: 'GET', baseUrl: '/api/activities', path: '/', query: {}, ...over }) as unknown as Request;
+
+const assistant = {
+  keyId: 'client-1',
+  scopes: [],
+  tenants: { allowed: ['g1'], default: 'g1' },
+  actingAs: 'g1',
+  kind: 'oauth' as const,
+  label: 'Claude',
+  credentialId: 'client-1'
+};
+
+const user = { id: 'u1', email: 'tom@example.com', name: 'Tom' };
+
+beforeEach(() => jest.clearAllMocks());
+
+describe('auditReads', () => {
+  it('records what an assistant read', async () => {
+    const res = mockRes();
+    const request = req({ user, apiKey: assistant });
+
+    auditReads('relationship-service')(request, res, jest.fn());
+    countRead(res, 47);
+    res.finish();
+
+    expect(created).toHaveBeenCalledWith(
+      expect.objectContaining({
+        action: 'read',
+        resource: 'activities',
+        actorKind: 'assistant',
+        actorLabel: 'Claude',
+        tenant: 'g1',
+        // The size of what left the server is most of what makes a read worth
+        // keeping: "read the journal" and "read 47 entries" are different events.
+        count: 47
+      })
+    );
+  });
+
+  it('records nothing for a person browsing their own journal', async () => {
+    // One dashboard load hits half a dozen endpoints. Logging those would bury
+    // the interesting rows and put a database write behind every page view.
+    const res = mockRes();
+
+    auditReads('relationship-service')(req({ user }), res, jest.fn());
+    res.finish();
+
+    expect(created).not.toHaveBeenCalled();
+  });
+
+  it('records nothing for a write, which the model plugin already covers', () => {
+    const res = mockRes();
+    auditReads('relationship-service')(req({ user, apiKey: assistant, method: 'POST' }), res, jest.fn());
+    res.finish();
+
+    expect(created).not.toHaveBeenCalled();
+  });
+
+  it('records nothing for a refused request', () => {
+    // A 403 is not a read: nothing left the server.
+    const res = mockRes(403);
+    auditReads('relationship-service')(req({ user, apiKey: assistant }), res, jest.fn());
+    res.finish();
+
+    expect(created).not.toHaveBeenCalled();
+  });
+
+  it('always continues the request, whether or not it recorded anything', () => {
+    const next = jest.fn();
+    auditReads('relationship-service')(req(), mockRes(), next);
+    expect(next).toHaveBeenCalled();
+  });
+
+  it('does not fail the request when the trail cannot be written', async () => {
+    created.mockRejectedValueOnce(new Error('audit db down'));
+    const res = mockRes();
+
+    auditReads('relationship-service')(req({ user, apiKey: assistant }), res, jest.fn());
+    expect(() => res.finish()).not.toThrow();
+  });
+
+  it('survives a handler that never counted anything', () => {
+    const res = mockRes();
+    auditReads('relationship-service')(req({ user, apiKey: assistant }), res, jest.fn());
+    res.finish();
+
+    expect(created).toHaveBeenCalledWith(expect.objectContaining({ count: undefined }));
+  });
+});
+
+describe('resourceOf', () => {
+  it.each([
+    ['/api/activities/', 'activities'],
+    ['/api/activities/68d6d7edfb26c947ecaa057a', 'activities'],
+    ['/api/activity-types/', 'activity_types'],
+    ['/api/statistics/trends', 'statistics']
+  ])('reads %s as %s', (path, expected) => {
+    // Deliberately crude: a trail naming the resource is readable, and one
+    // naming the full path with ids in it is a request log in disguise.
+    expect(resourceOf(path)).toBe(expected);
+  });
+
+  it('names something rather than nothing for an unrecognisable path', () => {
+    expect(resourceOf('/')).toBe('unknown');
+  });
+});
+
+describe('countRead', () => {
+  it('cannot break the request it is instrumenting', () => {
+    // Express always provides `locals`; a handler called directly does not.
+    expect(() => countRead({} as Response, 3)).not.toThrow();
+    expect(() => countRead(undefined as unknown as Response, 3)).not.toThrow();
+  });
+
+  it('leaves the count where the middleware will find it', () => {
+    const res = { locals: {} } as Response;
+    countRead(res, 12);
+    expect(res.locals.auditCount).toBe(12);
+  });
+});
+
+describe('PERSONAL_TENANT', () => {
+  it('is what a personal-tenant read is recorded against', () => {
+    const res = mockRes();
+    auditReads('relationship-service')(
+      req({ user, apiKey: { ...assistant, actingAs: PERSONAL_TENANT } }),
+      res,
+      jest.fn()
+    );
+    res.finish();
+
+    expect(created).toHaveBeenCalledWith(expect.objectContaining({ tenant: PERSONAL_TENANT }));
+  });
+});

package/src/audit/reads.ts

@@ -0,0 +1,88 @@
+import type { NextFunction, Request, RequestHandler, Response } from 'express';
+import { AuditEvent } from './AuditEvent';
+import { actorOf, type Actor } from './actor';
+import logger from '../logging/logger';
+
+/**
+ * Record what software read, and only what software read.
+ *
+ * A person browsing their own journal generates nothing here. That is not a
+ * shortcut: one dashboard load hits half a dozen endpoints, so logging it would
+ * bury the interesting rows under thousands of uninteresting ones and put a
+ * database write behind every page view.
+ *
+ * What an assistant read is the part worth keeping, because it is the part that
+ * left the server. A created entry leaves a record behind either way; a read
+ * leaves nothing at all, and it is exactly what the privacy page warns about —
+ * ask an assistant what you did last month and those entries go to whoever runs
+ * it. This is the only place that becomes visible.
+ */
+/**
+ * Arm the read trail for one request, if it is one worth recording.
+ *
+ * Nothing is written here — the listener fires when the response finishes, so a
+ * refused request is not recorded as a read and the count reflects what was
+ * actually sent.
+ */
+export function recordRead(req: Request, res: Response, actor: Actor, service: string): void {
+  if (req.method !== 'GET' || actor.kind === 'person') return;
+
+  res.on('finish', () => {
+    if (res.statusCode >= 400) return;
+
+    void AuditEvent.create({
+      at: new Date(),
+      service,
+      action: 'read',
+      resource: resourceOf(req.baseUrl + req.path),
+      actorKind: actor.kind,
+      userId: actor.userId,
+      actorLabel: actor.label,
+      actorCredentialId: actor.credentialId,
+      tenant: actor.tenant,
+      count: res.locals.auditCount as number | undefined
+    }).catch((error) => {
+      logger.warn('Could not record a read', { error: (error as Error)?.message });
+    });
+  });
+}
+
+/**
+ * The same thing as standalone middleware, for anything that authenticates its
+ * own way rather than through `authenticateAgent`.
+ */
+export const auditReads =
+  (service: string): RequestHandler =>
+  (req: Request, res: Response, next: NextFunction): void => {
+    const actor = actorOf(req);
+    if (actor) recordRead(req, res, actor, service);
+    next();
+  };
+
+/**
+ * What a path was asking for, in one word.
+ *
+ * The first path segment, which is the collection on every route here —
+ * `/api/activities/68d6…` is about activities. Deliberately crude: a trail
+ * naming the resource is readable, and one naming the full path with ids in it
+ * is a request log wearing an audit log's clothes.
+ */
+export function resourceOf(path: string): string {
+  const segments = path.split('/').filter((segment) => segment && segment !== 'api');
+  return segments[0]?.replace(/-/g, '_') ?? 'unknown';
+}
+
+/**
+ * How many records a response carried.
+ *
+ * Set by a handler that knows; absent otherwise. The size of what left the
+ * server is most of what makes a read worth recording — "read the journal" and
+ * "read four hundred entries of the journal" are different events.
+ */
+export const countRead = (res: Response, count: number): void => {
+  // Guarded for the same reason recording is fire-and-forget: instrumentation
+  // must not be able to fail the thing it instruments. Express always provides
+  // `locals`, but a handler called directly does not have to.
+  if (!res?.locals) return;
+  res.locals.auditCount = count;
+};

package/src/audit/service.test.ts

@@ -0,0 +1,115 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+
+import { AuditEvent } from './AuditEvent';
+import { listAuditEvents } from './service';
+
+let mongo: MongoMemoryServer;
+
+const event = (over: Record<string, unknown> = {}) => ({
+  at: new Date('2026-09-05T10:00:00Z'),
+  service: 'relationship-service',
+  action: 'create' as const,
+  resource: 'activity',
+  actorKind: 'assistant' as const,
+  userId: 'u1',
+  actorLabel: 'Claude',
+  actorCredentialId: 'client-1',
+  tenant: 'g1',
+  ...over
+});
+
+beforeAll(async () => {
+  mongo = await MongoMemoryServer.create();
+  await mongoose.connect(mongo.getUri());
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongo.stop();
+});
+
+beforeEach(() => AuditEvent.deleteMany({}));
+
+describe('listAuditEvents', () => {
+  it('returns one account’s trail and never anyone else’s', async () => {
+    await AuditEvent.insertMany([event(), event({ userId: 'u2', actorLabel: 'Someone else' })]);
+
+    const { events, total } = await listAuditEvents({ userId: 'u1' });
+
+    // Scoped by the query rather than filtered afterwards: the trail carries old
+    // values of journal entries, so a wider read must not be one forgotten
+    // condition away.
+    expect(total).toBe(1);
+    expect(events[0].actorLabel).toBe('Claude');
+  });
+
+  it('returns newest first, which is the only order anyone reads a trail in', async () => {
+    await AuditEvent.insertMany([
+      event({ at: new Date('2026-09-01T10:00:00Z'), resource: 'older' }),
+      event({ at: new Date('2026-09-05T10:00:00Z'), resource: 'newer' })
+    ]);
+
+    const { events } = await listAuditEvents({ userId: 'u1' });
+    expect(events.map((e) => e.resource)).toEqual(['newer', 'older']);
+  });
+
+  it('narrows to one assistant, which is the question people actually ask', async () => {
+    await AuditEvent.insertMany([
+      event({ actorCredentialId: 'client-1', actorLabel: 'Claude' }),
+      event({ actorCredentialId: 'client-2', actorLabel: 'ChatGPT' })
+    ]);
+
+    const { events } = await listAuditEvents({ userId: 'u1', credentialId: 'client-1' });
+
+    expect(events).toHaveLength(1);
+    expect(events[0].actorLabel).toBe('Claude');
+  });
+
+  it('separates what software did from what a person did', async () => {
+    await AuditEvent.insertMany([
+      event({ actorKind: 'assistant' }),
+      event({ actorKind: 'person', actorLabel: 'Tom' })
+    ]);
+
+    const { events } = await listAuditEvents({ userId: 'u1', actorKind: 'person' });
+    expect(events.map((e) => e.actorLabel)).toEqual(['Tom']);
+  });
+
+  it('narrows by action and by date range', async () => {
+    await AuditEvent.insertMany([
+      event({ action: 'delete', at: new Date('2026-09-01T10:00:00Z') }),
+      event({ action: 'delete', at: new Date('2026-09-05T10:00:00Z') }),
+      event({ action: 'read', at: new Date('2026-09-05T11:00:00Z') })
+    ]);
+
+    const { events } = await listAuditEvents({
+      userId: 'u1',
+      action: 'delete',
+      from: new Date('2026-09-03T00:00:00Z')
+    });
+
+    expect(events).toHaveLength(1);
+    expect(events[0].at).toBe('2026-09-05T10:00:00.000Z');
+  });
+
+  it('hands back the change detail, which is the reason to look', async () => {
+    await AuditEvent.insertMany([event({ action: 'update', changes: { hours: { from: 3, to: 2 } } })]);
+
+    const { events } = await listAuditEvents({ userId: 'u1' });
+    expect(events[0].changes).toEqual({ hours: { from: 3, to: 2 } });
+  });
+
+  it('pages, and caps a caller asking for everything at once', async () => {
+    await AuditEvent.insertMany(Array.from({ length: 5 }, (_, i) => event({ resource: `r${i}` })));
+
+    const { events, total, totalPages } = await listAuditEvents({ userId: 'u1', limit: 2, page: 2 });
+
+    expect(events).toHaveLength(2);
+    expect(total).toBe(5);
+    expect(totalPages).toBe(3);
+
+    const capped = await listAuditEvents({ userId: 'u1', limit: 5000 });
+    expect(capped.events.length).toBeLessThanOrEqual(200);
+  });
+});