@tumbaland/backend-core 1.25.0 → 1.27.0

package/src/entitlements/middleware.test.ts

@@ -35,7 +35,7 @@ function mockReq(overrides: Partial<Request> = {}): Request {
 }
 
 beforeEach(() => {
-  process.env = { ...ORIGINAL_ENV, ENTITLEMENTS_MODE: 'enforce' };
+  process.env = { ...ORIGINAL_ENV };
   jest.clearAllMocks();
   mockGetEntitlements.mockResolvedValue(entitlements());
 });
@@ -89,24 +89,10 @@ describe('requireEntitlement', () => {
     );
   });
 
-  it('logs instead of blocking in observe mode', async () => {
-    process.env.ENTITLEMENTS_MODE = 'observe';
-    const next = jest.fn();
-
-    await requireEntitlement('cleanExport')(mockReq(), {} as Response, next);
+  it('logs the denial alongside throwing it', async () => {
+    await requireEntitlement('cleanExport')(mockReq(), {} as Response, jest.fn());
 
-    expect(next).toHaveBeenCalledWith();
-    expect(logger.info).toHaveBeenCalledWith('entitlement.would_block', expect.objectContaining({ feature: 'cleanExport' }));
-  });
-
-  it('does not even look entitlements up when the mode is off', async () => {
-    process.env.ENTITLEMENTS_MODE = 'off';
-    const next = jest.fn();
-
-    await requireEntitlement('cleanExport')(mockReq(), {} as Response, next);
-
-    expect(next).toHaveBeenCalledWith();
-    expect(mockGetEntitlements).not.toHaveBeenCalled();
+    expect(logger.info).toHaveBeenCalledWith('entitlement.blocked', expect.objectContaining({ feature: 'cleanExport' }));
   });
 });
 
@@ -143,15 +129,6 @@ describe('requireQuota', () => {
     expect(next).toHaveBeenCalledWith(denial);
   });
 
-  it('is inert when the mode is off', async () => {
-    process.env.ENTITLEMENTS_MODE = 'off';
-    const next = jest.fn();
-
-    await requireQuota('storageBytes')(mockReq(), {} as Response, next);
-
-    expect(mockCheckQuota).not.toHaveBeenCalled();
-    expect(next).toHaveBeenCalledWith();
-  });
 });
 
 describe('retentionFloor', () => {
@@ -171,9 +148,49 @@ describe('retentionFloor', () => {
     expect(await retentionFloor('user@example.com', now)).toBeNull();
   });
 
-  it('never truncates history outside enforce mode', async () => {
-    process.env.ENTITLEMENTS_MODE = 'observe';
+  it('still applies the floor when the limits are a stale fallback', async () => {
+    mockGetEntitlements.mockResolvedValue(entitlements({ stale: true }));
 
-    expect(await retentionFloor('user@example.com', now)).toBeNull();
+    // Fail closed, same as every other gate: an outage clips history to the
+    // free tier for a minute rather than handing the full archive out.
+    expect(await retentionFloor('user@example.com', now)).toEqual(new Date('2026-05-11T00:00:00.000Z'));
+  });
+});
+
+describe('retentionFloor across several subjects', () => {
+  const now = new Date('2026-08-09T00:00:00.000Z');
+
+  it('takes the most generous floor, so a paid group lifts it for every member', () => {
+    mockGetEntitlements.mockImplementation(async (email: string) =>
+      email === 'owner@example.com'
+        ? entitlements({ planCode: 'family', limits: normalizeLimits({ retentionDays: UNLIMITED }, 'family') })
+        : entitlements()
+    );
+
+    // A free member reading a Family group's data must not be clipped to 90
+    // days of a household's history the household paid to keep.
+    return expect(retentionFloor(['member@example.com', 'owner@example.com'], now)).resolves.toBeNull();
+  });
+
+  it('picks the floor that reaches furthest back when none is unlimited', async () => {
+    mockGetEntitlements.mockImplementation(async (email: string) =>
+      entitlements({ limits: normalizeLimits({ retentionDays: email === 'owner@example.com' ? 365 : 90 }, 'free') })
+    );
+
+    expect(await retentionFloor(['member@example.com', 'owner@example.com'], now)).toEqual(
+      new Date('2025-08-09T00:00:00.000Z')
+    );
+  });
+
+  it('still accepts a single email', async () => {
+    mockGetEntitlements.mockResolvedValue(entitlements());
+
+    expect(await retentionFloor('user@example.com', now)).toEqual(new Date('2026-05-11T00:00:00.000Z'));
+  });
+
+  it('falls back to the free floor when no subject can be identified', async () => {
+    // An unidentifiable caller must never resolve to unlimited history.
+    expect(await retentionFloor([], now)).toEqual(new Date('2026-05-11T00:00:00.000Z'));
+    expect(mockGetEntitlements).not.toHaveBeenCalled();
   });
 });

package/src/entitlements/middleware.ts

@@ -1,10 +1,9 @@
 import { Request, RequestHandler } from 'express';
 import logger from '../logging/logger';
 import { PaymentRequiredError, UnauthorizedError } from '../errors/HttpError';
-import { FeatureKey, MeterKey, UNLIMITED } from './definitions';
+import { FALLBACK_PLAN, FeatureKey, MeterKey, UNLIMITED, resolveLimits } from './definitions';
 import { getEntitlements } from './client';
 import { checkQuota } from './usage';
-import { getEntitlementsMode } from './mode';
 import { Entitlements } from './types';
 
 declare global {
@@ -51,23 +50,17 @@ export async function loadEntitlements(req: Request, resolve: SubjectResolver =
 export function requireEntitlement(feature: FeatureKey, options: EntitlementOptions = {}): RequestHandler {
   return async (req, _res, next) => {
     try {
-      const mode = getEntitlementsMode();
-      if (mode === 'off') return next();
-
       const entitlements = await loadEntitlements(req, options.subject);
       if (entitlements.limits.features[feature]) return next();
 
-      if (mode === 'observe') {
-        logger.info('entitlement.would_block', {
-          event: 'entitlement.would_block',
-          feature,
-          subjectId: entitlements.email,
-          planCode: entitlements.planCode,
-          route: req.originalUrl,
-          stale: entitlements.stale === true
-        });
-        return next();
-      }
+      logger.info('entitlement.blocked', {
+        event: 'entitlement.blocked',
+        feature,
+        subjectId: entitlements.email,
+        planCode: entitlements.planCode,
+        route: req.originalUrl,
+        stale: entitlements.stale === true
+      });
 
       throw new PaymentRequiredError(`Your plan does not include ${feature}`, {
         code: 'FEATURE_NOT_IN_PLAN',
@@ -98,8 +91,6 @@ export interface QuotaOptions extends EntitlementOptions {
 export function requireQuota(meter: MeterKey, options: QuotaOptions = {}): RequestHandler {
   return async (req, _res, next) => {
     try {
-      if (getEntitlementsMode() === 'off') return next();
-
       const entitlements = await loadEntitlements(req, options.subject);
       const amount = typeof options.amount === 'function' ? options.amount(req) : (options.amount ?? 1);
 
@@ -118,19 +109,42 @@ export function requireQuota(meter: MeterKey, options: QuotaOptions = {}): Reque
 }
 
 /**
- * The oldest timestamp a user may read, or `null` for unlimited history.
+ * The oldest timestamp a caller may read, or `null` for unlimited history.
  *
- * Meant to be dropped straight into an aggregation `$match` — this is what
+ * Meant to be dropped straight into a `$match` or a `find` filter — this is what
  * gates the portfolio planner, since the planner runs entirely client-side and
  * any `<PaywallGate>` around it is bypassable from devtools. Truncating the
  * price history it depends on is not.
+ *
+ * **Several subjects resolve to the most generous floor.** Retention is not a
+ * counter, so unlike a quota it has no single payer: data held in a group is
+ * read by every member, and if it resolved to the *caller's* plan alone, a free
+ * member of a paid Family group would see ninety days of a household's history
+ * that the household paid to keep. Pass the caller together with the owners of
+ * the groups being read, and the group's plan lifts the floor for everyone in
+ * it — which is what "shared across a household" was sold as.
+ *
+ * An empty list resolves to the free plan's floor rather than to `null`: no
+ * identifiable subject must never mean unlimited history.
  */
-export async function retentionFloor(email: string, now: Date = new Date()): Promise<Date | null> {
-  if (getEntitlementsMode() !== 'enforce') return null;
+export async function retentionFloor(
+  subject: string | readonly string[],
+  now: Date = new Date()
+): Promise<Date | null> {
+  const emails = (Array.isArray(subject) ? subject : [subject as string]).filter(Boolean);
+  const floorFor = (days: number): Date | null =>
+    days === UNLIMITED || days < 0 ? null : new Date(now.getTime() - days * 24 * 60 * 60 * 1000);
+
+  if (emails.length === 0) {
+    return floorFor(resolveLimits(FALLBACK_PLAN).retentionDays);
+  }
 
-  const { limits } = await getEntitlements(email);
-  const days = limits.retentionDays;
-  if (days === UNLIMITED || days < 0) return null;
+  const floors = await Promise.all(
+    emails.map(async email => floorFor((await getEntitlements(email)).limits.retentionDays))
+  );
 
-  return new Date(now.getTime() - days * 24 * 60 * 60 * 1000);
+  // `null` is unlimited, which beats every date; otherwise the earliest floor
+  // is the one that reaches furthest back.
+  if (floors.some(floor => floor === null)) return null;
+  return (floors as Date[]).reduce((a, b) => (a < b ? a : b));
 }

package/src/entitlements/reconcile.test.ts

@@ -0,0 +1,333 @@
+jest.mock('../logging/logger', () => ({
+  __esModule: true,
+  default: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn(), http: jest.fn() }
+}));
+
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import logger from '../logging/logger';
+import { UsageMeter } from './UsageMeter';
+import { getUsage, setUsage } from './usage';
+import { reconcileUsageMeters, runUsageReconciliation } from './reconcile';
+
+/**
+ * Against a real Mongo, not mocks: the whole point of this module is the shape
+ * of the aggregations and the joins between four services' collections, and a
+ * mocked driver would assert nothing about either.
+ */
+let mongod: MongoMemoryServer;
+
+const OWNER = 'owner@example.com';
+const MEMBER = 'member@example.com';
+const LONER = 'loner@example.com';
+
+const OWNER_ID = '68987cf71078a6d4d52ba430';
+const MEMBER_ID = '689880b91078a6d4d52ba43f';
+const LONER_ID = '68b4e9fc05b3c0001a2286c3';
+const GROUP_ID = '689a8219e15515e8f6539ad2';
+
+const db = () => mongoose.connection.db!;
+
+beforeAll(async () => {
+  mongod = await MongoMemoryServer.create();
+  await mongoose.connect(mongod.getUri());
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongod.stop();
+});
+
+/** The shape the live database actually has: everything owned through one group. */
+async function seed(
+  overrides: {
+    groups?: unknown[];
+    albums?: unknown[];
+    photos?: unknown[];
+    tickers?: unknown[];
+  } = {}
+) {
+  await db()
+    .collection('users')
+    .insertMany([
+      { _id: new mongoose.Types.ObjectId(OWNER_ID), email: 'Owner@Example.com' },
+      { _id: new mongoose.Types.ObjectId(MEMBER_ID), email: MEMBER },
+      { _id: new mongoose.Types.ObjectId(LONER_ID), email: LONER }
+    ] as never);
+
+  const groups = overrides.groups ?? [
+    {
+      _id: new mongoose.Types.ObjectId(GROUP_ID),
+      name: 'Household',
+      createdBy: OWNER,
+      members: [
+        { email: OWNER, role: 'owner' },
+        { email: MEMBER, role: 'admin' }
+      ],
+      deletedAt: null
+    }
+  ];
+  if (groups.length) await db().collection('groups').insertMany(groups as never);
+
+  const albums = overrides.albums ?? [
+    { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, groupId: GROUP_ID, totalSize: 300, name: 'Shared' },
+    { _id: new mongoose.Types.ObjectId(), userId: MEMBER_ID, totalSize: 50, name: 'Personal' }
+  ];
+  if (albums.length) await db().collection('albums').insertMany(albums as never);
+
+  const photos =
+    overrides.photos ??
+    (albums as { _id: mongoose.Types.ObjectId }[]).flatMap((album, i) => [
+      { albumId: album._id, size: i === 0 ? 200 : 50 },
+      ...(i === 0 ? [{ albumId: album._id, size: 100 }] : [])
+    ]);
+  if ((photos as unknown[]).length) await db().collection('photos').insertMany(photos as never);
+
+  const tickers = overrides.tickers ?? [
+    { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, groupId: GROUP_ID, ticker: 'VWCE.DEX', isActive: true },
+    { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, groupId: GROUP_ID, ticker: 'VUSA.DEX', isActive: true }
+  ];
+  if ((tickers as unknown[]).length) await db().collection('trackedtickers').insertMany(tickers as never);
+}
+
+beforeEach(async () => {
+  for (const name of ['users', 'groups', 'albums', 'photos', 'trackedtickers']) {
+    await db().collection(name).deleteMany({});
+  }
+  await UsageMeter.deleteMany({});
+  await UsageMeter.syncIndexes();
+  jest.clearAllMocks();
+});
+
+describe('reconcileUsageMeters', () => {
+  it('bills a group album to the owner and a personal album to its own user', async () => {
+    await seed();
+
+    const report = await reconcileUsageMeters({ apply: true });
+
+    // 200 + 100 in the shared album, all to the owner; the member keeps only
+    // their own 50, rather than the shared bytes landing on their free tier.
+    expect(report.expected[OWNER].storageBytes).toBe(300);
+    expect(report.expected[MEMBER].storageBytes).toBe(50);
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(await getUsage(MEMBER, 'storageBytes')).toBe(50);
+  });
+
+  it('joins userId to email through the users collection, matching on lowercase', async () => {
+    await seed();
+
+    await reconcileUsageMeters({ apply: true });
+
+    // The seeded user has a capitalised address; meters are keyed lowercase, so
+    // a missed normalization would open a second counter instead of filling one.
+    expect(await getUsage('owner@example.com', 'storageBytes')).toBe(300);
+    expect(await UsageMeter.countDocuments({ meter: 'storageBytes' })).toBe(2);
+  });
+
+  it('counts distinct active symbols, not rows', async () => {
+    await seed({
+      tickers: [
+        { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, groupId: GROUP_ID, ticker: 'VWCE.DEX', isActive: true },
+        // Same symbol held personally as well — one Alpha Vantage call covers both.
+        { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, ticker: 'vwce.dex', isActive: true },
+        { _id: new mongoose.Types.ObjectId(), userId: OWNER_ID, ticker: 'DEAD.DEX', isActive: false }
+      ]
+    });
+
+    await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'trackedTickers')).toBe(1);
+  });
+
+  it('counts seats as distinct people in the groups a subject owns', async () => {
+    await seed();
+
+    await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'seats')).toBe(2);
+    // A member is not charged for a group somebody else owns.
+    expect(await getUsage(MEMBER, 'seats')).toBe(0);
+  });
+
+  it('keeps billing a soft-deleted group storage and tickers, but not seats', async () => {
+    await seed({
+      groups: [
+        {
+          _id: new mongoose.Types.ObjectId(GROUP_ID),
+          name: 'Household',
+          createdBy: OWNER,
+          members: [
+            { email: OWNER, role: 'owner' },
+            { email: MEMBER, role: 'admin' }
+          ],
+          deletedAt: new Date('2026-01-01')
+        }
+      ]
+    });
+
+    await reconcileUsageMeters({ apply: true });
+
+    // The photos are still in MinIO and the tickers are still fetched daily;
+    // the seats grant nobody access to anything.
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(await getUsage(OWNER, 'trackedTickers')).toBe(2);
+    expect(await getUsage(OWNER, 'seats')).toBe(0);
+  });
+
+  it('prefers the owner role over a stale createdBy', async () => {
+    await seed({
+      groups: [
+        {
+          _id: new mongoose.Types.ObjectId(GROUP_ID),
+          name: 'Household',
+          createdBy: LONER,
+          members: [
+            { email: OWNER, role: 'owner' },
+            { email: MEMBER, role: 'admin' }
+          ],
+          deletedAt: null
+        }
+      ]
+    });
+
+    await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(await getUsage(LONER, 'storageBytes')).toBe(0);
+  });
+
+  it('writes nothing without apply', async () => {
+    await seed();
+
+    const report = await reconcileUsageMeters();
+
+    expect(report.changes.length).toBeGreaterThan(0);
+    expect(report.written).toBe(0);
+    expect(await UsageMeter.countDocuments({})).toBe(0);
+  });
+
+  it('is idempotent — a second run in a row changes nothing', async () => {
+    await seed();
+
+    await reconcileUsageMeters({ apply: true });
+    const second = await reconcileUsageMeters({ apply: true });
+
+    expect(second.changes).toEqual([]);
+    expect(second.written).toBe(0);
+  });
+
+  it('repairs a leaked increment, which is the whole reason it is scheduled', async () => {
+    await seed();
+    // A crash between writing the photo and updating the meter, or a release
+    // that never ran: the counter says more than the photos do.
+    await setUsage(OWNER, 'storageBytes', 999999);
+
+    const report = await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(report.changes).toContainEqual({ subjectId: OWNER, meter: 'storageBytes', from: 999999, to: 300 });
+  });
+
+  it('corrects an existing counter down to zero but never creates a zero row', async () => {
+    await seed({ albums: [], photos: [], tickers: [] });
+    await setUsage(OWNER, 'storageBytes', 500);
+
+    await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(0);
+    // The other users had no counter and no usage — nothing was invented.
+    expect(await UsageMeter.countDocuments({ subjectId: LONER })).toBe(0);
+  });
+
+  it('reports meters for unknown subjects instead of deleting them', async () => {
+    await seed();
+    await setUsage('ghost@example.com', 'storageBytes', 42);
+
+    const report = await reconcileUsageMeters({ apply: true });
+
+    expect(report.orphanedMeters).toEqual([{ subjectId: 'ghost@example.com', meter: 'storageBytes', count: 42 }]);
+    expect(await getUsage('ghost@example.com', 'storageBytes')).toBe(42);
+  });
+
+  it('scopes to one subject when asked, leaving everyone else untouched', async () => {
+    await seed();
+    await setUsage(MEMBER, 'storageBytes', 777);
+
+    await reconcileUsageMeters({ apply: true, subject: 'Owner@Example.com' });
+
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(await getUsage(MEMBER, 'storageBytes')).toBe(777);
+  });
+
+  it('never touches aiJobs, which has no source data to recompute from', async () => {
+    await seed();
+    await setUsage(OWNER, 'aiJobs', 17);
+
+    await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'aiJobs')).toBe(17);
+  });
+
+  it('bills an album in a vanished group to its uploader rather than losing the bytes', async () => {
+    await seed({ groups: [] });
+
+    const report = await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(report.warnings.some(w => w.includes('references missing group'))).toBe(true);
+  });
+
+  it('warns when a denormalized album counter disagrees, without rewriting it', async () => {
+    const albumId = new mongoose.Types.ObjectId();
+    await seed({
+      albums: [{ _id: albumId, userId: MEMBER_ID, totalSize: 9999, name: 'Drifted' }],
+      photos: [{ albumId, size: 50 }],
+      tickers: []
+    });
+
+    const report = await reconcileUsageMeters({ apply: true });
+
+    expect(await getUsage(MEMBER, 'storageBytes')).toBe(50);
+    expect(report.warnings.some(w => w.includes('but photos sum to 50'))).toBe(true);
+    const album = await db().collection('albums').findOne({ _id: albumId });
+    expect(album?.totalSize).toBe(9999);
+  });
+});
+
+describe('runUsageReconciliation', () => {
+  it('applies, and logs a correction at warn so a persistent leak is visible', async () => {
+    await seed();
+    // Settle first, so the only change the scheduled run sees is the drift.
+    await reconcileUsageMeters({ apply: true });
+    await setUsage(OWNER, 'storageBytes', 999999);
+    jest.clearAllMocks();
+
+    const report = await runUsageReconciliation();
+
+    expect(report?.written).toBe(1);
+    expect(await getUsage(OWNER, 'storageBytes')).toBe(300);
+    expect(logger.warn).toHaveBeenCalledWith('entitlement.reconciled', expect.objectContaining({ corrected: 1 }));
+  });
+
+  it('logs at info when nothing drifted', async () => {
+    await seed();
+    await reconcileUsageMeters({ apply: true });
+    jest.clearAllMocks();
+
+    await runUsageReconciliation();
+
+    expect(logger.info).toHaveBeenCalledWith('entitlement.reconciled', expect.objectContaining({ corrected: 0 }));
+  });
+
+  it('swallows a failure rather than taking the process down from a cron callback', async () => {
+    const spy = jest.spyOn(UsageMeter, 'find').mockImplementationOnce(() => {
+      throw new Error('mongo is gone');
+    });
+    await seed();
+
+    await expect(runUsageReconciliation()).resolves.toBeNull();
+    expect(logger.error).toHaveBeenCalledWith('Usage meter reconciliation failed', expect.anything());
+
+    spy.mockRestore();
+  });
+});