@tumbaland/backend-core 1.25.0 → 1.26.0

package/src/entitlements/reconcile.ts

@@ -0,0 +1,384 @@
+import mongoose from 'mongoose';
+import logger from '../logging/logger';
+import { MeterKey } from './definitions';
+import { UsageMeter, currentPeriod } from './UsageMeter';
+
+/**
+ * Rebuild the cumulative usage meters from the data they describe.
+ *
+ * **Why this can exist at all:** every meter reconciled here is *derived*.
+ * `storageBytes` is the sum of photo sizes, `trackedTickers` the count of
+ * active symbols, `seats` the members of the groups a subject owns — in each
+ * case the underlying collection is the record and the counter is only a cache
+ * of it, kept hot because a quota check cannot afford the aggregation. So the
+ * counter can always be recomputed, and reconciliation is cache invalidation.
+ *
+ * `aiJobs` is deliberately absent, and its absence is the same rule read the
+ * other way: nothing records that a GPU job happened except the counter itself,
+ * so there is no truth to recompute it from. That is precisely why
+ * `consumeQuota` rolls its increment back on rejection rather than leaving it —
+ * a leaked increment on that meter is permanent, where here it is temporary.
+ *
+ * **On racing with live writes:** a `consumeQuota` landing between the
+ * aggregation and the write is lost. That is tolerable for exactly the reason
+ * above — the next run re-derives the same number from source and converges —
+ * so this is scheduled at a quiet hour rather than defended with a lock.
+ *
+ * Reads collections owned by other services through the raw driver rather than
+ * mongoose models. They are not this library's documents to model, every
+ * service shares one database, and going through the driver avoids registering
+ * a second model for a schema that already exists elsewhere.
+ */
+
+/** Meters whose value can be recomputed from source data. See the note above. */
+export const RECONCILED_METERS: readonly MeterKey[] = ['storageBytes', 'trackedTickers', 'seats'];
+
+export interface ReconcileOptions {
+  /** Write the corrections. Default false — the caller must opt in to mutating. */
+  apply?: boolean;
+  /** Restrict to a single subject, for a targeted repair. */
+  subject?: string;
+}
+
+export interface MeterChange {
+  subjectId: string;
+  meter: MeterKey;
+  /** `null` when no counter existed yet. */
+  from: number | null;
+  to: number;
+}
+
+export interface OrphanedMeter {
+  subjectId: string;
+  meter: string;
+  count: number;
+}
+
+export interface ReconcileReport {
+  applied: boolean;
+  scannedUsers: number;
+  /** Subjects with any attributable usage, before any `subject` filter. */
+  subjects: number;
+  expected: Record<string, Record<MeterKey, number>>;
+  changes: MeterChange[];
+  written: number;
+  orphanedMeters: OrphanedMeter[];
+  warnings: string[];
+}
+
+const normalizeEmail = (value: unknown): string | undefined =>
+  typeof value === 'string' && value.trim() ? value.trim().toLowerCase() : undefined;
+
+interface MemberRow {
+  email?: string;
+  role?: string;
+}
+
+interface GroupRow {
+  _id: unknown;
+  name?: string;
+  createdBy?: string;
+  members?: MemberRow[];
+  deletedAt?: Date | null;
+}
+
+interface OwnedRow {
+  _id: unknown;
+  userId?: string;
+  groupId?: string;
+}
+
+/**
+ * Whoever currently holds the `owner` role — that is what group-service
+ * authorizes against, and `updateMemberRole` can move it while `createdBy`
+ * stays as first written. Several owners resolve to the lowest email so that a
+ * re-run bills the same person twice rather than two people once.
+ */
+function groupOwner(group: GroupRow, warnings: string[]): string | undefined {
+  const owners = (group.members ?? [])
+    .filter(m => m.role === 'owner')
+    .map(m => normalizeEmail(m.email))
+    .filter((e): e is string => Boolean(e))
+    .sort();
+
+  if (owners.length === 1) return owners[0];
+
+  if (owners.length === 0) {
+    const fallback = normalizeEmail(group.createdBy);
+    warnings.push(
+      `group ${group._id} ("${group.name}") has no member with role 'owner'; billing to createdBy=${fallback ?? 'MISSING'}`
+    );
+    return fallback;
+  }
+
+  warnings.push(`group ${group._id} ("${group.name}") has ${owners.length} owners (${owners.join(', ')}); billing to ${owners[0]}`);
+  return owners[0];
+}
+
+interface Bucket {
+  storageBytes: number;
+  trackedTickers: Set<string>;
+  seats: Set<string>;
+}
+
+async function computeExpected(): Promise<{
+  expected: Map<string, Bucket>;
+  warnings: string[];
+  scannedUsers: number;
+}> {
+  const db = mongoose.connection.db;
+  if (!db) throw new Error('Cannot reconcile usage meters: no active database connection');
+
+  const warnings: string[] = [];
+
+  // Album.userId / TrackedTicker.userId hold `req.user.id`, which is the
+  // auth-service User._id as a string, while a meter's subjectId is an email.
+  // Nothing personal can be attributed without coming back through this map.
+  const users = await db.collection('users').find({}, { projection: { email: 1 } }).toArray();
+  const userIdToEmail = new Map<string, string>();
+  for (const user of users) {
+    const email = normalizeEmail(user.email);
+    if (email) userIdToEmail.set(String(user._id), email);
+  }
+
+  const groups = (await db.collection('groups').find({}).toArray()) as unknown as GroupRow[];
+  const groupsById = new Map<string, { doc: GroupRow; owner?: string; deleted: boolean }>();
+  for (const group of groups) {
+    groupsById.set(String(group._id), {
+      doc: group,
+      owner: groupOwner(group, warnings),
+      deleted: group.deletedAt !== null && group.deletedAt !== undefined
+    });
+  }
+
+  const expected = new Map<string, Bucket>();
+  const bucket = (email: string): Bucket => {
+    let existing = expected.get(email);
+    if (!existing) {
+      existing = { storageBytes: 0, trackedTickers: new Set(), seats: new Set() };
+      expected.set(email, existing);
+    }
+    return existing;
+  };
+  // Every known user gets a bucket, so a user whose real usage is zero can
+  // still correct a stale non-zero counter back down.
+  for (const email of userIdToEmail.values()) bucket(email);
+
+  /**
+   * Group rows bill to the group owner: Family sells a pooled allowance, and
+   * charging the uploader would spend their personal free tier on it instead.
+   */
+  const subjectFor = (row: OwnedRow, label: string): string | undefined => {
+    const groupId = row.groupId ? String(row.groupId) : undefined;
+    if (groupId) {
+      const group = groupsById.get(groupId);
+      if (group?.owner) return group.owner;
+      // The group is gone but its rows are not. The uploader is the only
+      // identity left, and dropping the bytes would understate real storage.
+      const uploader = userIdToEmail.get(String(row.userId));
+      warnings.push(`${label} ${row._id} references missing group ${groupId}; billing to uploader=${uploader ?? 'UNATTRIBUTED'}`);
+      return uploader;
+    }
+    const owner = userIdToEmail.get(String(row.userId));
+    if (!owner) warnings.push(`${label} ${row._id} has userId=${row.userId}, which matches no user; UNATTRIBUTED`);
+    return owner;
+  };
+
+  // --- storageBytes ---------------------------------------------------------
+  // Summed from the photos themselves rather than from the denormalized
+  // Album.totalSize: that counter is what a drifting $inc would have corrupted,
+  // so reconciling against it would re-import the drift instead of fixing it.
+  const photoTotals = await db
+    .collection('photos')
+    .aggregate<{ _id: unknown; totalSize: number; photoCount: number }>(
+      [{ $group: { _id: '$albumId', totalSize: { $sum: '$size' }, photoCount: { $sum: 1 } } }],
+      { allowDiskUse: true }
+    )
+    .toArray();
+  const sizeByAlbum = new Map(photoTotals.map(row => [String(row._id), row.totalSize || 0]));
+
+  const albums = (await db
+    .collection('albums')
+    .find({}, { projection: { userId: 1, groupId: 1, totalSize: 1, name: 1 } })
+    .toArray()) as unknown as (OwnedRow & { totalSize?: number; name?: string })[];
+  const albumIds = new Set(albums.map(a => String(a._id)));
+
+  for (const album of albums) {
+    const realSize = sizeByAlbum.get(String(album._id)) ?? 0;
+    if ((album.totalSize ?? 0) !== realSize) {
+      warnings.push(
+        `album ${album._id} ("${album.name}") totalSize=${album.totalSize} but photos sum to ${realSize}; ` +
+          `metering the photo sum. The album counter is left alone — it is album-service's to maintain, ` +
+          `and silently rewriting another service's denormalized field would hide the bug rather than surface it.`
+      );
+    }
+    const subject = subjectFor(album, 'album');
+    if (subject) bucket(subject).storageBytes += realSize;
+  }
+
+  for (const row of photoTotals) {
+    if (!albumIds.has(String(row._id))) {
+      warnings.push(`${row.photoCount} photo(s) totalling ${row.totalSize} B reference missing album ${row._id}; UNATTRIBUTED`);
+    }
+  }
+
+  // --- trackedTickers -------------------------------------------------------
+  // Distinct symbols, not rows: the daily Alpha Vantage spend this limit
+  // recovers is one call per distinct active symbol, however many rows hold it.
+  const tickers = (await db
+    .collection('trackedtickers')
+    .find({ isActive: true }, { projection: { userId: 1, groupId: 1, ticker: 1 } })
+    .toArray()) as unknown as (OwnedRow & { ticker?: string })[];
+
+  for (const ticker of tickers) {
+    const subject = subjectFor(ticker, 'trackedTicker');
+    if (subject && ticker.ticker) bucket(subject).trackedTickers.add(String(ticker.ticker).toUpperCase());
+  }
+
+  // --- seats ----------------------------------------------------------------
+  // Distinct people across every live group the subject owns. Soft-deleted
+  // groups are skipped: unlike storage and tickers they cost nothing and grant
+  // nobody access, so charging for them would charge for nothing.
+  for (const { doc, owner, deleted } of groupsById.values()) {
+    if (deleted || !owner) continue;
+    const seats = bucket(owner).seats;
+    for (const member of doc.members ?? []) {
+      const email = normalizeEmail(member.email);
+      if (email) seats.add(email);
+    }
+  }
+
+  return { expected, warnings, scannedUsers: users.length };
+}
+
+/**
+ * Recompute every derived meter and, with `apply`, write back the ones that
+ * disagree.
+ *
+ * Idempotent: it computes absolute values and writes only differences, so a
+ * second run in a row is a no-op.
+ */
+export async function reconcileUsageMeters(options: ReconcileOptions = {}): Promise<ReconcileReport> {
+  const { apply = false, subject: only } = options;
+  const onlySubject = normalizeEmail(only);
+
+  const { expected, warnings, scannedUsers } = await computeExpected();
+
+  const current = await UsageMeter.find({ meter: { $in: RECONCILED_METERS }, period: 'ALL' }).lean();
+  const currentByKey = new Map(current.map(doc => [`${normalizeEmail(doc.subjectId)}::${doc.meter}`, doc]));
+
+  const subjects = Array.from(expected.keys())
+    .filter(email => !onlySubject || email === onlySubject)
+    .sort();
+
+  const changes: MeterChange[] = [];
+  const ops = [];
+
+  for (const subjectId of subjects) {
+    const counts = expected.get(subjectId)!;
+    const resolved: Record<MeterKey, number> = {
+      storageBytes: counts.storageBytes,
+      trackedTickers: counts.trackedTickers.size,
+      seats: counts.seats.size
+    } as Record<MeterKey, number>;
+
+    for (const meter of RECONCILED_METERS) {
+      const to = resolved[meter];
+      const existing = currentByKey.get(`${subjectId}::${meter}`);
+      const from = existing ? existing.count : null;
+
+      if (from === to) continue;
+      // Never create a row just to say zero — absence already reads as zero
+      // everywhere. An *existing* row must still be corrected down to zero, or
+      // a stale count outlives the data it was counting.
+      if (from === null && to === 0) continue;
+
+      changes.push({ subjectId, meter, from, to });
+      ops.push({
+        updateOne: {
+          filter: { subjectId, meter, period: currentPeriod(meter) },
+          update: {
+            $set: { count: to, updatedAt: new Date() },
+            $setOnInsert: { subjectId, meter, period: currentPeriod(meter), createdAt: new Date() }
+          },
+          upsert: true
+        }
+      });
+    }
+  }
+
+  // Meters for subjects the scan no longer recognizes — a removed user, or a
+  // subjectId written before an ownership change. Reported, never deleted: a
+  // reconciliation that silently drops billing state is worse than the drift.
+  const orphanedMeters: OrphanedMeter[] = current
+    .filter(doc => !expected.has(normalizeEmail(doc.subjectId) ?? ''))
+    .map(doc => ({ subjectId: doc.subjectId, meter: doc.meter, count: doc.count }));
+
+  let written = 0;
+  if (apply && ops.length > 0) {
+    const result = await UsageMeter.bulkWrite(ops);
+    written = result.modifiedCount + result.upsertedCount;
+  }
+
+  const expectedRecord: Record<string, Record<MeterKey, number>> = {};
+  for (const subjectId of subjects) {
+    const counts = expected.get(subjectId)!;
+    expectedRecord[subjectId] = {
+      storageBytes: counts.storageBytes,
+      trackedTickers: counts.trackedTickers.size,
+      seats: counts.seats.size
+    } as Record<MeterKey, number>;
+  }
+
+  return {
+    applied: apply,
+    scannedUsers,
+    subjects: expected.size,
+    expected: expectedRecord,
+    changes,
+    written,
+    orphanedMeters,
+    warnings
+  };
+}
+
+/**
+ * Run a reconciliation and log the outcome. The scheduled entry point.
+ *
+ * Never throws: this runs from a cron callback with no caller to catch it, and
+ * an unhandled rejection there takes the process down over a housekeeping job.
+ */
+export async function runUsageReconciliation(): Promise<ReconcileReport | null> {
+  const startedAt = Date.now();
+  try {
+    const report = await reconcileUsageMeters({ apply: true });
+
+    // A correction is drift that already happened — the counter had gone wrong
+    // and users were being metered against a wrong number until this ran.
+    // Logged at `warn` so a persistent leak in consumeQuota/releaseQuota is
+    // visible as a recurring signal rather than buried in an info line.
+    const level = report.changes.length > 0 ? 'warn' : 'info';
+    logger[level]('entitlement.reconciled', {
+      event: 'entitlement.reconciled',
+      subjects: report.subjects,
+      corrected: report.changes.length,
+      written: report.written,
+      orphaned: report.orphanedMeters.length,
+      warnings: report.warnings.length,
+      durationMs: Date.now() - startedAt,
+      changes: report.changes.slice(0, 20)
+    });
+
+    for (const warning of report.warnings.slice(0, 20)) {
+      logger.warn('entitlement.reconcile_warning', { event: 'entitlement.reconcile_warning', detail: warning });
+    }
+
+    return report;
+  } catch (err) {
+    logger.error('Usage meter reconciliation failed', {
+      error: (err as Error)?.message,
+      durationMs: Date.now() - startedAt
+    });
+    return null;
+  }
+}

package/src/entitlements/types.ts

@@ -19,12 +19,3 @@ export interface Entitlements {
    */
   stale?: boolean;
 }
-
-/**
- * `off` — no metering, no blocking. `observe` — record usage and log what
- * would have been blocked, block nothing. `enforce` — block.
- *
- * Observe is the default because the configured limits start as guesses; two
- * weeks of `entitlement.would_block` volume is what turns them into numbers.
- */
-export type EntitlementsMode = 'off' | 'observe' | 'enforce';

package/src/entitlements/usage.test.ts

@@ -50,7 +50,7 @@ afterAll(async () => {
 });
 
 beforeEach(async () => {
-  process.env = { ...ORIGINAL_ENV, ENTITLEMENTS_MODE: 'enforce' };
+  process.env = { ...ORIGINAL_ENV };
   await UsageMeter.deleteMany({});
   // The unique index is what makes the concurrent upserts below collapse onto
   // one document instead of racing into duplicates.
@@ -136,32 +136,34 @@ describe('consumeQuota', () => {
   });
 });
 
-describe('modes', () => {
-  it('records nothing and blocks nothing when off', async () => {
-    process.env.ENTITLEMENTS_MODE = 'off';
+describe('rejection logging', () => {
+  it('logs the block alongside throwing, so an operator sees what a user saw', async () => {
     await setUsage('user@example.com', 'aiJobs', 5);
 
     await expect(
-      consumeQuota({ subjectId: 'user@example.com', meter: 'aiJobs', entitlements: entitlements() })
-    ).resolves.toBeUndefined();
-    expect(await getUsage('user@example.com', 'aiJobs')).toBe(5);
+      consumeQuota({ subjectId: 'user@example.com', meter: 'aiJobs', entitlements: entitlements(), route: '/ai/dispatch' })
+    ).rejects.toMatchObject({ statusCode: 402 });
+
+    expect(logger.info).toHaveBeenCalledWith(
+      'entitlement.blocked',
+      expect.objectContaining({ meter: 'aiJobs', used: 5, limit: 5, route: '/ai/dispatch' })
+    );
   });
 
-  it('keeps the increment and logs would_block instead of rejecting when observing', async () => {
-    process.env.ENTITLEMENTS_MODE = 'observe';
+  it('flags a block that came from stale fallback limits rather than the real plan', async () => {
     await setUsage('user@example.com', 'aiJobs', 5);
 
     await expect(
-      consumeQuota({ subjectId: 'user@example.com', meter: 'aiJobs', entitlements: entitlements(), route: '/ai/dispatch' })
-    ).resolves.toBeUndefined();
+      consumeQuota({
+        subjectId: 'user@example.com',
+        meter: 'aiJobs',
+        entitlements: entitlements({ stale: true })
+      })
+    ).rejects.toMatchObject({ statusCode: 402 });
 
-    // Observe mode exists to measure real demand — rolling the count back would
-    // under-report exactly the users the limits are being calibrated against.
-    expect(await getUsage('user@example.com', 'aiJobs')).toBe(6);
-    expect(logger.info).toHaveBeenCalledWith(
-      'entitlement.would_block',
-      expect.objectContaining({ meter: 'aiJobs', used: 5, limit: 5, route: '/ai/dispatch' })
-    );
+    // A run of these means payment-service is down and paying users are being
+    // held to free limits — a different incident from a limit set too tight.
+    expect(logger.info).toHaveBeenCalledWith('entitlement.blocked', expect.objectContaining({ stale: true }));
   });
 });
 
@@ -175,28 +177,22 @@ describe('checkQuota', () => {
     expect(await getUsage('user@example.com', 'storageBytes')).toBe(400);
   });
 
-  it('logs would_block instead of rejecting when observing', async () => {
-    process.env.ENTITLEMENTS_MODE = 'observe';
+  it('rejects, and writes nothing, when the requested amount would overrun', async () => {
     await setUsage('user@example.com', 'storageBytes', 400);
 
     await expect(
       checkQuota({ subjectId: 'user@example.com', meter: 'storageBytes', amount: 601, entitlements: entitlements() })
-    ).resolves.toBeUndefined();
+    ).rejects.toMatchObject({ statusCode: 402 });
+
+    // Read-only by contract: the increment belongs to the point the resource
+    // actually comes into existence, not to the gate in front of it.
+    expect(await getUsage('user@example.com', 'storageBytes')).toBe(400);
     expect(logger.info).toHaveBeenCalledWith(
-      'entitlement.would_block',
+      'entitlement.blocked',
       expect.objectContaining({ meter: 'storageBytes', used: 400, limit: 1000 })
     );
   });
 
-  it('is inert when the mode is off, even over the limit', async () => {
-    process.env.ENTITLEMENTS_MODE = 'off';
-    await setUsage('user@example.com', 'storageBytes', 5000);
-
-    await expect(
-      checkQuota({ subjectId: 'user@example.com', meter: 'storageBytes', amount: 1, entitlements: entitlements() })
-    ).resolves.toBeUndefined();
-  });
-
   it('skips the usage read entirely for an unlimited meter', async () => {
     const unlimited = entitlements({ limits: normalizeLimits({ meters: { storageBytes: UNLIMITED } }, 'free') });
 

package/src/entitlements/usage.ts

@@ -3,7 +3,6 @@ import { PaymentRequiredError } from '../errors/HttpError';
 import { UsageMeter, currentPeriod } from './UsageMeter';
 import { MeterKey, METER_KEYS, UNLIMITED, fitsWithin } from './definitions';
 import { Entitlements } from './types';
-import { getEntitlementsMode } from './mode';
 
 export interface QuotaRequest {
   /** Who the usage is billed to — see `IUsageMeter.subjectId`. */
@@ -12,7 +11,7 @@ export interface QuotaRequest {
   /** How much this request wants. Bytes for `storageBytes`, otherwise a count. */
   amount?: number;
   entitlements: Entitlements;
-  /** Attached to the observe-mode log line so a would-block can be traced to a route. */
+  /** Attached to the rejection log so a 402 can be traced back to a route. */
   route?: string;
 }
 
@@ -52,9 +51,15 @@ function quotaError(req: QuotaRequest, used: number, limit: number): PaymentRequ
   });
 }
 
-function logWouldBlock(req: QuotaRequest, used: number, limit: number): void {
-  logger.info('entitlement.would_block', {
-    event: 'entitlement.would_block',
+/**
+ * Every rejection is logged, not just thrown. The 402 reaches the user, but
+ * only this line reaches the operator — and the two questions worth asking of a
+ * paywall (which meter fires most, and how often it fires on a *stale* fallback
+ * rather than on real limits) are answerable from nothing else.
+ */
+function logBlocked(req: QuotaRequest, used: number, limit: number): void {
+  logger.info('entitlement.blocked', {
+    event: 'entitlement.blocked',
     meter: req.meter,
     subjectId: req.subjectId,
     amount: req.amount ?? 1,
@@ -75,19 +80,13 @@ function logWouldBlock(req: QuotaRequest, used: number, limit: number): void {
  * the underlying data. Use `consumeQuota` where the count is the only record.
  */
 export async function checkQuota(req: QuotaRequest): Promise<void> {
-  const mode = getEntitlementsMode();
-  if (mode === 'off') return;
-
   const limit = req.entitlements.limits.meters[req.meter] ?? UNLIMITED;
   if (limit === UNLIMITED) return;
 
   const used = await getUsage(req.subjectId, req.meter);
   if (fitsWithin(limit, used, req.amount ?? 1)) return;
 
-  if (mode === 'observe') {
-    logWouldBlock(req, used, limit);
-    return;
-  }
+  logBlocked(req, used, limit);
   throw quotaError(req, used, limit);
 }
 
@@ -106,9 +105,6 @@ export async function checkQuota(req: QuotaRequest): Promise<void> {
  * who cannot upload despite being well under their limit.
  */
 export async function consumeQuota(req: QuotaRequest): Promise<void> {
-  const mode = getEntitlementsMode();
-  if (mode === 'off') return;
-
   const amount = req.amount ?? 1;
   const limit = req.entitlements.limits.meters[req.meter] ?? UNLIMITED;
   const period = currentPeriod(req.meter);
@@ -124,13 +120,7 @@ export async function consumeQuota(req: QuotaRequest): Promise<void> {
 
   const usedBefore = usage.count - amount;
 
-  if (mode === 'observe') {
-    // Keep the increment: observe mode exists to measure real demand, and a
-    // rolled-back count would under-report exactly the users who matter.
-    logWouldBlock(req, usedBefore, limit);
-    return;
-  }
-
+  logBlocked(req, usedBefore, limit);
   await releaseQuota(subjectId, req.meter, amount);
   throw quotaError(req, usedBefore, limit);
 }