@dhyasama/totem-models 12.24.0 → 12.26.0

package/lib/Account.js

@@ -82,7 +82,12 @@ module.exports = function(mongoose, config) {
     // apiKey (see webhooks/src/granola). Opt-in — only synced when a key is set.
     granola: {
       active: { type: Boolean, default: false },
-      apiKey: { type: String, trim: true, default: '' }
+      apiKey: { type: String, trim: true, default: '' },
+      // Sync watermark, machine-written by the webhooks poller (same pattern as
+      // auth.google.nextSyncToken). Only ever update it via a targeted
+      // $set: { 'granola.updatedAfter': ... } — a blind rewrite of the whole
+      // granola subdocument (e.g. from a settings form) would clobber it.
+      updatedAfter: { type: Date, default: null }
     },
 
     summary: {

package/lib/Financials.js

@@ -76,6 +76,9 @@ module.exports = function(mongoose, config) {
 
       uuid: { type: String },
 
+      // DEPRECATED: currency/scale now live per-metric (see metrics[].currency / metrics[].scale).
+      // Retained during the dual-write transition as the fallback default for un-migrated metrics;
+      // remove once all read paths resolve currency/scale per metric.
       currency: { type: String, default: 'USD' },
       scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000], default: 1 },
 
@@ -145,6 +148,11 @@ module.exports = function(mongoose, config) {
           }
         }],
         verified: { type: Boolean, default: false },
+        // Per-metric currency/scale. Intentionally NO defaults: an undefined value means
+        // "not yet migrated" and read paths fall back to the (deprecated) snapshot-level
+        // currency/scale. See scripts/migrate-financials-schema.js for the backfill.
+        currency: { type: String, trim: true },
+        scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000] },
         history: [{
           _id: false,
           timestamp: { type: Date, required: true, default: Date.now },
@@ -156,6 +164,8 @@ module.exports = function(mongoose, config) {
           value: { type: Number },
           breakdown: { type: Schema.Types.Mixed },
           calculation: { type: String, trim: true },
+          currency: { type: String, trim: true },
+          scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000] },
           sources: [{
             _id: false,
             model: { type: String, enum: ['Document', 'Meeting', 'Message', 'Note'] },
@@ -443,6 +453,8 @@ module.exports = function(mongoose, config) {
             value: currentMetric.value != null ? currentMetric.value : null,
             breakdown: currentMetric.breakdown != null ? currentMetric.breakdown : null,
             calculation: currentMetric.calculation || null,
+            currency: currentMetric.currency != null ? currentMetric.currency : null,
+            scale: currentMetric.scale != null ? currentMetric.scale : null,
             sources: currentMetric.sources || [],
             verified: currentMetric.verified != null ? currentMetric.verified : null,
             history: currentMetric.history || []
@@ -459,6 +471,11 @@ module.exports = function(mongoose, config) {
             'snapshots.$.metrics.$[metric].verified': metric.verified
           };
 
+          // Persist per-metric currency/scale only when the caller actually supplies a value, so a
+          // caller that omits them (or passes null) never clobbers an already-migrated metric.
+          if (metric.currency != null) { $set['snapshots.$.metrics.$[metric].currency'] = metric.currency; }
+          if (metric.scale != null) { $set['snapshots.$.metrics.$[metric].scale'] = metric.scale; }
+
           // Save previous state if the metric already exists
           if (previousState) {
             $set['snapshots.$.metrics.$[metric].previous'] = previousState;
@@ -564,7 +581,7 @@ module.exports = function(mongoose, config) {
           const $set = {};
           const $unset = { [prefix + '.previous']: 1 };
 
-          const fields = ['value', 'breakdown', 'calculation', 'sources', 'verified'];
+          const fields = ['value', 'breakdown', 'calculation', 'currency', 'scale', 'sources', 'verified'];
           fields.forEach(field => {
             if (prev[field] != null) {
               $set[prefix + '.' + field] = prev[field];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dhyasama/totem-models",
-  "version": "12.24.0",
+  "version": "12.26.0",
   "author": "Jason Reynolds",
   "license": "UNLICENSED",
   "description": "Models for Totem platform",

package/scripts/audit-metric-currency-scale.js

@@ -0,0 +1,108 @@
+'use strict';
+
+/**
+ * READ-ONLY audit for the snapshot -> metric currency/scale migration.
+ *
+ * Reports how many snapshots and metrics are missing currency/scale, and the distribution of
+ * values present. Run this BEFORE the backfill to see how many metrics would receive the
+ * default ('USD' / 1) rather than a real snapshot value.
+ *
+ * Makes NO writes.
+ *
+ * Usage:
+ *   node scripts/audit-metric-currency-scale.js [<customerId>] [--config <path>]
+ *   node scripts/audit-metric-currency-scale.js --config ../totem-web/config/config.js
+ *   DB_URI="mongodb://..." node scripts/audit-metric-currency-scale.js
+ */
+
+const mongoose = require('mongoose');
+const path = require('path');
+
+const args = process.argv.slice(2);
+const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
+
+let dbUri = null;
+const configIndex = args.indexOf('--config');
+if (configIndex !== -1 && args[configIndex + 1]) dbUri = require(path.resolve(args[configIndex + 1])).db.uri;
+if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
+if (!dbUri) { console.error('Error: --config or DB_URI required'); process.exit(1); }
+
+console.log('Audit scope:', CUSTOMER_ID ? `customer ${CUSTOMER_ID}` : 'ALL customers');
+console.log('Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'), '\n');
+
+mongoose.connect(dbUri);
+const db = mongoose.connection;
+db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
+db.once('open', async () => {
+  try { await audit(db); process.exit(0); }
+  catch (err) { console.error('Audit failed:', err); process.exit(1); }
+});
+
+// A field is "missing" if absent or null (empty string counted separately as its own bucket).
+const isNull = (f) => ({ $eq: [{ $ifNull: [f, null] }, null] });
+
+async function audit(db) {
+  const Financials = db.collection('financials');
+  const match = CUSTOMER_ID ? { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) } : {};
+
+  const docs = await Financials.countDocuments(match);
+
+  // ---- snapshot level ----
+  const [snap] = await Financials.aggregate([
+    { $match: match },
+    { $unwind: '$snapshots' },
+    { $group: {
+        _id: null,
+        total: { $sum: 1 },
+        missingCurrency: { $sum: { $cond: [isNull('$snapshots.currency'), 1, 0] } },
+        emptyCurrency: { $sum: { $cond: [{ $eq: ['$snapshots.currency', ''] }, 1, 0] } },
+        missingScale: { $sum: { $cond: [isNull('$snapshots.scale'), 1, 0] } },
+    } },
+  ]).toArray();
+
+  // ---- metric level (what the backfill actually writes) ----
+  const [met] = await Financials.aggregate([
+    { $match: match },
+    { $unwind: '$snapshots' },
+    { $unwind: '$snapshots.metrics' },
+    { $group: {
+        _id: null,
+        total: { $sum: 1 },
+        missingCurrency: { $sum: { $cond: [isNull('$snapshots.metrics.currency'), 1, 0] } },
+        missingScale: { $sum: { $cond: [isNull('$snapshots.metrics.scale'), 1, 0] } },
+        // metrics whose snapshot ALSO lacks currency -> these get the 'USD' default on backfill
+        metricAndSnapshotMissingCurrency: { $sum: { $cond: [{ $and: [isNull('$snapshots.metrics.currency'), isNull('$snapshots.currency')] }, 1, 0] } },
+    } },
+  ]).toArray();
+
+  // ---- distribution of snapshot currency/scale values present ----
+  const curDist = await Financials.aggregate([
+    { $match: match }, { $unwind: '$snapshots' },
+    { $group: { _id: '$snapshots.currency', n: { $sum: 1 } } }, { $sort: { n: -1 } },
+  ]).toArray();
+  const scaleDist = await Financials.aggregate([
+    { $match: match }, { $unwind: '$snapshots' },
+    { $group: { _id: '$snapshots.scale', n: { $sum: 1 } } }, { $sort: { n: -1 } },
+  ]).toArray();
+
+  const s = snap || { total: 0, missingCurrency: 0, emptyCurrency: 0, missingScale: 0 };
+  const m = met || { total: 0, missingCurrency: 0, missingScale: 0, metricAndSnapshotMissingCurrency: 0 };
+  const pct = (a, b) => b ? (100 * a / b).toFixed(1) + '%' : '0%';
+
+  console.log('=== Financials ===');
+  console.log(`  documents: ${docs}`);
+  console.log('\n=== Snapshots ===');
+  console.log(`  total:              ${s.total}`);
+  console.log(`  missing currency:   ${s.missingCurrency} (${pct(s.missingCurrency, s.total)})`);
+  console.log(`  empty-string currency: ${s.emptyCurrency}`);
+  console.log(`  missing scale:      ${s.missingScale} (${pct(s.missingScale, s.total)})`);
+  console.log('\n=== Metrics (what the backfill writes) ===');
+  console.log(`  total:              ${m.total}`);
+  console.log(`  missing currency:   ${m.missingCurrency} (${pct(m.missingCurrency, m.total)})  <- will be backfilled`);
+  console.log(`  missing scale:      ${m.missingScale} (${pct(m.missingScale, m.total)})  <- will be backfilled`);
+  console.log(`  missing currency AND snapshot has none -> would get 'USD' default: ${m.metricAndSnapshotMissingCurrency}`);
+  console.log('\n=== Snapshot currency distribution ===');
+  curDist.forEach((c) => console.log(`  ${JSON.stringify(c._id)}: ${c.n}`));
+  console.log('\n=== Snapshot scale distribution ===');
+  scaleDist.forEach((c) => console.log(`  ${JSON.stringify(c._id)}: ${c.n}`));
+}

package/scripts/backfill-metric-currency-scale.js

@@ -0,0 +1,193 @@
+'use strict';
+
+/**
+ * SAFE, STANDALONE backfill: snapshot-level currency/scale -> per-metric currency/scale.
+ *
+ * For every metric that does not already carry its own `currency`/`scale`, copy it down from the
+ * (now-deprecated) snapshot-level `currency`/`scale`. Also backfills the metric's `previous`
+ * sub-state when present (so undo round-trips), and normalizes any null/absent snapshot-level
+ * `scale` to 1 (a null scale already means "unscaled" = 1) so no null scales remain anywhere.
+ * A missing snapshot `scale` yields metric `scale: 1`; a missing snapshot `currency` yields 'USD'
+ * (per the prod audit this never occurs — every snapshot has a real currency).
+ *
+ * Why a dedicated script (not scripts/migrate-financials-schema.js): that script bundles 11 other
+ * transformations, including a metric "convert branch" that rebuilds the whole metrics array and can
+ * DROP already-migrated metrics in a mixed-state snapshot. This script has NO such path — it only
+ * adds fields, never rewrites arrays or removes anything. It is:
+ *   - additive      : only ever $set's currency/scale on metrics that lack them
+ *   - idempotent    : re-running is a no-op
+ *   - value-safe    : never touches metric.value/breakdown/etc.; no displayed number changes
+ *     (metric.currency := snapshot.currency, which read paths already fall back to today)
+ *
+ * Usage:
+ *   node scripts/backfill-metric-currency-scale.js <customerId> [--config <path>] [--dry-run]
+ *   node scripts/backfill-metric-currency-scale.js --all-customers --config <path> [--dry-run]
+ *   DB_URI="mongodb://..." node scripts/backfill-metric-currency-scale.js <customerId> --dry-run
+ *
+ * ALWAYS run with --dry-run first and inspect the output.
+ */
+
+// Valid metric scales (must match the enum in lib/Financials.js). A corrupt legacy snapshot scale
+// is sanitized to 1 rather than propagated into a metric (which would fail a future validated write).
+const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
+function sanitizeScale(scale) {
+  return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
+}
+
+/**
+ * Pure function: compute the $set backfill for a single financials document.
+ * Returns { updates: {dotPath: value}, changes: [string] }. Empty updates => nothing to do.
+ * No DB access, so this is directly unit-testable.
+ */
+function computeBackfillUpdates(financial) {
+  const updates = {};
+  const changes = [];
+  if (!financial || !Array.isArray(financial.snapshots)) return { updates, changes };
+
+  financial.snapshots.forEach((snapshot, i) => {
+    if (!snapshot) return;
+
+    const currency = snapshot.currency || 'USD';
+    const scale = sanitizeScale(snapshot.scale);
+
+    // Normalize a null/absent/invalid snapshot-level scale to 1 so no null scales remain anywhere
+    // (a null scale already means "unscaled" = 1, so this is a value-safe cleanup).
+    if (!VALID_METRIC_SCALES.includes(snapshot.scale)) {
+      updates[`snapshots.${i}.scale`] = 1;
+      changes.push(`  - snapshot ${i}: scale ${JSON.stringify(snapshot.scale)} -> 1`);
+    }
+
+    if (!Array.isArray(snapshot.metrics)) return;
+    snapshot.metrics.forEach((metric, j) => {
+      if (!metric) return;
+
+      if (!('currency' in metric)) {
+        updates[`snapshots.${i}.metrics.${j}.currency`] = currency;
+        changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): currency := ${currency}`);
+      }
+      if (!('scale' in metric)) {
+        updates[`snapshots.${i}.metrics.${j}.scale`] = scale;
+        changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): scale := ${scale}`);
+      }
+
+      // Backfill the prior-state sub-doc too, so undo restores currency/scale rather than unsetting them.
+      const prev = metric.previous;
+      if (prev && typeof prev === 'object') {
+        if (!('currency' in prev)) {
+          updates[`snapshots.${i}.metrics.${j}.previous.currency`] = currency;
+          changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): previous.currency := ${currency}`);
+        }
+        if (!('scale' in prev)) {
+          updates[`snapshots.${i}.metrics.${j}.previous.scale`] = scale;
+          changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): previous.scale := ${scale}`);
+        }
+      }
+    });
+  });
+
+  return { updates, changes };
+}
+
+module.exports = { computeBackfillUpdates, sanitizeScale };
+
+// ---------------------------------------------------------------------------
+// DB runner (only when executed directly, so the pure logic above stays testable)
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const mongoose = require('mongoose');
+  const path = require('path');
+
+  const args = process.argv.slice(2);
+  const DRY_RUN = args.includes('--dry-run');
+  const ALL_CUSTOMERS = args.includes('--all-customers');
+  const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
+
+  let configPath = null;
+  let dbUri = null;
+
+  const configIndex = args.indexOf('--config');
+  if (configIndex !== -1 && args[configIndex + 1]) configPath = path.resolve(args[configIndex + 1]);
+  const dbUriIndex = args.indexOf('--db-uri');
+  if (dbUriIndex !== -1 && args[dbUriIndex + 1]) dbUri = args[dbUriIndex + 1];
+  if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
+
+  if (!CUSTOMER_ID && !ALL_CUSTOMERS) {
+    console.error('Error: provide a <customerId> or --all-customers');
+    process.exit(1);
+  }
+  if (configPath) {
+    console.log(`Loading config from: ${configPath}`);
+    dbUri = require(configPath).db.uri;
+  }
+  if (!dbUri) {
+    console.error('Error: --config, --db-uri, or DB_URI is required');
+    process.exit(1);
+  }
+
+  console.log('Backfill Configuration:');
+  console.log('  Scope:', ALL_CUSTOMERS ? 'ALL customers' : `customer ${CUSTOMER_ID}`);
+  console.log('  Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'));
+  console.log('  Dry Run:', DRY_RUN);
+  console.log('');
+
+  mongoose.connect(dbUri);
+  const db = mongoose.connection;
+  db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
+  db.once('open', async () => {
+    console.log('Connected to MongoDB');
+    try {
+      await run(db);
+      console.log('\nBackfill completed successfully!');
+      process.exit(0);
+    } catch (err) {
+      console.error('\nBackfill failed:', err);
+      process.exit(1);
+    }
+  });
+
+  async function run(db) {
+    const Financials = db.collection('financials');
+    const query = ALL_CUSTOMERS ? {} : { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) };
+    const financials = await Financials.find(query).toArray();
+    console.log(`Found ${financials.length} financial document(s)\n`);
+
+    // Detailed per-field logging is far too noisy at prod scale (100k+ writes), so print full
+    // detail only for the first few changed docs as a sample (or all docs with --verbose), and
+    // otherwise track category totals derived from the update key suffixes.
+    const VERBOSE = process.argv.includes('--verbose');
+    const SAMPLE_DOCS = 3;
+    let changed = 0, unchanged = 0, fieldWrites = 0;
+    let metricCurrency = 0, metricScale = 0, prevWrites = 0, snapshotScaleFixes = 0;
+
+    for (const financial of financials) {
+      const { updates, changes } = computeBackfillUpdates(financial);
+      const keys = Object.keys(updates);
+      if (keys.length === 0) { unchanged++; continue; }
+      fieldWrites += keys.length;
+      keys.forEach((k) => {
+        if (/\.metrics\.\d+\.currency$/.test(k)) metricCurrency++;
+        else if (/\.metrics\.\d+\.scale$/.test(k)) metricScale++;
+        else if (/\.previous\./.test(k)) prevWrites++;
+        else if (/^snapshots\.\d+\.scale$/.test(k)) snapshotScaleFixes++;
+      });
+
+      if (!DRY_RUN) await Financials.updateOne({ _id: financial._id }, { $set: updates });
+
+      if (VERBOSE || changed < SAMPLE_DOCS) {
+        console.log(`${DRY_RUN ? '[DRY RUN] would update' : 'updated'} ${financial._id} (${keys.length} writes)${(!VERBOSE && changed === SAMPLE_DOCS - 1) ? '  [sample; further per-doc detail suppressed]' : ''}`);
+        changes.forEach((c) => console.log(c));
+        console.log('');
+      }
+      changed++;
+    }
+
+    console.log('=== Backfill Summary ===');
+    console.log(`Documents ${DRY_RUN ? 'that would change' : 'changed'}: ${changed} / ${financials.length} (already complete: ${unchanged})`);
+    console.log(`Total field writes: ${fieldWrites}`);
+    console.log(`  metric currency:      ${metricCurrency}`);
+    console.log(`  metric scale:         ${metricScale}`);
+    console.log(`  metric.previous:      ${prevWrites}`);
+    console.log(`  snapshot null-scale->1: ${snapshotScaleFixes}`);
+    if (DRY_RUN) console.log('\nThis was a dry run. No changes were made.');
+  }
+}

package/scripts/migrate-financials-schema.js

@@ -21,6 +21,10 @@
  * 10. Link metrics to their source financial statement documents (Income Statement,
  *     Balance Sheet, Cash Flow Statement) when available
  * 11. Remove projections array from root level
+ * 12. Backfill per-metric currency/scale from the (now-deprecated) snapshot-level
+ *     currency/scale. Idempotent: metrics that already carry their own are left untouched.
+ *     Defaults to 'USD' / 1 when the snapshot has neither. Part of the snapshot→metric
+ *     currency migration (dual-write transition; snapshot fields retained as fallback).
  *
  * Usage:
  *   node scripts/migrate-financials-schema.js <customerId> [--config <configPath>] [--dry-run]
@@ -81,6 +85,14 @@ if (configPath) {
   dbUri = config.db.uri;
 }
 
+// Valid metric scales (must match the enum in lib/Financials.js). The backfill sanitizes the
+// snapshot's stored scale against this set so a corrupt legacy value can't propagate into a
+// metric and fail a future validated write. Unknown/absent values fall back to 1.
+const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
+function sanitizeScale(scale) {
+  return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
+}
+
 console.log('Migration Configuration:');
 console.log('  Customer ID:', CUSTOMER_ID);
 console.log('  Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@')); // Mask credentials
@@ -312,12 +324,18 @@ async function migrateFinancials() {
               const source = getSourceForMetric(metric.name);
 
               // Create separate metric entries for each scenario that has a value
+              // Backfill per-metric currency/scale from the (now-deprecated) snapshot-level values
+              const metricCurrency = snapshot.currency || 'USD';
+              const metricScale = sanitizeScale(snapshot.scale);
+
               if ('actual' in metric && metric.actual !== undefined && metric.actual !== null) {
                 const newMetric = {
                   name: metric.name,
                   scenario: 'actual',
                   value: metric.actual,
                   breakdown: metric.breakdown || {},
+                  currency: metricCurrency,
+                  scale: metricScale,
                   verified: true,
                   history: historyArrayForMetrics
                 };
@@ -331,6 +349,8 @@ async function migrateFinancials() {
                   scenario: 'budget',
                   value: metric.budget,
                   breakdown: {},
+                  currency: metricCurrency,
+                  scale: metricScale,
                   verified: true,
                   history: historyArrayForMetrics
                 };
@@ -344,6 +364,8 @@ async function migrateFinancials() {
                   scenario: 'forecast',
                   value: metric.forecast,
                   breakdown: {},
+                  currency: metricCurrency,
+                  scale: metricScale,
                   verified: true,
                   history: historyArrayForMetrics
                 };
@@ -356,7 +378,7 @@ async function migrateFinancials() {
             updates[`snapshots.${index}.metrics`] = newMetrics;
             changes.push(`  - Snapshot ${index}: Converted ${oldMetrics.length} metric(s) to new structure (${newMetrics.length} entries, ${metricsWithSources} with sources)`);
           } else {
-            // Already in new structure, just add verified, history, and source if missing
+            // Already in new structure, just add verified, history, source, and currency/scale if missing
             oldMetrics.forEach((metric, mIndex) => {
               if (!('verified' in metric)) {
                 updates[`snapshots.${index}.metrics.${mIndex}.verified`] = true;
@@ -373,6 +395,17 @@ async function migrateFinancials() {
                   changes.push(`  - Snapshot ${index}: Added sources to metric ${mIndex} (${metric.name})`);
                 }
               }
+              // Backfill per-metric currency/scale from the (now-deprecated) snapshot-level values.
+              // Idempotent: only set when the metric doesn't already carry its own.
+              if (!('currency' in metric)) {
+                updates[`snapshots.${index}.metrics.${mIndex}.currency`] = snapshot.currency || 'USD';
+                changes.push(`  - Snapshot ${index}: Backfilled currency (${snapshot.currency || 'USD'}) to metric ${mIndex} (${metric.name})`);
+              }
+              if (!('scale' in metric)) {
+                const backfillScale = sanitizeScale(snapshot.scale);
+                updates[`snapshots.${index}.metrics.${mIndex}.scale`] = backfillScale;
+                changes.push(`  - Snapshot ${index}: Backfilled scale (${backfillScale}) to metric ${mIndex} (${metric.name})`);
+              }
             });
           }
         }