@dhyasama/totem-models 12.25.0 → 12.26.0

package/lib/Account.js

@@ -77,6 +77,19 @@ module.exports = function(mongoose, config) {
       sendSystemEmails: { type: Boolean, default: false },
     },
 
+    // Granola native REST API integration. Keys are per-user, so they live on
+    // the account: when active, webhooks polls this user's Granola notes using
+    // apiKey (see webhooks/src/granola). Opt-in — only synced when a key is set.
+    granola: {
+      active: { type: Boolean, default: false },
+      apiKey: { type: String, trim: true, default: '' },
+      // Sync watermark, machine-written by the webhooks poller (same pattern as
+      // auth.google.nextSyncToken). Only ever update it via a targeted
+      // $set: { 'granola.updatedAfter': ... } — a blind rewrite of the whole
+      // granola subdocument (e.g. from a settings form) would clobber it.
+      updatedAfter: { type: Date, default: null }
+    },
+
     summary: {
       active: { type: Boolean, default: false }
     }

package/lib/Organization.js

@@ -225,13 +225,6 @@ module.exports = function(mongoose, config) {
           active: { type: Boolean, default: false }
         },
 
-        // Granola native REST API integration. When active, webhooks polls this
-        // customer's Granola notes using apiKey (see webhooks/src/granola).
-        granola: {
-          active: { type: Boolean, default: false },
-          apiKey: { type: String, trim: true }
-        },
-
         sheet: {
           id: { type: String, trim: true, unique: true, partialFilterExpression : { type :"string" } },
           format: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dhyasama/totem-models",
-  "version": "12.25.0",
+  "version": "12.26.0",
   "author": "Jason Reynolds",
   "license": "UNLICENSED",
   "description": "Models for Totem platform",

package/scripts/audit-metric-currency-scale.js

@@ -0,0 +1,108 @@
+'use strict';
+
+/**
+ * READ-ONLY audit for the snapshot -> metric currency/scale migration.
+ *
+ * Reports how many snapshots and metrics are missing currency/scale, and the distribution of
+ * values present. Run this BEFORE the backfill to see how many metrics would receive the
+ * default ('USD' / 1) rather than a real snapshot value.
+ *
+ * Makes NO writes.
+ *
+ * Usage:
+ *   node scripts/audit-metric-currency-scale.js [<customerId>] [--config <path>]
+ *   node scripts/audit-metric-currency-scale.js --config ../totem-web/config/config.js
+ *   DB_URI="mongodb://..." node scripts/audit-metric-currency-scale.js
+ */
+
+const mongoose = require('mongoose');
+const path = require('path');
+
+const args = process.argv.slice(2);
+const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
+
+let dbUri = null;
+const configIndex = args.indexOf('--config');
+if (configIndex !== -1 && args[configIndex + 1]) dbUri = require(path.resolve(args[configIndex + 1])).db.uri;
+if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
+if (!dbUri) { console.error('Error: --config or DB_URI required'); process.exit(1); }
+
+console.log('Audit scope:', CUSTOMER_ID ? `customer ${CUSTOMER_ID}` : 'ALL customers');
+console.log('Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'), '\n');
+
+mongoose.connect(dbUri);
+const db = mongoose.connection;
+db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
+db.once('open', async () => {
+  try { await audit(db); process.exit(0); }
+  catch (err) { console.error('Audit failed:', err); process.exit(1); }
+});
+
+// A field is "missing" if absent or null (empty string counted separately as its own bucket).
+const isNull = (f) => ({ $eq: [{ $ifNull: [f, null] }, null] });
+
+async function audit(db) {
+  const Financials = db.collection('financials');
+  const match = CUSTOMER_ID ? { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) } : {};
+
+  const docs = await Financials.countDocuments(match);
+
+  // ---- snapshot level ----
+  const [snap] = await Financials.aggregate([
+    { $match: match },
+    { $unwind: '$snapshots' },
+    { $group: {
+        _id: null,
+        total: { $sum: 1 },
+        missingCurrency: { $sum: { $cond: [isNull('$snapshots.currency'), 1, 0] } },
+        emptyCurrency: { $sum: { $cond: [{ $eq: ['$snapshots.currency', ''] }, 1, 0] } },
+        missingScale: { $sum: { $cond: [isNull('$snapshots.scale'), 1, 0] } },
+    } },
+  ]).toArray();
+
+  // ---- metric level (what the backfill actually writes) ----
+  const [met] = await Financials.aggregate([
+    { $match: match },
+    { $unwind: '$snapshots' },
+    { $unwind: '$snapshots.metrics' },
+    { $group: {
+        _id: null,
+        total: { $sum: 1 },
+        missingCurrency: { $sum: { $cond: [isNull('$snapshots.metrics.currency'), 1, 0] } },
+        missingScale: { $sum: { $cond: [isNull('$snapshots.metrics.scale'), 1, 0] } },
+        // metrics whose snapshot ALSO lacks currency -> these get the 'USD' default on backfill
+        metricAndSnapshotMissingCurrency: { $sum: { $cond: [{ $and: [isNull('$snapshots.metrics.currency'), isNull('$snapshots.currency')] }, 1, 0] } },
+    } },
+  ]).toArray();
+
+  // ---- distribution of snapshot currency/scale values present ----
+  const curDist = await Financials.aggregate([
+    { $match: match }, { $unwind: '$snapshots' },
+    { $group: { _id: '$snapshots.currency', n: { $sum: 1 } } }, { $sort: { n: -1 } },
+  ]).toArray();
+  const scaleDist = await Financials.aggregate([
+    { $match: match }, { $unwind: '$snapshots' },
+    { $group: { _id: '$snapshots.scale', n: { $sum: 1 } } }, { $sort: { n: -1 } },
+  ]).toArray();
+
+  const s = snap || { total: 0, missingCurrency: 0, emptyCurrency: 0, missingScale: 0 };
+  const m = met || { total: 0, missingCurrency: 0, missingScale: 0, metricAndSnapshotMissingCurrency: 0 };
+  const pct = (a, b) => b ? (100 * a / b).toFixed(1) + '%' : '0%';
+
+  console.log('=== Financials ===');
+  console.log(`  documents: ${docs}`);
+  console.log('\n=== Snapshots ===');
+  console.log(`  total:              ${s.total}`);
+  console.log(`  missing currency:   ${s.missingCurrency} (${pct(s.missingCurrency, s.total)})`);
+  console.log(`  empty-string currency: ${s.emptyCurrency}`);
+  console.log(`  missing scale:      ${s.missingScale} (${pct(s.missingScale, s.total)})`);
+  console.log('\n=== Metrics (what the backfill writes) ===');
+  console.log(`  total:              ${m.total}`);
+  console.log(`  missing currency:   ${m.missingCurrency} (${pct(m.missingCurrency, m.total)})  <- will be backfilled`);
+  console.log(`  missing scale:      ${m.missingScale} (${pct(m.missingScale, m.total)})  <- will be backfilled`);
+  console.log(`  missing currency AND snapshot has none -> would get 'USD' default: ${m.metricAndSnapshotMissingCurrency}`);
+  console.log('\n=== Snapshot currency distribution ===');
+  curDist.forEach((c) => console.log(`  ${JSON.stringify(c._id)}: ${c.n}`));
+  console.log('\n=== Snapshot scale distribution ===');
+  scaleDist.forEach((c) => console.log(`  ${JSON.stringify(c._id)}: ${c.n}`));
+}

package/scripts/backfill-metric-currency-scale.js

@@ -0,0 +1,193 @@
+'use strict';
+
+/**
+ * SAFE, STANDALONE backfill: snapshot-level currency/scale -> per-metric currency/scale.
+ *
+ * For every metric that does not already carry its own `currency`/`scale`, copy it down from the
+ * (now-deprecated) snapshot-level `currency`/`scale`. Also backfills the metric's `previous`
+ * sub-state when present (so undo round-trips), and normalizes any null/absent snapshot-level
+ * `scale` to 1 (a null scale already means "unscaled" = 1) so no null scales remain anywhere.
+ * A missing snapshot `scale` yields metric `scale: 1`; a missing snapshot `currency` yields 'USD'
+ * (per the prod audit this never occurs — every snapshot has a real currency).
+ *
+ * Why a dedicated script (not scripts/migrate-financials-schema.js): that script bundles 11 other
+ * transformations, including a metric "convert branch" that rebuilds the whole metrics array and can
+ * DROP already-migrated metrics in a mixed-state snapshot. This script has NO such path — it only
+ * adds fields, never rewrites arrays or removes anything. It is:
+ *   - additive      : only ever $set's currency/scale on metrics that lack them
+ *   - idempotent    : re-running is a no-op
+ *   - value-safe    : never touches metric.value/breakdown/etc.; no displayed number changes
+ *     (metric.currency := snapshot.currency, which read paths already fall back to today)
+ *
+ * Usage:
+ *   node scripts/backfill-metric-currency-scale.js <customerId> [--config <path>] [--dry-run]
+ *   node scripts/backfill-metric-currency-scale.js --all-customers --config <path> [--dry-run]
+ *   DB_URI="mongodb://..." node scripts/backfill-metric-currency-scale.js <customerId> --dry-run
+ *
+ * ALWAYS run with --dry-run first and inspect the output.
+ */
+
+// Valid metric scales (must match the enum in lib/Financials.js). A corrupt legacy snapshot scale
+// is sanitized to 1 rather than propagated into a metric (which would fail a future validated write).
+const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
+function sanitizeScale(scale) {
+  return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
+}
+
+/**
+ * Pure function: compute the $set backfill for a single financials document.
+ * Returns { updates: {dotPath: value}, changes: [string] }. Empty updates => nothing to do.
+ * No DB access, so this is directly unit-testable.
+ */
+function computeBackfillUpdates(financial) {
+  const updates = {};
+  const changes = [];
+  if (!financial || !Array.isArray(financial.snapshots)) return { updates, changes };
+
+  financial.snapshots.forEach((snapshot, i) => {
+    if (!snapshot) return;
+
+    const currency = snapshot.currency || 'USD';
+    const scale = sanitizeScale(snapshot.scale);
+
+    // Normalize a null/absent/invalid snapshot-level scale to 1 so no null scales remain anywhere
+    // (a null scale already means "unscaled" = 1, so this is a value-safe cleanup).
+    if (!VALID_METRIC_SCALES.includes(snapshot.scale)) {
+      updates[`snapshots.${i}.scale`] = 1;
+      changes.push(`  - snapshot ${i}: scale ${JSON.stringify(snapshot.scale)} -> 1`);
+    }
+
+    if (!Array.isArray(snapshot.metrics)) return;
+    snapshot.metrics.forEach((metric, j) => {
+      if (!metric) return;
+
+      if (!('currency' in metric)) {
+        updates[`snapshots.${i}.metrics.${j}.currency`] = currency;
+        changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): currency := ${currency}`);
+      }
+      if (!('scale' in metric)) {
+        updates[`snapshots.${i}.metrics.${j}.scale`] = scale;
+        changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): scale := ${scale}`);
+      }
+
+      // Backfill the prior-state sub-doc too, so undo restores currency/scale rather than unsetting them.
+      const prev = metric.previous;
+      if (prev && typeof prev === 'object') {
+        if (!('currency' in prev)) {
+          updates[`snapshots.${i}.metrics.${j}.previous.currency`] = currency;
+          changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): previous.currency := ${currency}`);
+        }
+        if (!('scale' in prev)) {
+          updates[`snapshots.${i}.metrics.${j}.previous.scale`] = scale;
+          changes.push(`  - snapshot ${i} metric ${j} (${metric.name}): previous.scale := ${scale}`);
+        }
+      }
+    });
+  });
+
+  return { updates, changes };
+}
+
+module.exports = { computeBackfillUpdates, sanitizeScale };
+
+// ---------------------------------------------------------------------------
+// DB runner (only when executed directly, so the pure logic above stays testable)
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const mongoose = require('mongoose');
+  const path = require('path');
+
+  const args = process.argv.slice(2);
+  const DRY_RUN = args.includes('--dry-run');
+  const ALL_CUSTOMERS = args.includes('--all-customers');
+  const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
+
+  let configPath = null;
+  let dbUri = null;
+
+  const configIndex = args.indexOf('--config');
+  if (configIndex !== -1 && args[configIndex + 1]) configPath = path.resolve(args[configIndex + 1]);
+  const dbUriIndex = args.indexOf('--db-uri');
+  if (dbUriIndex !== -1 && args[dbUriIndex + 1]) dbUri = args[dbUriIndex + 1];
+  if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
+
+  if (!CUSTOMER_ID && !ALL_CUSTOMERS) {
+    console.error('Error: provide a <customerId> or --all-customers');
+    process.exit(1);
+  }
+  if (configPath) {
+    console.log(`Loading config from: ${configPath}`);
+    dbUri = require(configPath).db.uri;
+  }
+  if (!dbUri) {
+    console.error('Error: --config, --db-uri, or DB_URI is required');
+    process.exit(1);
+  }
+
+  console.log('Backfill Configuration:');
+  console.log('  Scope:', ALL_CUSTOMERS ? 'ALL customers' : `customer ${CUSTOMER_ID}`);
+  console.log('  Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'));
+  console.log('  Dry Run:', DRY_RUN);
+  console.log('');
+
+  mongoose.connect(dbUri);
+  const db = mongoose.connection;
+  db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
+  db.once('open', async () => {
+    console.log('Connected to MongoDB');
+    try {
+      await run(db);
+      console.log('\nBackfill completed successfully!');
+      process.exit(0);
+    } catch (err) {
+      console.error('\nBackfill failed:', err);
+      process.exit(1);
+    }
+  });
+
+  async function run(db) {
+    const Financials = db.collection('financials');
+    const query = ALL_CUSTOMERS ? {} : { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) };
+    const financials = await Financials.find(query).toArray();
+    console.log(`Found ${financials.length} financial document(s)\n`);
+
+    // Detailed per-field logging is far too noisy at prod scale (100k+ writes), so print full
+    // detail only for the first few changed docs as a sample (or all docs with --verbose), and
+    // otherwise track category totals derived from the update key suffixes.
+    const VERBOSE = process.argv.includes('--verbose');
+    const SAMPLE_DOCS = 3;
+    let changed = 0, unchanged = 0, fieldWrites = 0;
+    let metricCurrency = 0, metricScale = 0, prevWrites = 0, snapshotScaleFixes = 0;
+
+    for (const financial of financials) {
+      const { updates, changes } = computeBackfillUpdates(financial);
+      const keys = Object.keys(updates);
+      if (keys.length === 0) { unchanged++; continue; }
+      fieldWrites += keys.length;
+      keys.forEach((k) => {
+        if (/\.metrics\.\d+\.currency$/.test(k)) metricCurrency++;
+        else if (/\.metrics\.\d+\.scale$/.test(k)) metricScale++;
+        else if (/\.previous\./.test(k)) prevWrites++;
+        else if (/^snapshots\.\d+\.scale$/.test(k)) snapshotScaleFixes++;
+      });
+
+      if (!DRY_RUN) await Financials.updateOne({ _id: financial._id }, { $set: updates });
+
+      if (VERBOSE || changed < SAMPLE_DOCS) {
+        console.log(`${DRY_RUN ? '[DRY RUN] would update' : 'updated'} ${financial._id} (${keys.length} writes)${(!VERBOSE && changed === SAMPLE_DOCS - 1) ? '  [sample; further per-doc detail suppressed]' : ''}`);
+        changes.forEach((c) => console.log(c));
+        console.log('');
+      }
+      changed++;
+    }
+
+    console.log('=== Backfill Summary ===');
+    console.log(`Documents ${DRY_RUN ? 'that would change' : 'changed'}: ${changed} / ${financials.length} (already complete: ${unchanged})`);
+    console.log(`Total field writes: ${fieldWrites}`);
+    console.log(`  metric currency:      ${metricCurrency}`);
+    console.log(`  metric scale:         ${metricScale}`);
+    console.log(`  metric.previous:      ${prevWrites}`);
+    console.log(`  snapshot null-scale->1: ${snapshotScaleFixes}`);
+    if (DRY_RUN) console.log('\nThis was a dry run. No changes were made.');
+  }
+}