@dhyasama/totem-models 12.26.0 → 12.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/Financials.js CHANGED
@@ -76,12 +76,6 @@ module.exports = function(mongoose, config) {
76
76
 
77
77
  uuid: { type: String },
78
78
 
79
- // DEPRECATED: currency/scale now live per-metric (see metrics[].currency / metrics[].scale).
80
- // Retained during the dual-write transition as the fallback default for un-migrated metrics;
81
- // remove once all read paths resolve currency/scale per metric.
82
- currency: { type: String, default: 'USD' },
83
- scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000], default: 1 },
84
-
85
79
  approval:{ type: Boolean, default: false },
86
80
 
87
81
  status: { type: String, enum: ['Scheduled', 'Delivered', 'Opened', 'Overdue', 'Skipped', 'Pending', 'Completed', 'Archived'] },
@@ -148,9 +142,7 @@ module.exports = function(mongoose, config) {
148
142
  }
149
143
  }],
150
144
  verified: { type: Boolean, default: false },
151
- // Per-metric currency/scale. Intentionally NO defaults: an undefined value means
152
- // "not yet migrated" and read paths fall back to the (deprecated) snapshot-level
153
- // currency/scale. See scripts/migrate-financials-schema.js for the backfill.
145
+ // Currency and scale live exclusively on each metric/scenario.
154
146
  currency: { type: String, trim: true },
155
147
  scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000] },
156
148
  history: [{
@@ -678,6 +670,29 @@ module.exports = function(mongoose, config) {
678
670
 
679
671
  };
680
672
 
673
+ // Atomic find-or-create for the one Financials doc per (customer, organization).
674
+ // Callers must use this instead of getByOrg + new Financials() — that pattern races
675
+ // under concurrent requests and used to mint duplicate documents.
676
+ Financials.statics.getOrCreateByOrg = function getOrCreateByOrg(customerId, orgId, cb) {
677
+
678
+ const run = async () => {
679
+ const result = await this.findOneAndUpdate(
680
+ { 'customer': customerId, 'organization': orgId },
681
+ { $setOnInsert: { customer: customerId, organization: orgId, snapshots: [] } },
682
+ { upsert: true, new: true }
683
+ ).populate({
684
+ path: 'snapshots.documents.document',
685
+ match: { customer: customerId }
686
+ });
687
+
688
+ return result;
689
+ };
690
+
691
+ if (typeof cb === 'function') { run().then(result => cb(null, result), err => cb(err)); return; }
692
+ else { return run(); }
693
+
694
+ };
695
+
681
696
  Financials.statics.getByUUID = function getByUUID(uuid, cb) {
682
697
 
683
698
  const run = async () => {
@@ -1362,6 +1377,10 @@ module.exports = function(mongoose, config) {
1362
1377
  Financials.set('toJSON', { virtuals: true });
1363
1378
  Financials.set('autoIndex', false);
1364
1379
 
1380
+ // One Financials document per org per customer. autoIndex is off, so this is
1381
+ // documentation only — the index exists on production (customer_1_organization_1_unique).
1382
+ Financials.index({ customer: 1, organization: 1 }, { unique: true });
1383
+
1365
1384
  Financials.on('index', function(err) { console.log('error building financials indexes: ' + err); });
1366
1385
 
1367
1386
  mongoose.model('Financials', Financials);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dhyasama/totem-models",
3
- "version": "12.26.0",
3
+ "version": "12.28.0",
4
4
  "author": "Jason Reynolds",
5
5
  "license": "UNLICENSED",
6
6
  "description": "Models for Totem platform",
@@ -25,6 +25,8 @@
25
25
  * currency/scale. Idempotent: metrics that already carry their own are left untouched.
26
26
  * Defaults to 'USD' / 1 when the snapshot has neither. Part of the snapshot→metric
27
27
  * currency migration (dual-write transition; snapshot fields retained as fallback).
28
+ * Unitless metrics (currently Months of Runway) are normalized to scale 1 with no currency,
29
+ * including their optional previous/undo state.
28
30
  *
29
31
  * Usage:
30
32
  * node scripts/migrate-financials-schema.js <customerId> [--config <configPath>] [--dry-run]
@@ -40,6 +42,13 @@
40
42
 
41
43
  const mongoose = require('mongoose');
42
44
  const path = require('path');
45
+ // Legacy snapshot-level units are copied down to each metric verbatim; format-aware unit rules
46
+ // (currency on money metrics only) are applied afterwards by the registry-driven reconcile
47
+ // (totem-services/document-intelligence/scripts/reconcile_metric_units.js). No unit rule here is
48
+ // keyed on a metric's name.
49
+ const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
50
+ const sanitizeScale = (scale) => VALID_METRIC_SCALES.includes(scale) ? scale : 1;
51
+ const getMetricUnits = (snapshot) => ({ currency: snapshot.currency || 'USD', scale: sanitizeScale(snapshot.scale) });
43
52
 
44
53
  // Parse command line arguments
45
54
  const args = process.argv.slice(2);
@@ -85,14 +94,6 @@ if (configPath) {
85
94
  dbUri = config.db.uri;
86
95
  }
87
96
 
88
- // Valid metric scales (must match the enum in lib/Financials.js). The backfill sanitizes the
89
- // snapshot's stored scale against this set so a corrupt legacy value can't propagate into a
90
- // metric and fail a future validated write. Unknown/absent values fall back to 1.
91
- const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
92
- function sanitizeScale(scale) {
93
- return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
94
- }
95
-
96
97
  console.log('Migration Configuration:');
97
98
  console.log(' Customer ID:', CUSTOMER_ID);
98
99
  console.log(' Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@')); // Mask credentials
@@ -323,10 +324,8 @@ async function migrateFinancials() {
323
324
  oldMetrics.forEach((metric) => {
324
325
  const source = getSourceForMetric(metric.name);
325
326
 
326
- // Create separate metric entries for each scenario that has a value
327
- // Backfill per-metric currency/scale from the (now-deprecated) snapshot-level values
328
- const metricCurrency = snapshot.currency || 'USD';
329
- const metricScale = sanitizeScale(snapshot.scale);
327
+ // Create separate metric entries for each scenario that has a value.
328
+ const metricUnits = getMetricUnits(snapshot);
330
329
 
331
330
  if ('actual' in metric && metric.actual !== undefined && metric.actual !== null) {
332
331
  const newMetric = {
@@ -334,11 +333,11 @@ async function migrateFinancials() {
334
333
  scenario: 'actual',
335
334
  value: metric.actual,
336
335
  breakdown: metric.breakdown || {},
337
- currency: metricCurrency,
338
- scale: metricScale,
336
+ scale: metricUnits.scale,
339
337
  verified: true,
340
338
  history: historyArrayForMetrics
341
339
  };
340
+ if (metricUnits.currency) newMetric.currency = metricUnits.currency;
342
341
  if (source) newMetric.sources = [source];
343
342
  newMetrics.push(newMetric);
344
343
  }
@@ -349,11 +348,11 @@ async function migrateFinancials() {
349
348
  scenario: 'budget',
350
349
  value: metric.budget,
351
350
  breakdown: {},
352
- currency: metricCurrency,
353
- scale: metricScale,
351
+ scale: metricUnits.scale,
354
352
  verified: true,
355
353
  history: historyArrayForMetrics
356
354
  };
355
+ if (metricUnits.currency) newMetric.currency = metricUnits.currency;
357
356
  if (source) newMetric.sources = [source];
358
357
  newMetrics.push(newMetric);
359
358
  }
@@ -364,11 +363,11 @@ async function migrateFinancials() {
364
363
  scenario: 'forecast',
365
364
  value: metric.forecast,
366
365
  breakdown: {},
367
- currency: metricCurrency,
368
- scale: metricScale,
366
+ scale: metricUnits.scale,
369
367
  verified: true,
370
368
  history: historyArrayForMetrics
371
369
  };
370
+ if (metricUnits.currency) newMetric.currency = metricUnits.currency;
372
371
  if (source) newMetric.sources = [source];
373
372
  newMetrics.push(newMetric);
374
373
  }
@@ -395,16 +394,27 @@ async function migrateFinancials() {
395
394
  changes.push(` - Snapshot ${index}: Added sources to metric ${mIndex} (${metric.name})`);
396
395
  }
397
396
  }
398
- // Backfill per-metric currency/scale from the (now-deprecated) snapshot-level values.
399
- // Idempotent: only set when the metric doesn't already carry its own.
400
- if (!('currency' in metric)) {
401
- updates[`snapshots.${index}.metrics.${mIndex}.currency`] = snapshot.currency || 'USD';
402
- changes.push(` - Snapshot ${index}: Backfilled currency (${snapshot.currency || 'USD'}) to metric ${mIndex} (${metric.name})`);
403
- }
404
- if (!('scale' in metric)) {
405
- const backfillScale = sanitizeScale(snapshot.scale);
406
- updates[`snapshots.${index}.metrics.${mIndex}.scale`] = backfillScale;
407
- changes.push(` - Snapshot ${index}: Backfilled scale (${backfillScale}) to metric ${mIndex} (${metric.name})`);
397
+ const metricUnits = getMetricUnits(snapshot);
398
+ {
399
+ // Idempotent: only set when the metric doesn't already carry its own units.
400
+ if (!('currency' in metric)) {
401
+ updates[`snapshots.${index}.metrics.${mIndex}.currency`] = metricUnits.currency;
402
+ changes.push(` - Snapshot ${index}: Backfilled currency (${metricUnits.currency}) to metric ${mIndex} (${metric.name})`);
403
+ }
404
+ if (!('scale' in metric)) {
405
+ updates[`snapshots.${index}.metrics.${mIndex}.scale`] = metricUnits.scale;
406
+ changes.push(` - Snapshot ${index}: Backfilled scale (${metricUnits.scale}) to metric ${mIndex} (${metric.name})`);
407
+ }
408
+ if (metric.previous && typeof metric.previous === 'object') {
409
+ if (!('currency' in metric.previous)) {
410
+ updates[`snapshots.${index}.metrics.${mIndex}.previous.currency`] = metricUnits.currency;
411
+ changes.push(` - Snapshot ${index}: Backfilled previous currency (${metricUnits.currency}) on metric ${mIndex} (${metric.name})`);
412
+ }
413
+ if (!('scale' in metric.previous)) {
414
+ updates[`snapshots.${index}.metrics.${mIndex}.previous.scale`] = metricUnits.scale;
415
+ changes.push(` - Snapshot ${index}: Backfilled previous scale (${metricUnits.scale}) on metric ${mIndex} (${metric.name})`);
416
+ }
417
+ }
408
418
  }
409
419
  });
410
420
  }
@@ -5,8 +5,12 @@ var
5
5
  should = require("should"),
6
6
  config = require('./config')['test'],
7
7
  mongoose = require('mongoose'),
8
- databaseHelpers = require('./databaseHelpers'),
9
- Financials = mongoose.model('Financials');
8
+ databaseHelpers = require('./databaseHelpers');
9
+
10
+ // Allow this file to run standalone (models are otherwise registered by earlier test files)
11
+ if (!mongoose.models.Financials) { require('../index')(mongoose, config); }
12
+
13
+ var Financials = mongoose.model('Financials');
10
14
 
11
15
  var customerA = new mongoose.Types.ObjectId();
12
16
  var customerB = new mongoose.Types.ObjectId();
@@ -15,6 +19,27 @@ var orgB = new mongoose.Types.ObjectId();
15
19
 
16
20
  describe('Financials', function() {
17
21
 
22
+ it('does not define or persist snapshot-level currency and scale', function() {
23
+ var f = new Financials({
24
+ organization: orgA,
25
+ customer: customerA,
26
+ snapshots: [{
27
+ uuid: 'metric-units-only',
28
+ year: 2025,
29
+ period: 'FY',
30
+ currency: 'EUR',
31
+ scale: 1000,
32
+ metrics: [{ name: 'Revenue', scenario: 'actual', value: 10, currency: 'EUR', scale: 1000 }]
33
+ }]
34
+ });
35
+
36
+ var snapshot = f.snapshots[0].toObject();
37
+ should.not.exist(snapshot.currency);
38
+ should.not.exist(snapshot.scale);
39
+ snapshot.metrics[0].currency.should.equal('EUR');
40
+ snapshot.metrics[0].scale.should.equal(1000);
41
+ });
42
+
18
43
  before(function(done) {
19
44
  databaseHelpers.connect(mongoose, done);
20
45
  });
@@ -164,4 +189,91 @@ describe('Financials', function() {
164
189
 
165
190
  });
166
191
 
192
+ describe('metric currency/scale round-trip (upsertMetrics/undoMetrics)', function() {
193
+
194
+ var UUID = 'units-roundtrip';
195
+ var docId;
196
+
197
+ // lean() so assertions see the raw persisted fields (subdoc getters would otherwise
198
+ // materialize an empty `previous` even after $unset)
199
+ var getMetric = async function(name, scenario) {
200
+ var doc = await Financials.findById(docId).lean();
201
+ var snapshot = doc.snapshots.find(s => s.uuid === UUID);
202
+ return snapshot.metrics.find(m => m.name === name && m.scenario === scenario);
203
+ };
204
+
205
+ before(async function() {
206
+ var f = await Financials.create({
207
+ organization: new mongoose.Types.ObjectId(),
208
+ customer: new mongoose.Types.ObjectId(),
209
+ snapshots: [{
210
+ uuid: UUID,
211
+ year: 2026,
212
+ period: 'FY',
213
+ metrics: [{ name: 'Revenue', scenario: 'actual', value: 100, currency: 'EUR', scale: 1000 }]
214
+ }]
215
+ });
216
+ docId = f._id;
217
+ });
218
+
219
+ it('persists currency/scale on update and captures them in previous', async function() {
220
+ await Financials.upsertMetrics(docId, UUID, [
221
+ { name: 'Revenue', scenario: 'actual', value: 200, currency: 'USD', scale: 1000000 }
222
+ ]);
223
+ var metric = await getMetric('Revenue', 'actual');
224
+ metric.value.should.equal(200);
225
+ metric.currency.should.equal('USD');
226
+ metric.scale.should.equal(1000000);
227
+ metric.previous.value.should.equal(100);
228
+ metric.previous.currency.should.equal('EUR');
229
+ metric.previous.scale.should.equal(1000);
230
+ });
231
+
232
+ it('does not clobber currency/scale when the caller omits them', async function() {
233
+ await Financials.upsertMetrics(docId, UUID, [
234
+ { name: 'Revenue', scenario: 'actual', value: 300 }
235
+ ]);
236
+ var metric = await getMetric('Revenue', 'actual');
237
+ metric.value.should.equal(300);
238
+ metric.currency.should.equal('USD');
239
+ metric.scale.should.equal(1000000);
240
+ });
241
+
242
+ it('stamps currency/scale on a newly-inserted metric', async function() {
243
+ await Financials.upsertMetrics(docId, UUID, [
244
+ { name: 'ARR', scenario: 'budget', value: 50, currency: 'GBP', scale: 1000 }
245
+ ]);
246
+ var metric = await getMetric('ARR', 'budget');
247
+ metric.currency.should.equal('GBP');
248
+ metric.scale.should.equal(1000);
249
+ });
250
+
251
+ it('restores currency/scale from previous on undo', async function() {
252
+ await Financials.upsertMetrics(docId, UUID, [
253
+ { name: 'ARR', scenario: 'budget', value: 75, currency: 'CAD', scale: 1000000 }
254
+ ]);
255
+ await Financials.undoMetrics(docId, UUID, [{ name: 'ARR', scenario: 'budget' }]);
256
+ var metric = await getMetric('ARR', 'budget');
257
+ metric.value.should.equal(50);
258
+ metric.currency.should.equal('GBP');
259
+ metric.scale.should.equal(1000);
260
+ should.not.exist(metric.previous);
261
+ });
262
+
263
+ it('unsets currency on undo when the metric had none before (unitless)', async function() {
264
+ await Financials.upsertMetrics(docId, UUID, [
265
+ { name: 'Months of Runway', scenario: 'actual', value: 12, scale: 1 }
266
+ ]);
267
+ await Financials.upsertMetrics(docId, UUID, [
268
+ { name: 'Months of Runway', scenario: 'actual', value: 9, currency: 'USD', scale: 1 }
269
+ ]);
270
+ await Financials.undoMetrics(docId, UUID, [{ name: 'Months of Runway', scenario: 'actual' }]);
271
+ var metric = await getMetric('Months of Runway', 'actual');
272
+ metric.value.should.equal(12);
273
+ should.not.exist(metric.currency);
274
+ metric.scale.should.equal(1);
275
+ });
276
+
277
+ });
278
+
167
279
  });
package/test/config.js CHANGED
@@ -13,7 +13,9 @@ module.exports = {
13
13
  suppressDebugLog: true,
14
14
 
15
15
  db: {
16
- uri: 'mongodb://gZvwpF5aLFvdc7KFa7qLLiZYv6CTLdfs:TKfdi3LD4583f8EL11k5Y0uukZ151d9n@test-shard-00-00-gv0f3.mongodb.net:27017,test-shard-00-01-gv0f3.mongodb.net:27017,test-shard-00-02-gv0f3.mongodb.net:27017/test?ssl=true&replicaSet=Test-shard-0&authSource=admin'
16
+ // The legacy Atlas test cluster (test-shard-*-gv0f3) no longer resolves; override with a
17
+ // local replica-set mongod (transactions require a replset) via TEST_DB_URI when running locally.
18
+ uri: process.env.TEST_DB_URI || 'mongodb://gZvwpF5aLFvdc7KFa7qLLiZYv6CTLdfs:TKfdi3LD4583f8EL11k5Y0uukZ151d9n@test-shard-00-00-gv0f3.mongodb.net:27017,test-shard-00-01-gv0f3.mongodb.net:27017,test-shard-00-02-gv0f3.mongodb.net:27017/test?ssl=true&replicaSet=Test-shard-0&authSource=admin'
17
19
  },
18
20
 
19
21
  crypto: {
@@ -1,108 +0,0 @@
1
- 'use strict';
2
-
3
- /**
4
- * READ-ONLY audit for the snapshot -> metric currency/scale migration.
5
- *
6
- * Reports how many snapshots and metrics are missing currency/scale, and the distribution of
7
- * values present. Run this BEFORE the backfill to see how many metrics would receive the
8
- * default ('USD' / 1) rather than a real snapshot value.
9
- *
10
- * Makes NO writes.
11
- *
12
- * Usage:
13
- * node scripts/audit-metric-currency-scale.js [<customerId>] [--config <path>]
14
- * node scripts/audit-metric-currency-scale.js --config ../totem-web/config/config.js
15
- * DB_URI="mongodb://..." node scripts/audit-metric-currency-scale.js
16
- */
17
-
18
- const mongoose = require('mongoose');
19
- const path = require('path');
20
-
21
- const args = process.argv.slice(2);
22
- const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
23
-
24
- let dbUri = null;
25
- const configIndex = args.indexOf('--config');
26
- if (configIndex !== -1 && args[configIndex + 1]) dbUri = require(path.resolve(args[configIndex + 1])).db.uri;
27
- if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
28
- if (!dbUri) { console.error('Error: --config or DB_URI required'); process.exit(1); }
29
-
30
- console.log('Audit scope:', CUSTOMER_ID ? `customer ${CUSTOMER_ID}` : 'ALL customers');
31
- console.log('Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'), '\n');
32
-
33
- mongoose.connect(dbUri);
34
- const db = mongoose.connection;
35
- db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
36
- db.once('open', async () => {
37
- try { await audit(db); process.exit(0); }
38
- catch (err) { console.error('Audit failed:', err); process.exit(1); }
39
- });
40
-
41
- // A field is "missing" if absent or null (empty string counted separately as its own bucket).
42
- const isNull = (f) => ({ $eq: [{ $ifNull: [f, null] }, null] });
43
-
44
- async function audit(db) {
45
- const Financials = db.collection('financials');
46
- const match = CUSTOMER_ID ? { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) } : {};
47
-
48
- const docs = await Financials.countDocuments(match);
49
-
50
- // ---- snapshot level ----
51
- const [snap] = await Financials.aggregate([
52
- { $match: match },
53
- { $unwind: '$snapshots' },
54
- { $group: {
55
- _id: null,
56
- total: { $sum: 1 },
57
- missingCurrency: { $sum: { $cond: [isNull('$snapshots.currency'), 1, 0] } },
58
- emptyCurrency: { $sum: { $cond: [{ $eq: ['$snapshots.currency', ''] }, 1, 0] } },
59
- missingScale: { $sum: { $cond: [isNull('$snapshots.scale'), 1, 0] } },
60
- } },
61
- ]).toArray();
62
-
63
- // ---- metric level (what the backfill actually writes) ----
64
- const [met] = await Financials.aggregate([
65
- { $match: match },
66
- { $unwind: '$snapshots' },
67
- { $unwind: '$snapshots.metrics' },
68
- { $group: {
69
- _id: null,
70
- total: { $sum: 1 },
71
- missingCurrency: { $sum: { $cond: [isNull('$snapshots.metrics.currency'), 1, 0] } },
72
- missingScale: { $sum: { $cond: [isNull('$snapshots.metrics.scale'), 1, 0] } },
73
- // metrics whose snapshot ALSO lacks currency -> these get the 'USD' default on backfill
74
- metricAndSnapshotMissingCurrency: { $sum: { $cond: [{ $and: [isNull('$snapshots.metrics.currency'), isNull('$snapshots.currency')] }, 1, 0] } },
75
- } },
76
- ]).toArray();
77
-
78
- // ---- distribution of snapshot currency/scale values present ----
79
- const curDist = await Financials.aggregate([
80
- { $match: match }, { $unwind: '$snapshots' },
81
- { $group: { _id: '$snapshots.currency', n: { $sum: 1 } } }, { $sort: { n: -1 } },
82
- ]).toArray();
83
- const scaleDist = await Financials.aggregate([
84
- { $match: match }, { $unwind: '$snapshots' },
85
- { $group: { _id: '$snapshots.scale', n: { $sum: 1 } } }, { $sort: { n: -1 } },
86
- ]).toArray();
87
-
88
- const s = snap || { total: 0, missingCurrency: 0, emptyCurrency: 0, missingScale: 0 };
89
- const m = met || { total: 0, missingCurrency: 0, missingScale: 0, metricAndSnapshotMissingCurrency: 0 };
90
- const pct = (a, b) => b ? (100 * a / b).toFixed(1) + '%' : '0%';
91
-
92
- console.log('=== Financials ===');
93
- console.log(` documents: ${docs}`);
94
- console.log('\n=== Snapshots ===');
95
- console.log(` total: ${s.total}`);
96
- console.log(` missing currency: ${s.missingCurrency} (${pct(s.missingCurrency, s.total)})`);
97
- console.log(` empty-string currency: ${s.emptyCurrency}`);
98
- console.log(` missing scale: ${s.missingScale} (${pct(s.missingScale, s.total)})`);
99
- console.log('\n=== Metrics (what the backfill writes) ===');
100
- console.log(` total: ${m.total}`);
101
- console.log(` missing currency: ${m.missingCurrency} (${pct(m.missingCurrency, m.total)}) <- will be backfilled`);
102
- console.log(` missing scale: ${m.missingScale} (${pct(m.missingScale, m.total)}) <- will be backfilled`);
103
- console.log(` missing currency AND snapshot has none -> would get 'USD' default: ${m.metricAndSnapshotMissingCurrency}`);
104
- console.log('\n=== Snapshot currency distribution ===');
105
- curDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
106
- console.log('\n=== Snapshot scale distribution ===');
107
- scaleDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
108
- }
@@ -1,193 +0,0 @@
1
- 'use strict';
2
-
3
- /**
4
- * SAFE, STANDALONE backfill: snapshot-level currency/scale -> per-metric currency/scale.
5
- *
6
- * For every metric that does not already carry its own `currency`/`scale`, copy it down from the
7
- * (now-deprecated) snapshot-level `currency`/`scale`. Also backfills the metric's `previous`
8
- * sub-state when present (so undo round-trips), and normalizes any null/absent snapshot-level
9
- * `scale` to 1 (a null scale already means "unscaled" = 1) so no null scales remain anywhere.
10
- * A missing snapshot `scale` yields metric `scale: 1`; a missing snapshot `currency` yields 'USD'
11
- * (per the prod audit this never occurs — every snapshot has a real currency).
12
- *
13
- * Why a dedicated script (not scripts/migrate-financials-schema.js): that script bundles 11 other
14
- * transformations, including a metric "convert branch" that rebuilds the whole metrics array and can
15
- * DROP already-migrated metrics in a mixed-state snapshot. This script has NO such path — it only
16
- * adds fields, never rewrites arrays or removes anything. It is:
17
- * - additive : only ever $set's currency/scale on metrics that lack them
18
- * - idempotent : re-running is a no-op
19
- * - value-safe : never touches metric.value/breakdown/etc.; no displayed number changes
20
- * (metric.currency := snapshot.currency, which read paths already fall back to today)
21
- *
22
- * Usage:
23
- * node scripts/backfill-metric-currency-scale.js <customerId> [--config <path>] [--dry-run]
24
- * node scripts/backfill-metric-currency-scale.js --all-customers --config <path> [--dry-run]
25
- * DB_URI="mongodb://..." node scripts/backfill-metric-currency-scale.js <customerId> --dry-run
26
- *
27
- * ALWAYS run with --dry-run first and inspect the output.
28
- */
29
-
30
- // Valid metric scales (must match the enum in lib/Financials.js). A corrupt legacy snapshot scale
31
- // is sanitized to 1 rather than propagated into a metric (which would fail a future validated write).
32
- const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
33
- function sanitizeScale(scale) {
34
- return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
35
- }
36
-
37
- /**
38
- * Pure function: compute the $set backfill for a single financials document.
39
- * Returns { updates: {dotPath: value}, changes: [string] }. Empty updates => nothing to do.
40
- * No DB access, so this is directly unit-testable.
41
- */
42
- function computeBackfillUpdates(financial) {
43
- const updates = {};
44
- const changes = [];
45
- if (!financial || !Array.isArray(financial.snapshots)) return { updates, changes };
46
-
47
- financial.snapshots.forEach((snapshot, i) => {
48
- if (!snapshot) return;
49
-
50
- const currency = snapshot.currency || 'USD';
51
- const scale = sanitizeScale(snapshot.scale);
52
-
53
- // Normalize a null/absent/invalid snapshot-level scale to 1 so no null scales remain anywhere
54
- // (a null scale already means "unscaled" = 1, so this is a value-safe cleanup).
55
- if (!VALID_METRIC_SCALES.includes(snapshot.scale)) {
56
- updates[`snapshots.${i}.scale`] = 1;
57
- changes.push(` - snapshot ${i}: scale ${JSON.stringify(snapshot.scale)} -> 1`);
58
- }
59
-
60
- if (!Array.isArray(snapshot.metrics)) return;
61
- snapshot.metrics.forEach((metric, j) => {
62
- if (!metric) return;
63
-
64
- if (!('currency' in metric)) {
65
- updates[`snapshots.${i}.metrics.${j}.currency`] = currency;
66
- changes.push(` - snapshot ${i} metric ${j} (${metric.name}): currency := ${currency}`);
67
- }
68
- if (!('scale' in metric)) {
69
- updates[`snapshots.${i}.metrics.${j}.scale`] = scale;
70
- changes.push(` - snapshot ${i} metric ${j} (${metric.name}): scale := ${scale}`);
71
- }
72
-
73
- // Backfill the prior-state sub-doc too, so undo restores currency/scale rather than unsetting them.
74
- const prev = metric.previous;
75
- if (prev && typeof prev === 'object') {
76
- if (!('currency' in prev)) {
77
- updates[`snapshots.${i}.metrics.${j}.previous.currency`] = currency;
78
- changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.currency := ${currency}`);
79
- }
80
- if (!('scale' in prev)) {
81
- updates[`snapshots.${i}.metrics.${j}.previous.scale`] = scale;
82
- changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.scale := ${scale}`);
83
- }
84
- }
85
- });
86
- });
87
-
88
- return { updates, changes };
89
- }
90
-
91
- module.exports = { computeBackfillUpdates, sanitizeScale };
92
-
93
- // ---------------------------------------------------------------------------
94
- // DB runner (only when executed directly, so the pure logic above stays testable)
95
- // ---------------------------------------------------------------------------
96
- if (require.main === module) {
97
- const mongoose = require('mongoose');
98
- const path = require('path');
99
-
100
- const args = process.argv.slice(2);
101
- const DRY_RUN = args.includes('--dry-run');
102
- const ALL_CUSTOMERS = args.includes('--all-customers');
103
- const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
104
-
105
- let configPath = null;
106
- let dbUri = null;
107
-
108
- const configIndex = args.indexOf('--config');
109
- if (configIndex !== -1 && args[configIndex + 1]) configPath = path.resolve(args[configIndex + 1]);
110
- const dbUriIndex = args.indexOf('--db-uri');
111
- if (dbUriIndex !== -1 && args[dbUriIndex + 1]) dbUri = args[dbUriIndex + 1];
112
- if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
113
-
114
- if (!CUSTOMER_ID && !ALL_CUSTOMERS) {
115
- console.error('Error: provide a <customerId> or --all-customers');
116
- process.exit(1);
117
- }
118
- if (configPath) {
119
- console.log(`Loading config from: ${configPath}`);
120
- dbUri = require(configPath).db.uri;
121
- }
122
- if (!dbUri) {
123
- console.error('Error: --config, --db-uri, or DB_URI is required');
124
- process.exit(1);
125
- }
126
-
127
- console.log('Backfill Configuration:');
128
- console.log(' Scope:', ALL_CUSTOMERS ? 'ALL customers' : `customer ${CUSTOMER_ID}`);
129
- console.log(' Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'));
130
- console.log(' Dry Run:', DRY_RUN);
131
- console.log('');
132
-
133
- mongoose.connect(dbUri);
134
- const db = mongoose.connection;
135
- db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
136
- db.once('open', async () => {
137
- console.log('Connected to MongoDB');
138
- try {
139
- await run(db);
140
- console.log('\nBackfill completed successfully!');
141
- process.exit(0);
142
- } catch (err) {
143
- console.error('\nBackfill failed:', err);
144
- process.exit(1);
145
- }
146
- });
147
-
148
- async function run(db) {
149
- const Financials = db.collection('financials');
150
- const query = ALL_CUSTOMERS ? {} : { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) };
151
- const financials = await Financials.find(query).toArray();
152
- console.log(`Found ${financials.length} financial document(s)\n`);
153
-
154
- // Detailed per-field logging is far too noisy at prod scale (100k+ writes), so print full
155
- // detail only for the first few changed docs as a sample (or all docs with --verbose), and
156
- // otherwise track category totals derived from the update key suffixes.
157
- const VERBOSE = process.argv.includes('--verbose');
158
- const SAMPLE_DOCS = 3;
159
- let changed = 0, unchanged = 0, fieldWrites = 0;
160
- let metricCurrency = 0, metricScale = 0, prevWrites = 0, snapshotScaleFixes = 0;
161
-
162
- for (const financial of financials) {
163
- const { updates, changes } = computeBackfillUpdates(financial);
164
- const keys = Object.keys(updates);
165
- if (keys.length === 0) { unchanged++; continue; }
166
- fieldWrites += keys.length;
167
- keys.forEach((k) => {
168
- if (/\.metrics\.\d+\.currency$/.test(k)) metricCurrency++;
169
- else if (/\.metrics\.\d+\.scale$/.test(k)) metricScale++;
170
- else if (/\.previous\./.test(k)) prevWrites++;
171
- else if (/^snapshots\.\d+\.scale$/.test(k)) snapshotScaleFixes++;
172
- });
173
-
174
- if (!DRY_RUN) await Financials.updateOne({ _id: financial._id }, { $set: updates });
175
-
176
- if (VERBOSE || changed < SAMPLE_DOCS) {
177
- console.log(`${DRY_RUN ? '[DRY RUN] would update' : 'updated'} ${financial._id} (${keys.length} writes)${(!VERBOSE && changed === SAMPLE_DOCS - 1) ? ' [sample; further per-doc detail suppressed]' : ''}`);
178
- changes.forEach((c) => console.log(c));
179
- console.log('');
180
- }
181
- changed++;
182
- }
183
-
184
- console.log('=== Backfill Summary ===');
185
- console.log(`Documents ${DRY_RUN ? 'that would change' : 'changed'}: ${changed} / ${financials.length} (already complete: ${unchanged})`);
186
- console.log(`Total field writes: ${fieldWrites}`);
187
- console.log(` metric currency: ${metricCurrency}`);
188
- console.log(` metric scale: ${metricScale}`);
189
- console.log(` metric.previous: ${prevWrites}`);
190
- console.log(` snapshot null-scale->1: ${snapshotScaleFixes}`);
191
- if (DRY_RUN) console.log('\nThis was a dry run. No changes were made.');
192
- }
193
- }