@dhyasama/totem-models 12.27.0 → 12.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/Financials.js +27 -0
- package/package.json +1 -1
- package/scripts/migrate-financials-schema.js +11 -26
- package/test/Financials.js +93 -2
- package/test/config.js +3 -1
- package/scripts/audit-metric-currency-scale.js +0 -108
- package/scripts/backfill-metric-currency-scale.js +0 -193
- package/scripts/libs/financial_metric_units.js +0 -21
- package/test/FinancialMetricUnits.js +0 -32
package/lib/Financials.js
CHANGED
|
@@ -670,6 +670,29 @@ module.exports = function(mongoose, config) {
|
|
|
670
670
|
|
|
671
671
|
};
|
|
672
672
|
|
|
673
|
+
// Atomic find-or-create for the one Financials doc per (customer, organization).
|
|
674
|
+
// Callers must use this instead of getByOrg + new Financials() — that pattern races
|
|
675
|
+
// under concurrent requests and used to mint duplicate documents.
|
|
676
|
+
Financials.statics.getOrCreateByOrg = function getOrCreateByOrg(customerId, orgId, cb) {
|
|
677
|
+
|
|
678
|
+
const run = async () => {
|
|
679
|
+
const result = await this.findOneAndUpdate(
|
|
680
|
+
{ 'customer': customerId, 'organization': orgId },
|
|
681
|
+
{ $setOnInsert: { customer: customerId, organization: orgId, snapshots: [] } },
|
|
682
|
+
{ upsert: true, new: true }
|
|
683
|
+
).populate({
|
|
684
|
+
path: 'snapshots.documents.document',
|
|
685
|
+
match: { customer: customerId }
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
return result;
|
|
689
|
+
};
|
|
690
|
+
|
|
691
|
+
if (typeof cb === 'function') { run().then(result => cb(null, result), err => cb(err)); return; }
|
|
692
|
+
else { return run(); }
|
|
693
|
+
|
|
694
|
+
};
|
|
695
|
+
|
|
673
696
|
Financials.statics.getByUUID = function getByUUID(uuid, cb) {
|
|
674
697
|
|
|
675
698
|
const run = async () => {
|
|
@@ -1354,6 +1377,10 @@ module.exports = function(mongoose, config) {
|
|
|
1354
1377
|
Financials.set('toJSON', { virtuals: true });
|
|
1355
1378
|
Financials.set('autoIndex', false);
|
|
1356
1379
|
|
|
1380
|
+
// One Financials document per org per customer. autoIndex is off, so this is
|
|
1381
|
+
// documentation only — the index exists on production (customer_1_organization_1_unique).
|
|
1382
|
+
Financials.index({ customer: 1, organization: 1 }, { unique: true });
|
|
1383
|
+
|
|
1357
1384
|
Financials.on('index', function(err) { console.log('error building financials indexes: ' + err); });
|
|
1358
1385
|
|
|
1359
1386
|
mongoose.model('Financials', Financials);
|
package/package.json
CHANGED
|
@@ -42,7 +42,13 @@
|
|
|
42
42
|
|
|
43
43
|
const mongoose = require('mongoose');
|
|
44
44
|
const path = require('path');
|
|
45
|
-
|
|
45
|
+
// Legacy snapshot-level units are copied down to each metric verbatim; format-aware unit rules
|
|
46
|
+
// (currency on money metrics only) are applied afterwards by the registry-driven reconcile
|
|
47
|
+
// (totem-services/document-intelligence/scripts/reconcile_metric_units.js). No unit rule here is
|
|
48
|
+
// keyed on a metric's name.
|
|
49
|
+
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
50
|
+
const sanitizeScale = (scale) => VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
51
|
+
const getMetricUnits = (snapshot) => ({ currency: snapshot.currency || 'USD', scale: sanitizeScale(snapshot.scale) });
|
|
46
52
|
|
|
47
53
|
// Parse command line arguments
|
|
48
54
|
const args = process.argv.slice(2);
|
|
@@ -319,7 +325,7 @@ async function migrateFinancials() {
|
|
|
319
325
|
const source = getSourceForMetric(metric.name);
|
|
320
326
|
|
|
321
327
|
// Create separate metric entries for each scenario that has a value.
|
|
322
|
-
const metricUnits = getMetricUnits(
|
|
328
|
+
const metricUnits = getMetricUnits(snapshot);
|
|
323
329
|
|
|
324
330
|
if ('actual' in metric && metric.actual !== undefined && metric.actual !== null) {
|
|
325
331
|
const newMetric = {
|
|
@@ -388,30 +394,9 @@ async function migrateFinancials() {
|
|
|
388
394
|
changes.push(` - Snapshot ${index}: Added sources to metric ${mIndex} (${metric.name})`);
|
|
389
395
|
}
|
|
390
396
|
}
|
|
391
|
-
const metricUnits = getMetricUnits(
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
updates.$unset = updates.$unset || {};
|
|
395
|
-
updates.$unset[`snapshots.${index}.metrics.${mIndex}.currency`] = '';
|
|
396
|
-
changes.push(` - Snapshot ${index}: Removed currency from unitless metric ${mIndex} (${metric.name})`);
|
|
397
|
-
}
|
|
398
|
-
if (metric.scale !== 1) {
|
|
399
|
-
updates[`snapshots.${index}.metrics.${mIndex}.scale`] = 1;
|
|
400
|
-
changes.push(` - Snapshot ${index}: Set unitless scale (1) on metric ${mIndex} (${metric.name})`);
|
|
401
|
-
}
|
|
402
|
-
if (metric.previous && typeof metric.previous === 'object') {
|
|
403
|
-
if ('currency' in metric.previous) {
|
|
404
|
-
updates.$unset = updates.$unset || {};
|
|
405
|
-
updates.$unset[`snapshots.${index}.metrics.${mIndex}.previous.currency`] = '';
|
|
406
|
-
changes.push(` - Snapshot ${index}: Removed currency from previous state for unitless metric ${mIndex} (${metric.name})`);
|
|
407
|
-
}
|
|
408
|
-
if (metric.previous.scale !== 1) {
|
|
409
|
-
updates[`snapshots.${index}.metrics.${mIndex}.previous.scale`] = 1;
|
|
410
|
-
changes.push(` - Snapshot ${index}: Set unitless scale (1) on previous state for metric ${mIndex} (${metric.name})`);
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
} else {
|
|
414
|
-
// Idempotent: only set when the money metric doesn't already carry its own units.
|
|
397
|
+
const metricUnits = getMetricUnits(snapshot);
|
|
398
|
+
{
|
|
399
|
+
// Idempotent: only set when the metric doesn't already carry its own units.
|
|
415
400
|
if (!('currency' in metric)) {
|
|
416
401
|
updates[`snapshots.${index}.metrics.${mIndex}.currency`] = metricUnits.currency;
|
|
417
402
|
changes.push(` - Snapshot ${index}: Backfilled currency (${metricUnits.currency}) to metric ${mIndex} (${metric.name})`);
|
package/test/Financials.js
CHANGED
|
@@ -5,8 +5,12 @@ var
|
|
|
5
5
|
should = require("should"),
|
|
6
6
|
config = require('./config')['test'],
|
|
7
7
|
mongoose = require('mongoose'),
|
|
8
|
-
databaseHelpers = require('./databaseHelpers')
|
|
9
|
-
|
|
8
|
+
databaseHelpers = require('./databaseHelpers');
|
|
9
|
+
|
|
10
|
+
// Allow this file to run standalone (models are otherwise registered by earlier test files)
|
|
11
|
+
if (!mongoose.models.Financials) { require('../index')(mongoose, config); }
|
|
12
|
+
|
|
13
|
+
var Financials = mongoose.model('Financials');
|
|
10
14
|
|
|
11
15
|
var customerA = new mongoose.Types.ObjectId();
|
|
12
16
|
var customerB = new mongoose.Types.ObjectId();
|
|
@@ -185,4 +189,91 @@ describe('Financials', function() {
|
|
|
185
189
|
|
|
186
190
|
});
|
|
187
191
|
|
|
192
|
+
describe('metric currency/scale round-trip (upsertMetrics/undoMetrics)', function() {
|
|
193
|
+
|
|
194
|
+
var UUID = 'units-roundtrip';
|
|
195
|
+
var docId;
|
|
196
|
+
|
|
197
|
+
// lean() so assertions see the raw persisted fields (subdoc getters would otherwise
|
|
198
|
+
// materialize an empty `previous` even after $unset)
|
|
199
|
+
var getMetric = async function(name, scenario) {
|
|
200
|
+
var doc = await Financials.findById(docId).lean();
|
|
201
|
+
var snapshot = doc.snapshots.find(s => s.uuid === UUID);
|
|
202
|
+
return snapshot.metrics.find(m => m.name === name && m.scenario === scenario);
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
before(async function() {
|
|
206
|
+
var f = await Financials.create({
|
|
207
|
+
organization: new mongoose.Types.ObjectId(),
|
|
208
|
+
customer: new mongoose.Types.ObjectId(),
|
|
209
|
+
snapshots: [{
|
|
210
|
+
uuid: UUID,
|
|
211
|
+
year: 2026,
|
|
212
|
+
period: 'FY',
|
|
213
|
+
metrics: [{ name: 'Revenue', scenario: 'actual', value: 100, currency: 'EUR', scale: 1000 }]
|
|
214
|
+
}]
|
|
215
|
+
});
|
|
216
|
+
docId = f._id;
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it('persists currency/scale on update and captures them in previous', async function() {
|
|
220
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
221
|
+
{ name: 'Revenue', scenario: 'actual', value: 200, currency: 'USD', scale: 1000000 }
|
|
222
|
+
]);
|
|
223
|
+
var metric = await getMetric('Revenue', 'actual');
|
|
224
|
+
metric.value.should.equal(200);
|
|
225
|
+
metric.currency.should.equal('USD');
|
|
226
|
+
metric.scale.should.equal(1000000);
|
|
227
|
+
metric.previous.value.should.equal(100);
|
|
228
|
+
metric.previous.currency.should.equal('EUR');
|
|
229
|
+
metric.previous.scale.should.equal(1000);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it('does not clobber currency/scale when the caller omits them', async function() {
|
|
233
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
234
|
+
{ name: 'Revenue', scenario: 'actual', value: 300 }
|
|
235
|
+
]);
|
|
236
|
+
var metric = await getMetric('Revenue', 'actual');
|
|
237
|
+
metric.value.should.equal(300);
|
|
238
|
+
metric.currency.should.equal('USD');
|
|
239
|
+
metric.scale.should.equal(1000000);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it('stamps currency/scale on a newly-inserted metric', async function() {
|
|
243
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
244
|
+
{ name: 'ARR', scenario: 'budget', value: 50, currency: 'GBP', scale: 1000 }
|
|
245
|
+
]);
|
|
246
|
+
var metric = await getMetric('ARR', 'budget');
|
|
247
|
+
metric.currency.should.equal('GBP');
|
|
248
|
+
metric.scale.should.equal(1000);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it('restores currency/scale from previous on undo', async function() {
|
|
252
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
253
|
+
{ name: 'ARR', scenario: 'budget', value: 75, currency: 'CAD', scale: 1000000 }
|
|
254
|
+
]);
|
|
255
|
+
await Financials.undoMetrics(docId, UUID, [{ name: 'ARR', scenario: 'budget' }]);
|
|
256
|
+
var metric = await getMetric('ARR', 'budget');
|
|
257
|
+
metric.value.should.equal(50);
|
|
258
|
+
metric.currency.should.equal('GBP');
|
|
259
|
+
metric.scale.should.equal(1000);
|
|
260
|
+
should.not.exist(metric.previous);
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it('unsets currency on undo when the metric had none before (unitless)', async function() {
|
|
264
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
265
|
+
{ name: 'Months of Runway', scenario: 'actual', value: 12, scale: 1 }
|
|
266
|
+
]);
|
|
267
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
268
|
+
{ name: 'Months of Runway', scenario: 'actual', value: 9, currency: 'USD', scale: 1 }
|
|
269
|
+
]);
|
|
270
|
+
await Financials.undoMetrics(docId, UUID, [{ name: 'Months of Runway', scenario: 'actual' }]);
|
|
271
|
+
var metric = await getMetric('Months of Runway', 'actual');
|
|
272
|
+
metric.value.should.equal(12);
|
|
273
|
+
should.not.exist(metric.currency);
|
|
274
|
+
metric.scale.should.equal(1);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
});
|
|
278
|
+
|
|
188
279
|
});
|
package/test/config.js
CHANGED
|
@@ -13,7 +13,9 @@ module.exports = {
|
|
|
13
13
|
suppressDebugLog: true,
|
|
14
14
|
|
|
15
15
|
db: {
|
|
16
|
-
|
|
16
|
+
// The legacy Atlas test cluster (test-shard-*-gv0f3) no longer resolves; override with a
|
|
17
|
+
// local replica-set mongod (transactions require a replset) via TEST_DB_URI when running locally.
|
|
18
|
+
uri: process.env.TEST_DB_URI || 'mongodb://gZvwpF5aLFvdc7KFa7qLLiZYv6CTLdfs:TKfdi3LD4583f8EL11k5Y0uukZ151d9n@test-shard-00-00-gv0f3.mongodb.net:27017,test-shard-00-01-gv0f3.mongodb.net:27017,test-shard-00-02-gv0f3.mongodb.net:27017/test?ssl=true&replicaSet=Test-shard-0&authSource=admin'
|
|
17
19
|
},
|
|
18
20
|
|
|
19
21
|
crypto: {
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* READ-ONLY audit for the snapshot -> metric currency/scale migration.
|
|
5
|
-
*
|
|
6
|
-
* Reports how many snapshots and metrics are missing currency/scale, and the distribution of
|
|
7
|
-
* values present. Run this BEFORE the backfill to see how many metrics would receive the
|
|
8
|
-
* default ('USD' / 1) rather than a real snapshot value.
|
|
9
|
-
*
|
|
10
|
-
* Makes NO writes.
|
|
11
|
-
*
|
|
12
|
-
* Usage:
|
|
13
|
-
* node scripts/audit-metric-currency-scale.js [<customerId>] [--config <path>]
|
|
14
|
-
* node scripts/audit-metric-currency-scale.js --config ../totem-web/config/config.js
|
|
15
|
-
* DB_URI="mongodb://..." node scripts/audit-metric-currency-scale.js
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
const mongoose = require('mongoose');
|
|
19
|
-
const path = require('path');
|
|
20
|
-
|
|
21
|
-
const args = process.argv.slice(2);
|
|
22
|
-
const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
|
|
23
|
-
|
|
24
|
-
let dbUri = null;
|
|
25
|
-
const configIndex = args.indexOf('--config');
|
|
26
|
-
if (configIndex !== -1 && args[configIndex + 1]) dbUri = require(path.resolve(args[configIndex + 1])).db.uri;
|
|
27
|
-
if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
|
|
28
|
-
if (!dbUri) { console.error('Error: --config or DB_URI required'); process.exit(1); }
|
|
29
|
-
|
|
30
|
-
console.log('Audit scope:', CUSTOMER_ID ? `customer ${CUSTOMER_ID}` : 'ALL customers');
|
|
31
|
-
console.log('Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'), '\n');
|
|
32
|
-
|
|
33
|
-
mongoose.connect(dbUri);
|
|
34
|
-
const db = mongoose.connection;
|
|
35
|
-
db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
|
|
36
|
-
db.once('open', async () => {
|
|
37
|
-
try { await audit(db); process.exit(0); }
|
|
38
|
-
catch (err) { console.error('Audit failed:', err); process.exit(1); }
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
// A field is "missing" if absent or null (empty string counted separately as its own bucket).
|
|
42
|
-
const isNull = (f) => ({ $eq: [{ $ifNull: [f, null] }, null] });
|
|
43
|
-
|
|
44
|
-
async function audit(db) {
|
|
45
|
-
const Financials = db.collection('financials');
|
|
46
|
-
const match = CUSTOMER_ID ? { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) } : {};
|
|
47
|
-
|
|
48
|
-
const docs = await Financials.countDocuments(match);
|
|
49
|
-
|
|
50
|
-
// ---- snapshot level ----
|
|
51
|
-
const [snap] = await Financials.aggregate([
|
|
52
|
-
{ $match: match },
|
|
53
|
-
{ $unwind: '$snapshots' },
|
|
54
|
-
{ $group: {
|
|
55
|
-
_id: null,
|
|
56
|
-
total: { $sum: 1 },
|
|
57
|
-
missingCurrency: { $sum: { $cond: [isNull('$snapshots.currency'), 1, 0] } },
|
|
58
|
-
emptyCurrency: { $sum: { $cond: [{ $eq: ['$snapshots.currency', ''] }, 1, 0] } },
|
|
59
|
-
missingScale: { $sum: { $cond: [isNull('$snapshots.scale'), 1, 0] } },
|
|
60
|
-
} },
|
|
61
|
-
]).toArray();
|
|
62
|
-
|
|
63
|
-
// ---- metric level (what the backfill actually writes) ----
|
|
64
|
-
const [met] = await Financials.aggregate([
|
|
65
|
-
{ $match: match },
|
|
66
|
-
{ $unwind: '$snapshots' },
|
|
67
|
-
{ $unwind: '$snapshots.metrics' },
|
|
68
|
-
{ $group: {
|
|
69
|
-
_id: null,
|
|
70
|
-
total: { $sum: 1 },
|
|
71
|
-
missingCurrency: { $sum: { $cond: [isNull('$snapshots.metrics.currency'), 1, 0] } },
|
|
72
|
-
missingScale: { $sum: { $cond: [isNull('$snapshots.metrics.scale'), 1, 0] } },
|
|
73
|
-
// metrics whose snapshot ALSO lacks currency -> these get the 'USD' default on backfill
|
|
74
|
-
metricAndSnapshotMissingCurrency: { $sum: { $cond: [{ $and: [isNull('$snapshots.metrics.currency'), isNull('$snapshots.currency')] }, 1, 0] } },
|
|
75
|
-
} },
|
|
76
|
-
]).toArray();
|
|
77
|
-
|
|
78
|
-
// ---- distribution of snapshot currency/scale values present ----
|
|
79
|
-
const curDist = await Financials.aggregate([
|
|
80
|
-
{ $match: match }, { $unwind: '$snapshots' },
|
|
81
|
-
{ $group: { _id: '$snapshots.currency', n: { $sum: 1 } } }, { $sort: { n: -1 } },
|
|
82
|
-
]).toArray();
|
|
83
|
-
const scaleDist = await Financials.aggregate([
|
|
84
|
-
{ $match: match }, { $unwind: '$snapshots' },
|
|
85
|
-
{ $group: { _id: '$snapshots.scale', n: { $sum: 1 } } }, { $sort: { n: -1 } },
|
|
86
|
-
]).toArray();
|
|
87
|
-
|
|
88
|
-
const s = snap || { total: 0, missingCurrency: 0, emptyCurrency: 0, missingScale: 0 };
|
|
89
|
-
const m = met || { total: 0, missingCurrency: 0, missingScale: 0, metricAndSnapshotMissingCurrency: 0 };
|
|
90
|
-
const pct = (a, b) => b ? (100 * a / b).toFixed(1) + '%' : '0%';
|
|
91
|
-
|
|
92
|
-
console.log('=== Financials ===');
|
|
93
|
-
console.log(` documents: ${docs}`);
|
|
94
|
-
console.log('\n=== Snapshots ===');
|
|
95
|
-
console.log(` total: ${s.total}`);
|
|
96
|
-
console.log(` missing currency: ${s.missingCurrency} (${pct(s.missingCurrency, s.total)})`);
|
|
97
|
-
console.log(` empty-string currency: ${s.emptyCurrency}`);
|
|
98
|
-
console.log(` missing scale: ${s.missingScale} (${pct(s.missingScale, s.total)})`);
|
|
99
|
-
console.log('\n=== Metrics (what the backfill writes) ===');
|
|
100
|
-
console.log(` total: ${m.total}`);
|
|
101
|
-
console.log(` missing currency: ${m.missingCurrency} (${pct(m.missingCurrency, m.total)}) <- will be backfilled`);
|
|
102
|
-
console.log(` missing scale: ${m.missingScale} (${pct(m.missingScale, m.total)}) <- will be backfilled`);
|
|
103
|
-
console.log(` missing currency AND snapshot has none -> would get 'USD' default: ${m.metricAndSnapshotMissingCurrency}`);
|
|
104
|
-
console.log('\n=== Snapshot currency distribution ===');
|
|
105
|
-
curDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
|
|
106
|
-
console.log('\n=== Snapshot scale distribution ===');
|
|
107
|
-
scaleDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
|
|
108
|
-
}
|
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* SAFE, STANDALONE backfill: snapshot-level currency/scale -> per-metric currency/scale.
|
|
5
|
-
*
|
|
6
|
-
* For every metric that does not already carry its own `currency`/`scale`, copy it down from the
|
|
7
|
-
* (now-deprecated) snapshot-level `currency`/`scale`. Also backfills the metric's `previous`
|
|
8
|
-
* sub-state when present (so undo round-trips), and normalizes any null/absent snapshot-level
|
|
9
|
-
* `scale` to 1 (a null scale already means "unscaled" = 1) so no null scales remain anywhere.
|
|
10
|
-
* A missing snapshot `scale` yields metric `scale: 1`; a missing snapshot `currency` yields 'USD'
|
|
11
|
-
* (per the prod audit this never occurs — every snapshot has a real currency).
|
|
12
|
-
*
|
|
13
|
-
* Why a dedicated script (not scripts/migrate-financials-schema.js): that script bundles 11 other
|
|
14
|
-
* transformations, including a metric "convert branch" that rebuilds the whole metrics array and can
|
|
15
|
-
* DROP already-migrated metrics in a mixed-state snapshot. This script has NO such path — it only
|
|
16
|
-
* adds fields, never rewrites arrays or removes anything. It is:
|
|
17
|
-
* - additive : only ever $set's currency/scale on metrics that lack them
|
|
18
|
-
* - idempotent : re-running is a no-op
|
|
19
|
-
* - value-safe : never touches metric.value/breakdown/etc.; no displayed number changes
|
|
20
|
-
* (metric.currency := snapshot.currency, which read paths already fall back to today)
|
|
21
|
-
*
|
|
22
|
-
* Usage:
|
|
23
|
-
* node scripts/backfill-metric-currency-scale.js <customerId> [--config <path>] [--dry-run]
|
|
24
|
-
* node scripts/backfill-metric-currency-scale.js --all-customers --config <path> [--dry-run]
|
|
25
|
-
* DB_URI="mongodb://..." node scripts/backfill-metric-currency-scale.js <customerId> --dry-run
|
|
26
|
-
*
|
|
27
|
-
* ALWAYS run with --dry-run first and inspect the output.
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
// Valid metric scales (must match the enum in lib/Financials.js). A corrupt legacy snapshot scale
|
|
31
|
-
// is sanitized to 1 rather than propagated into a metric (which would fail a future validated write).
|
|
32
|
-
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
33
|
-
function sanitizeScale(scale) {
|
|
34
|
-
return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Pure function: compute the $set backfill for a single financials document.
|
|
39
|
-
* Returns { updates: {dotPath: value}, changes: [string] }. Empty updates => nothing to do.
|
|
40
|
-
* No DB access, so this is directly unit-testable.
|
|
41
|
-
*/
|
|
42
|
-
function computeBackfillUpdates(financial) {
|
|
43
|
-
const updates = {};
|
|
44
|
-
const changes = [];
|
|
45
|
-
if (!financial || !Array.isArray(financial.snapshots)) return { updates, changes };
|
|
46
|
-
|
|
47
|
-
financial.snapshots.forEach((snapshot, i) => {
|
|
48
|
-
if (!snapshot) return;
|
|
49
|
-
|
|
50
|
-
const currency = snapshot.currency || 'USD';
|
|
51
|
-
const scale = sanitizeScale(snapshot.scale);
|
|
52
|
-
|
|
53
|
-
// Normalize a null/absent/invalid snapshot-level scale to 1 so no null scales remain anywhere
|
|
54
|
-
// (a null scale already means "unscaled" = 1, so this is a value-safe cleanup).
|
|
55
|
-
if (!VALID_METRIC_SCALES.includes(snapshot.scale)) {
|
|
56
|
-
updates[`snapshots.${i}.scale`] = 1;
|
|
57
|
-
changes.push(` - snapshot ${i}: scale ${JSON.stringify(snapshot.scale)} -> 1`);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (!Array.isArray(snapshot.metrics)) return;
|
|
61
|
-
snapshot.metrics.forEach((metric, j) => {
|
|
62
|
-
if (!metric) return;
|
|
63
|
-
|
|
64
|
-
if (!('currency' in metric)) {
|
|
65
|
-
updates[`snapshots.${i}.metrics.${j}.currency`] = currency;
|
|
66
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): currency := ${currency}`);
|
|
67
|
-
}
|
|
68
|
-
if (!('scale' in metric)) {
|
|
69
|
-
updates[`snapshots.${i}.metrics.${j}.scale`] = scale;
|
|
70
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): scale := ${scale}`);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// Backfill the prior-state sub-doc too, so undo restores currency/scale rather than unsetting them.
|
|
74
|
-
const prev = metric.previous;
|
|
75
|
-
if (prev && typeof prev === 'object') {
|
|
76
|
-
if (!('currency' in prev)) {
|
|
77
|
-
updates[`snapshots.${i}.metrics.${j}.previous.currency`] = currency;
|
|
78
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.currency := ${currency}`);
|
|
79
|
-
}
|
|
80
|
-
if (!('scale' in prev)) {
|
|
81
|
-
updates[`snapshots.${i}.metrics.${j}.previous.scale`] = scale;
|
|
82
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.scale := ${scale}`);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
return { updates, changes };
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
module.exports = { computeBackfillUpdates, sanitizeScale };
|
|
92
|
-
|
|
93
|
-
// ---------------------------------------------------------------------------
|
|
94
|
-
// DB runner (only when executed directly, so the pure logic above stays testable)
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
|
-
if (require.main === module) {
|
|
97
|
-
const mongoose = require('mongoose');
|
|
98
|
-
const path = require('path');
|
|
99
|
-
|
|
100
|
-
const args = process.argv.slice(2);
|
|
101
|
-
const DRY_RUN = args.includes('--dry-run');
|
|
102
|
-
const ALL_CUSTOMERS = args.includes('--all-customers');
|
|
103
|
-
const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
|
|
104
|
-
|
|
105
|
-
let configPath = null;
|
|
106
|
-
let dbUri = null;
|
|
107
|
-
|
|
108
|
-
const configIndex = args.indexOf('--config');
|
|
109
|
-
if (configIndex !== -1 && args[configIndex + 1]) configPath = path.resolve(args[configIndex + 1]);
|
|
110
|
-
const dbUriIndex = args.indexOf('--db-uri');
|
|
111
|
-
if (dbUriIndex !== -1 && args[dbUriIndex + 1]) dbUri = args[dbUriIndex + 1];
|
|
112
|
-
if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
|
|
113
|
-
|
|
114
|
-
if (!CUSTOMER_ID && !ALL_CUSTOMERS) {
|
|
115
|
-
console.error('Error: provide a <customerId> or --all-customers');
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
|
118
|
-
if (configPath) {
|
|
119
|
-
console.log(`Loading config from: ${configPath}`);
|
|
120
|
-
dbUri = require(configPath).db.uri;
|
|
121
|
-
}
|
|
122
|
-
if (!dbUri) {
|
|
123
|
-
console.error('Error: --config, --db-uri, or DB_URI is required');
|
|
124
|
-
process.exit(1);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
console.log('Backfill Configuration:');
|
|
128
|
-
console.log(' Scope:', ALL_CUSTOMERS ? 'ALL customers' : `customer ${CUSTOMER_ID}`);
|
|
129
|
-
console.log(' Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'));
|
|
130
|
-
console.log(' Dry Run:', DRY_RUN);
|
|
131
|
-
console.log('');
|
|
132
|
-
|
|
133
|
-
mongoose.connect(dbUri);
|
|
134
|
-
const db = mongoose.connection;
|
|
135
|
-
db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
|
|
136
|
-
db.once('open', async () => {
|
|
137
|
-
console.log('Connected to MongoDB');
|
|
138
|
-
try {
|
|
139
|
-
await run(db);
|
|
140
|
-
console.log('\nBackfill completed successfully!');
|
|
141
|
-
process.exit(0);
|
|
142
|
-
} catch (err) {
|
|
143
|
-
console.error('\nBackfill failed:', err);
|
|
144
|
-
process.exit(1);
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
async function run(db) {
|
|
149
|
-
const Financials = db.collection('financials');
|
|
150
|
-
const query = ALL_CUSTOMERS ? {} : { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) };
|
|
151
|
-
const financials = await Financials.find(query).toArray();
|
|
152
|
-
console.log(`Found ${financials.length} financial document(s)\n`);
|
|
153
|
-
|
|
154
|
-
// Detailed per-field logging is far too noisy at prod scale (100k+ writes), so print full
|
|
155
|
-
// detail only for the first few changed docs as a sample (or all docs with --verbose), and
|
|
156
|
-
// otherwise track category totals derived from the update key suffixes.
|
|
157
|
-
const VERBOSE = process.argv.includes('--verbose');
|
|
158
|
-
const SAMPLE_DOCS = 3;
|
|
159
|
-
let changed = 0, unchanged = 0, fieldWrites = 0;
|
|
160
|
-
let metricCurrency = 0, metricScale = 0, prevWrites = 0, snapshotScaleFixes = 0;
|
|
161
|
-
|
|
162
|
-
for (const financial of financials) {
|
|
163
|
-
const { updates, changes } = computeBackfillUpdates(financial);
|
|
164
|
-
const keys = Object.keys(updates);
|
|
165
|
-
if (keys.length === 0) { unchanged++; continue; }
|
|
166
|
-
fieldWrites += keys.length;
|
|
167
|
-
keys.forEach((k) => {
|
|
168
|
-
if (/\.metrics\.\d+\.currency$/.test(k)) metricCurrency++;
|
|
169
|
-
else if (/\.metrics\.\d+\.scale$/.test(k)) metricScale++;
|
|
170
|
-
else if (/\.previous\./.test(k)) prevWrites++;
|
|
171
|
-
else if (/^snapshots\.\d+\.scale$/.test(k)) snapshotScaleFixes++;
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
if (!DRY_RUN) await Financials.updateOne({ _id: financial._id }, { $set: updates });
|
|
175
|
-
|
|
176
|
-
if (VERBOSE || changed < SAMPLE_DOCS) {
|
|
177
|
-
console.log(`${DRY_RUN ? '[DRY RUN] would update' : 'updated'} ${financial._id} (${keys.length} writes)${(!VERBOSE && changed === SAMPLE_DOCS - 1) ? ' [sample; further per-doc detail suppressed]' : ''}`);
|
|
178
|
-
changes.forEach((c) => console.log(c));
|
|
179
|
-
console.log('');
|
|
180
|
-
}
|
|
181
|
-
changed++;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
console.log('=== Backfill Summary ===');
|
|
185
|
-
console.log(`Documents ${DRY_RUN ? 'that would change' : 'changed'}: ${changed} / ${financials.length} (already complete: ${unchanged})`);
|
|
186
|
-
console.log(`Total field writes: ${fieldWrites}`);
|
|
187
|
-
console.log(` metric currency: ${metricCurrency}`);
|
|
188
|
-
console.log(` metric scale: ${metricScale}`);
|
|
189
|
-
console.log(` metric.previous: ${prevWrites}`);
|
|
190
|
-
console.log(` snapshot null-scale->1: ${snapshotScaleFixes}`);
|
|
191
|
-
if (DRY_RUN) console.log('\nThis was a dry run. No changes were made.');
|
|
192
|
-
}
|
|
193
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
4
|
-
const UNITLESS_METRIC_NAMES = new Set(['Months of Runway']);
|
|
5
|
-
|
|
6
|
-
const sanitizeScale = (scale) => VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
7
|
-
const isUnitlessMetricName = (name) => UNITLESS_METRIC_NAMES.has(name);
|
|
8
|
-
|
|
9
|
-
const getMetricUnits = (metricName, snapshot = {}) => {
|
|
10
|
-
if (isUnitlessMetricName(metricName)) {
|
|
11
|
-
return { currency: undefined, scale: 1 };
|
|
12
|
-
}
|
|
13
|
-
return { currency: snapshot.currency || 'USD', scale: sanitizeScale(snapshot.scale) };
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
module.exports = {
|
|
17
|
-
getMetricUnits,
|
|
18
|
-
isUnitlessMetricName,
|
|
19
|
-
sanitizeScale
|
|
20
|
-
};
|
|
21
|
-
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const assert = require('assert');
|
|
4
|
-
const {
|
|
5
|
-
getMetricUnits,
|
|
6
|
-
isUnitlessMetricName,
|
|
7
|
-
sanitizeScale
|
|
8
|
-
} = require('../scripts/libs/financial_metric_units');
|
|
9
|
-
|
|
10
|
-
describe('financial metric migration units', function() {
|
|
11
|
-
it('stores Months of Runway as unscaled units without currency', function() {
|
|
12
|
-
assert.strictEqual(isUnitlessMetricName('Months of Runway'), true);
|
|
13
|
-
assert.deepStrictEqual(
|
|
14
|
-
getMetricUnits('Months of Runway', { currency: 'EUR', scale: 1000 }),
|
|
15
|
-
{ currency: undefined, scale: 1 }
|
|
16
|
-
);
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
it('inherits snapshot units for money metrics', function() {
|
|
20
|
-
assert.deepStrictEqual(
|
|
21
|
-
getMetricUnits('Revenue', { currency: 'EUR', scale: 1000 }),
|
|
22
|
-
{ currency: 'EUR', scale: 1000 }
|
|
23
|
-
);
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it('sanitizes absent and invalid scales', function() {
|
|
27
|
-
assert.strictEqual(sanitizeScale(undefined), 1);
|
|
28
|
-
assert.strictEqual(sanitizeScale(123), 1);
|
|
29
|
-
assert.strictEqual(sanitizeScale(1000000), 1000000);
|
|
30
|
-
});
|
|
31
|
-
});
|
|
32
|
-
|