@dhyasama/totem-models 12.26.0 → 12.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/Financials.js
CHANGED
|
@@ -76,12 +76,6 @@ module.exports = function(mongoose, config) {
|
|
|
76
76
|
|
|
77
77
|
uuid: { type: String },
|
|
78
78
|
|
|
79
|
-
// DEPRECATED: currency/scale now live per-metric (see metrics[].currency / metrics[].scale).
|
|
80
|
-
// Retained during the dual-write transition as the fallback default for un-migrated metrics;
|
|
81
|
-
// remove once all read paths resolve currency/scale per metric.
|
|
82
|
-
currency: { type: String, default: 'USD' },
|
|
83
|
-
scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000], default: 1 },
|
|
84
|
-
|
|
85
79
|
approval:{ type: Boolean, default: false },
|
|
86
80
|
|
|
87
81
|
status: { type: String, enum: ['Scheduled', 'Delivered', 'Opened', 'Overdue', 'Skipped', 'Pending', 'Completed', 'Archived'] },
|
|
@@ -148,9 +142,7 @@ module.exports = function(mongoose, config) {
|
|
|
148
142
|
}
|
|
149
143
|
}],
|
|
150
144
|
verified: { type: Boolean, default: false },
|
|
151
|
-
//
|
|
152
|
-
// "not yet migrated" and read paths fall back to the (deprecated) snapshot-level
|
|
153
|
-
// currency/scale. See scripts/migrate-financials-schema.js for the backfill.
|
|
145
|
+
// Currency and scale live exclusively on each metric/scenario.
|
|
154
146
|
currency: { type: String, trim: true },
|
|
155
147
|
scale: { type: Number, enum: [1, 1000, 1000000, 1000000000, 1000000000000] },
|
|
156
148
|
history: [{
|
|
@@ -678,6 +670,29 @@ module.exports = function(mongoose, config) {
|
|
|
678
670
|
|
|
679
671
|
};
|
|
680
672
|
|
|
673
|
+
// Atomic find-or-create for the one Financials doc per (customer, organization).
|
|
674
|
+
// Callers must use this instead of getByOrg + new Financials() — that pattern races
|
|
675
|
+
// under concurrent requests and used to mint duplicate documents.
|
|
676
|
+
Financials.statics.getOrCreateByOrg = function getOrCreateByOrg(customerId, orgId, cb) {
|
|
677
|
+
|
|
678
|
+
const run = async () => {
|
|
679
|
+
const result = await this.findOneAndUpdate(
|
|
680
|
+
{ 'customer': customerId, 'organization': orgId },
|
|
681
|
+
{ $setOnInsert: { customer: customerId, organization: orgId, snapshots: [] } },
|
|
682
|
+
{ upsert: true, new: true }
|
|
683
|
+
).populate({
|
|
684
|
+
path: 'snapshots.documents.document',
|
|
685
|
+
match: { customer: customerId }
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
return result;
|
|
689
|
+
};
|
|
690
|
+
|
|
691
|
+
if (typeof cb === 'function') { run().then(result => cb(null, result), err => cb(err)); return; }
|
|
692
|
+
else { return run(); }
|
|
693
|
+
|
|
694
|
+
};
|
|
695
|
+
|
|
681
696
|
Financials.statics.getByUUID = function getByUUID(uuid, cb) {
|
|
682
697
|
|
|
683
698
|
const run = async () => {
|
|
@@ -1362,6 +1377,10 @@ module.exports = function(mongoose, config) {
|
|
|
1362
1377
|
Financials.set('toJSON', { virtuals: true });
|
|
1363
1378
|
Financials.set('autoIndex', false);
|
|
1364
1379
|
|
|
1380
|
+
// One Financials document per org per customer. autoIndex is off, so this is
|
|
1381
|
+
// documentation only — the index exists on production (customer_1_organization_1_unique).
|
|
1382
|
+
Financials.index({ customer: 1, organization: 1 }, { unique: true });
|
|
1383
|
+
|
|
1365
1384
|
Financials.on('index', function(err) { console.log('error building financials indexes: ' + err); });
|
|
1366
1385
|
|
|
1367
1386
|
mongoose.model('Financials', Financials);
|
package/package.json
CHANGED
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
* currency/scale. Idempotent: metrics that already carry their own are left untouched.
|
|
26
26
|
* Defaults to 'USD' / 1 when the snapshot has neither. Part of the snapshot→metric
|
|
27
27
|
* currency migration (dual-write transition; snapshot fields retained as fallback).
|
|
28
|
+
* Unitless metrics (currently Months of Runway) are normalized to scale 1 with no currency,
|
|
29
|
+
* including their optional previous/undo state.
|
|
28
30
|
*
|
|
29
31
|
* Usage:
|
|
30
32
|
* node scripts/migrate-financials-schema.js <customerId> [--config <configPath>] [--dry-run]
|
|
@@ -40,6 +42,13 @@
|
|
|
40
42
|
|
|
41
43
|
const mongoose = require('mongoose');
|
|
42
44
|
const path = require('path');
|
|
45
|
+
// Legacy snapshot-level units are copied down to each metric verbatim; format-aware unit rules
|
|
46
|
+
// (currency on money metrics only) are applied afterwards by the registry-driven reconcile
|
|
47
|
+
// (totem-services/document-intelligence/scripts/reconcile_metric_units.js). No unit rule here is
|
|
48
|
+
// keyed on a metric's name.
|
|
49
|
+
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
50
|
+
const sanitizeScale = (scale) => VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
51
|
+
const getMetricUnits = (snapshot) => ({ currency: snapshot.currency || 'USD', scale: sanitizeScale(snapshot.scale) });
|
|
43
52
|
|
|
44
53
|
// Parse command line arguments
|
|
45
54
|
const args = process.argv.slice(2);
|
|
@@ -85,14 +94,6 @@ if (configPath) {
|
|
|
85
94
|
dbUri = config.db.uri;
|
|
86
95
|
}
|
|
87
96
|
|
|
88
|
-
// Valid metric scales (must match the enum in lib/Financials.js). The backfill sanitizes the
|
|
89
|
-
// snapshot's stored scale against this set so a corrupt legacy value can't propagate into a
|
|
90
|
-
// metric and fail a future validated write. Unknown/absent values fall back to 1.
|
|
91
|
-
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
92
|
-
function sanitizeScale(scale) {
|
|
93
|
-
return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
97
|
console.log('Migration Configuration:');
|
|
97
98
|
console.log(' Customer ID:', CUSTOMER_ID);
|
|
98
99
|
console.log(' Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@')); // Mask credentials
|
|
@@ -323,10 +324,8 @@ async function migrateFinancials() {
|
|
|
323
324
|
oldMetrics.forEach((metric) => {
|
|
324
325
|
const source = getSourceForMetric(metric.name);
|
|
325
326
|
|
|
326
|
-
// Create separate metric entries for each scenario that has a value
|
|
327
|
-
|
|
328
|
-
const metricCurrency = snapshot.currency || 'USD';
|
|
329
|
-
const metricScale = sanitizeScale(snapshot.scale);
|
|
327
|
+
// Create separate metric entries for each scenario that has a value.
|
|
328
|
+
const metricUnits = getMetricUnits(snapshot);
|
|
330
329
|
|
|
331
330
|
if ('actual' in metric && metric.actual !== undefined && metric.actual !== null) {
|
|
332
331
|
const newMetric = {
|
|
@@ -334,11 +333,11 @@ async function migrateFinancials() {
|
|
|
334
333
|
scenario: 'actual',
|
|
335
334
|
value: metric.actual,
|
|
336
335
|
breakdown: metric.breakdown || {},
|
|
337
|
-
|
|
338
|
-
scale: metricScale,
|
|
336
|
+
scale: metricUnits.scale,
|
|
339
337
|
verified: true,
|
|
340
338
|
history: historyArrayForMetrics
|
|
341
339
|
};
|
|
340
|
+
if (metricUnits.currency) newMetric.currency = metricUnits.currency;
|
|
342
341
|
if (source) newMetric.sources = [source];
|
|
343
342
|
newMetrics.push(newMetric);
|
|
344
343
|
}
|
|
@@ -349,11 +348,11 @@ async function migrateFinancials() {
|
|
|
349
348
|
scenario: 'budget',
|
|
350
349
|
value: metric.budget,
|
|
351
350
|
breakdown: {},
|
|
352
|
-
|
|
353
|
-
scale: metricScale,
|
|
351
|
+
scale: metricUnits.scale,
|
|
354
352
|
verified: true,
|
|
355
353
|
history: historyArrayForMetrics
|
|
356
354
|
};
|
|
355
|
+
if (metricUnits.currency) newMetric.currency = metricUnits.currency;
|
|
357
356
|
if (source) newMetric.sources = [source];
|
|
358
357
|
newMetrics.push(newMetric);
|
|
359
358
|
}
|
|
@@ -364,11 +363,11 @@ async function migrateFinancials() {
|
|
|
364
363
|
scenario: 'forecast',
|
|
365
364
|
value: metric.forecast,
|
|
366
365
|
breakdown: {},
|
|
367
|
-
|
|
368
|
-
scale: metricScale,
|
|
366
|
+
scale: metricUnits.scale,
|
|
369
367
|
verified: true,
|
|
370
368
|
history: historyArrayForMetrics
|
|
371
369
|
};
|
|
370
|
+
if (metricUnits.currency) newMetric.currency = metricUnits.currency;
|
|
372
371
|
if (source) newMetric.sources = [source];
|
|
373
372
|
newMetrics.push(newMetric);
|
|
374
373
|
}
|
|
@@ -395,16 +394,27 @@ async function migrateFinancials() {
|
|
|
395
394
|
changes.push(` - Snapshot ${index}: Added sources to metric ${mIndex} (${metric.name})`);
|
|
396
395
|
}
|
|
397
396
|
}
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
397
|
+
const metricUnits = getMetricUnits(snapshot);
|
|
398
|
+
{
|
|
399
|
+
// Idempotent: only set when the metric doesn't already carry its own units.
|
|
400
|
+
if (!('currency' in metric)) {
|
|
401
|
+
updates[`snapshots.${index}.metrics.${mIndex}.currency`] = metricUnits.currency;
|
|
402
|
+
changes.push(` - Snapshot ${index}: Backfilled currency (${metricUnits.currency}) to metric ${mIndex} (${metric.name})`);
|
|
403
|
+
}
|
|
404
|
+
if (!('scale' in metric)) {
|
|
405
|
+
updates[`snapshots.${index}.metrics.${mIndex}.scale`] = metricUnits.scale;
|
|
406
|
+
changes.push(` - Snapshot ${index}: Backfilled scale (${metricUnits.scale}) to metric ${mIndex} (${metric.name})`);
|
|
407
|
+
}
|
|
408
|
+
if (metric.previous && typeof metric.previous === 'object') {
|
|
409
|
+
if (!('currency' in metric.previous)) {
|
|
410
|
+
updates[`snapshots.${index}.metrics.${mIndex}.previous.currency`] = metricUnits.currency;
|
|
411
|
+
changes.push(` - Snapshot ${index}: Backfilled previous currency (${metricUnits.currency}) on metric ${mIndex} (${metric.name})`);
|
|
412
|
+
}
|
|
413
|
+
if (!('scale' in metric.previous)) {
|
|
414
|
+
updates[`snapshots.${index}.metrics.${mIndex}.previous.scale`] = metricUnits.scale;
|
|
415
|
+
changes.push(` - Snapshot ${index}: Backfilled previous scale (${metricUnits.scale}) on metric ${mIndex} (${metric.name})`);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
408
418
|
}
|
|
409
419
|
});
|
|
410
420
|
}
|
package/test/Financials.js
CHANGED
|
@@ -5,8 +5,12 @@ var
|
|
|
5
5
|
should = require("should"),
|
|
6
6
|
config = require('./config')['test'],
|
|
7
7
|
mongoose = require('mongoose'),
|
|
8
|
-
databaseHelpers = require('./databaseHelpers')
|
|
9
|
-
|
|
8
|
+
databaseHelpers = require('./databaseHelpers');
|
|
9
|
+
|
|
10
|
+
// Allow this file to run standalone (models are otherwise registered by earlier test files)
|
|
11
|
+
if (!mongoose.models.Financials) { require('../index')(mongoose, config); }
|
|
12
|
+
|
|
13
|
+
var Financials = mongoose.model('Financials');
|
|
10
14
|
|
|
11
15
|
var customerA = new mongoose.Types.ObjectId();
|
|
12
16
|
var customerB = new mongoose.Types.ObjectId();
|
|
@@ -15,6 +19,27 @@ var orgB = new mongoose.Types.ObjectId();
|
|
|
15
19
|
|
|
16
20
|
describe('Financials', function() {
|
|
17
21
|
|
|
22
|
+
it('does not define or persist snapshot-level currency and scale', function() {
|
|
23
|
+
var f = new Financials({
|
|
24
|
+
organization: orgA,
|
|
25
|
+
customer: customerA,
|
|
26
|
+
snapshots: [{
|
|
27
|
+
uuid: 'metric-units-only',
|
|
28
|
+
year: 2025,
|
|
29
|
+
period: 'FY',
|
|
30
|
+
currency: 'EUR',
|
|
31
|
+
scale: 1000,
|
|
32
|
+
metrics: [{ name: 'Revenue', scenario: 'actual', value: 10, currency: 'EUR', scale: 1000 }]
|
|
33
|
+
}]
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
var snapshot = f.snapshots[0].toObject();
|
|
37
|
+
should.not.exist(snapshot.currency);
|
|
38
|
+
should.not.exist(snapshot.scale);
|
|
39
|
+
snapshot.metrics[0].currency.should.equal('EUR');
|
|
40
|
+
snapshot.metrics[0].scale.should.equal(1000);
|
|
41
|
+
});
|
|
42
|
+
|
|
18
43
|
before(function(done) {
|
|
19
44
|
databaseHelpers.connect(mongoose, done);
|
|
20
45
|
});
|
|
@@ -164,4 +189,91 @@ describe('Financials', function() {
|
|
|
164
189
|
|
|
165
190
|
});
|
|
166
191
|
|
|
192
|
+
describe('metric currency/scale round-trip (upsertMetrics/undoMetrics)', function() {
|
|
193
|
+
|
|
194
|
+
var UUID = 'units-roundtrip';
|
|
195
|
+
var docId;
|
|
196
|
+
|
|
197
|
+
// lean() so assertions see the raw persisted fields (subdoc getters would otherwise
|
|
198
|
+
// materialize an empty `previous` even after $unset)
|
|
199
|
+
var getMetric = async function(name, scenario) {
|
|
200
|
+
var doc = await Financials.findById(docId).lean();
|
|
201
|
+
var snapshot = doc.snapshots.find(s => s.uuid === UUID);
|
|
202
|
+
return snapshot.metrics.find(m => m.name === name && m.scenario === scenario);
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
before(async function() {
|
|
206
|
+
var f = await Financials.create({
|
|
207
|
+
organization: new mongoose.Types.ObjectId(),
|
|
208
|
+
customer: new mongoose.Types.ObjectId(),
|
|
209
|
+
snapshots: [{
|
|
210
|
+
uuid: UUID,
|
|
211
|
+
year: 2026,
|
|
212
|
+
period: 'FY',
|
|
213
|
+
metrics: [{ name: 'Revenue', scenario: 'actual', value: 100, currency: 'EUR', scale: 1000 }]
|
|
214
|
+
}]
|
|
215
|
+
});
|
|
216
|
+
docId = f._id;
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it('persists currency/scale on update and captures them in previous', async function() {
|
|
220
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
221
|
+
{ name: 'Revenue', scenario: 'actual', value: 200, currency: 'USD', scale: 1000000 }
|
|
222
|
+
]);
|
|
223
|
+
var metric = await getMetric('Revenue', 'actual');
|
|
224
|
+
metric.value.should.equal(200);
|
|
225
|
+
metric.currency.should.equal('USD');
|
|
226
|
+
metric.scale.should.equal(1000000);
|
|
227
|
+
metric.previous.value.should.equal(100);
|
|
228
|
+
metric.previous.currency.should.equal('EUR');
|
|
229
|
+
metric.previous.scale.should.equal(1000);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it('does not clobber currency/scale when the caller omits them', async function() {
|
|
233
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
234
|
+
{ name: 'Revenue', scenario: 'actual', value: 300 }
|
|
235
|
+
]);
|
|
236
|
+
var metric = await getMetric('Revenue', 'actual');
|
|
237
|
+
metric.value.should.equal(300);
|
|
238
|
+
metric.currency.should.equal('USD');
|
|
239
|
+
metric.scale.should.equal(1000000);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it('stamps currency/scale on a newly-inserted metric', async function() {
|
|
243
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
244
|
+
{ name: 'ARR', scenario: 'budget', value: 50, currency: 'GBP', scale: 1000 }
|
|
245
|
+
]);
|
|
246
|
+
var metric = await getMetric('ARR', 'budget');
|
|
247
|
+
metric.currency.should.equal('GBP');
|
|
248
|
+
metric.scale.should.equal(1000);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it('restores currency/scale from previous on undo', async function() {
|
|
252
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
253
|
+
{ name: 'ARR', scenario: 'budget', value: 75, currency: 'CAD', scale: 1000000 }
|
|
254
|
+
]);
|
|
255
|
+
await Financials.undoMetrics(docId, UUID, [{ name: 'ARR', scenario: 'budget' }]);
|
|
256
|
+
var metric = await getMetric('ARR', 'budget');
|
|
257
|
+
metric.value.should.equal(50);
|
|
258
|
+
metric.currency.should.equal('GBP');
|
|
259
|
+
metric.scale.should.equal(1000);
|
|
260
|
+
should.not.exist(metric.previous);
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it('unsets currency on undo when the metric had none before (unitless)', async function() {
|
|
264
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
265
|
+
{ name: 'Months of Runway', scenario: 'actual', value: 12, scale: 1 }
|
|
266
|
+
]);
|
|
267
|
+
await Financials.upsertMetrics(docId, UUID, [
|
|
268
|
+
{ name: 'Months of Runway', scenario: 'actual', value: 9, currency: 'USD', scale: 1 }
|
|
269
|
+
]);
|
|
270
|
+
await Financials.undoMetrics(docId, UUID, [{ name: 'Months of Runway', scenario: 'actual' }]);
|
|
271
|
+
var metric = await getMetric('Months of Runway', 'actual');
|
|
272
|
+
metric.value.should.equal(12);
|
|
273
|
+
should.not.exist(metric.currency);
|
|
274
|
+
metric.scale.should.equal(1);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
});
|
|
278
|
+
|
|
167
279
|
});
|
package/test/config.js
CHANGED
|
@@ -13,7 +13,9 @@ module.exports = {
|
|
|
13
13
|
suppressDebugLog: true,
|
|
14
14
|
|
|
15
15
|
db: {
|
|
16
|
-
|
|
16
|
+
// The legacy Atlas test cluster (test-shard-*-gv0f3) no longer resolves; override with a
|
|
17
|
+
// local replica-set mongod (transactions require a replset) via TEST_DB_URI when running locally.
|
|
18
|
+
uri: process.env.TEST_DB_URI || 'mongodb://gZvwpF5aLFvdc7KFa7qLLiZYv6CTLdfs:TKfdi3LD4583f8EL11k5Y0uukZ151d9n@test-shard-00-00-gv0f3.mongodb.net:27017,test-shard-00-01-gv0f3.mongodb.net:27017,test-shard-00-02-gv0f3.mongodb.net:27017/test?ssl=true&replicaSet=Test-shard-0&authSource=admin'
|
|
17
19
|
},
|
|
18
20
|
|
|
19
21
|
crypto: {
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* READ-ONLY audit for the snapshot -> metric currency/scale migration.
|
|
5
|
-
*
|
|
6
|
-
* Reports how many snapshots and metrics are missing currency/scale, and the distribution of
|
|
7
|
-
* values present. Run this BEFORE the backfill to see how many metrics would receive the
|
|
8
|
-
* default ('USD' / 1) rather than a real snapshot value.
|
|
9
|
-
*
|
|
10
|
-
* Makes NO writes.
|
|
11
|
-
*
|
|
12
|
-
* Usage:
|
|
13
|
-
* node scripts/audit-metric-currency-scale.js [<customerId>] [--config <path>]
|
|
14
|
-
* node scripts/audit-metric-currency-scale.js --config ../totem-web/config/config.js
|
|
15
|
-
* DB_URI="mongodb://..." node scripts/audit-metric-currency-scale.js
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
const mongoose = require('mongoose');
|
|
19
|
-
const path = require('path');
|
|
20
|
-
|
|
21
|
-
const args = process.argv.slice(2);
|
|
22
|
-
const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
|
|
23
|
-
|
|
24
|
-
let dbUri = null;
|
|
25
|
-
const configIndex = args.indexOf('--config');
|
|
26
|
-
if (configIndex !== -1 && args[configIndex + 1]) dbUri = require(path.resolve(args[configIndex + 1])).db.uri;
|
|
27
|
-
if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
|
|
28
|
-
if (!dbUri) { console.error('Error: --config or DB_URI required'); process.exit(1); }
|
|
29
|
-
|
|
30
|
-
console.log('Audit scope:', CUSTOMER_ID ? `customer ${CUSTOMER_ID}` : 'ALL customers');
|
|
31
|
-
console.log('Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'), '\n');
|
|
32
|
-
|
|
33
|
-
mongoose.connect(dbUri);
|
|
34
|
-
const db = mongoose.connection;
|
|
35
|
-
db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
|
|
36
|
-
db.once('open', async () => {
|
|
37
|
-
try { await audit(db); process.exit(0); }
|
|
38
|
-
catch (err) { console.error('Audit failed:', err); process.exit(1); }
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
// A field is "missing" if absent or null (empty string counted separately as its own bucket).
|
|
42
|
-
const isNull = (f) => ({ $eq: [{ $ifNull: [f, null] }, null] });
|
|
43
|
-
|
|
44
|
-
async function audit(db) {
|
|
45
|
-
const Financials = db.collection('financials');
|
|
46
|
-
const match = CUSTOMER_ID ? { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) } : {};
|
|
47
|
-
|
|
48
|
-
const docs = await Financials.countDocuments(match);
|
|
49
|
-
|
|
50
|
-
// ---- snapshot level ----
|
|
51
|
-
const [snap] = await Financials.aggregate([
|
|
52
|
-
{ $match: match },
|
|
53
|
-
{ $unwind: '$snapshots' },
|
|
54
|
-
{ $group: {
|
|
55
|
-
_id: null,
|
|
56
|
-
total: { $sum: 1 },
|
|
57
|
-
missingCurrency: { $sum: { $cond: [isNull('$snapshots.currency'), 1, 0] } },
|
|
58
|
-
emptyCurrency: { $sum: { $cond: [{ $eq: ['$snapshots.currency', ''] }, 1, 0] } },
|
|
59
|
-
missingScale: { $sum: { $cond: [isNull('$snapshots.scale'), 1, 0] } },
|
|
60
|
-
} },
|
|
61
|
-
]).toArray();
|
|
62
|
-
|
|
63
|
-
// ---- metric level (what the backfill actually writes) ----
|
|
64
|
-
const [met] = await Financials.aggregate([
|
|
65
|
-
{ $match: match },
|
|
66
|
-
{ $unwind: '$snapshots' },
|
|
67
|
-
{ $unwind: '$snapshots.metrics' },
|
|
68
|
-
{ $group: {
|
|
69
|
-
_id: null,
|
|
70
|
-
total: { $sum: 1 },
|
|
71
|
-
missingCurrency: { $sum: { $cond: [isNull('$snapshots.metrics.currency'), 1, 0] } },
|
|
72
|
-
missingScale: { $sum: { $cond: [isNull('$snapshots.metrics.scale'), 1, 0] } },
|
|
73
|
-
// metrics whose snapshot ALSO lacks currency -> these get the 'USD' default on backfill
|
|
74
|
-
metricAndSnapshotMissingCurrency: { $sum: { $cond: [{ $and: [isNull('$snapshots.metrics.currency'), isNull('$snapshots.currency')] }, 1, 0] } },
|
|
75
|
-
} },
|
|
76
|
-
]).toArray();
|
|
77
|
-
|
|
78
|
-
// ---- distribution of snapshot currency/scale values present ----
|
|
79
|
-
const curDist = await Financials.aggregate([
|
|
80
|
-
{ $match: match }, { $unwind: '$snapshots' },
|
|
81
|
-
{ $group: { _id: '$snapshots.currency', n: { $sum: 1 } } }, { $sort: { n: -1 } },
|
|
82
|
-
]).toArray();
|
|
83
|
-
const scaleDist = await Financials.aggregate([
|
|
84
|
-
{ $match: match }, { $unwind: '$snapshots' },
|
|
85
|
-
{ $group: { _id: '$snapshots.scale', n: { $sum: 1 } } }, { $sort: { n: -1 } },
|
|
86
|
-
]).toArray();
|
|
87
|
-
|
|
88
|
-
const s = snap || { total: 0, missingCurrency: 0, emptyCurrency: 0, missingScale: 0 };
|
|
89
|
-
const m = met || { total: 0, missingCurrency: 0, missingScale: 0, metricAndSnapshotMissingCurrency: 0 };
|
|
90
|
-
const pct = (a, b) => b ? (100 * a / b).toFixed(1) + '%' : '0%';
|
|
91
|
-
|
|
92
|
-
console.log('=== Financials ===');
|
|
93
|
-
console.log(` documents: ${docs}`);
|
|
94
|
-
console.log('\n=== Snapshots ===');
|
|
95
|
-
console.log(` total: ${s.total}`);
|
|
96
|
-
console.log(` missing currency: ${s.missingCurrency} (${pct(s.missingCurrency, s.total)})`);
|
|
97
|
-
console.log(` empty-string currency: ${s.emptyCurrency}`);
|
|
98
|
-
console.log(` missing scale: ${s.missingScale} (${pct(s.missingScale, s.total)})`);
|
|
99
|
-
console.log('\n=== Metrics (what the backfill writes) ===');
|
|
100
|
-
console.log(` total: ${m.total}`);
|
|
101
|
-
console.log(` missing currency: ${m.missingCurrency} (${pct(m.missingCurrency, m.total)}) <- will be backfilled`);
|
|
102
|
-
console.log(` missing scale: ${m.missingScale} (${pct(m.missingScale, m.total)}) <- will be backfilled`);
|
|
103
|
-
console.log(` missing currency AND snapshot has none -> would get 'USD' default: ${m.metricAndSnapshotMissingCurrency}`);
|
|
104
|
-
console.log('\n=== Snapshot currency distribution ===');
|
|
105
|
-
curDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
|
|
106
|
-
console.log('\n=== Snapshot scale distribution ===');
|
|
107
|
-
scaleDist.forEach((c) => console.log(` ${JSON.stringify(c._id)}: ${c.n}`));
|
|
108
|
-
}
|
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* SAFE, STANDALONE backfill: snapshot-level currency/scale -> per-metric currency/scale.
|
|
5
|
-
*
|
|
6
|
-
* For every metric that does not already carry its own `currency`/`scale`, copy it down from the
|
|
7
|
-
* (now-deprecated) snapshot-level `currency`/`scale`. Also backfills the metric's `previous`
|
|
8
|
-
* sub-state when present (so undo round-trips), and normalizes any null/absent snapshot-level
|
|
9
|
-
* `scale` to 1 (a null scale already means "unscaled" = 1) so no null scales remain anywhere.
|
|
10
|
-
* A missing snapshot `scale` yields metric `scale: 1`; a missing snapshot `currency` yields 'USD'
|
|
11
|
-
* (per the prod audit this never occurs — every snapshot has a real currency).
|
|
12
|
-
*
|
|
13
|
-
* Why a dedicated script (not scripts/migrate-financials-schema.js): that script bundles 11 other
|
|
14
|
-
* transformations, including a metric "convert branch" that rebuilds the whole metrics array and can
|
|
15
|
-
* DROP already-migrated metrics in a mixed-state snapshot. This script has NO such path — it only
|
|
16
|
-
* adds fields, never rewrites arrays or removes anything. It is:
|
|
17
|
-
* - additive : only ever $set's currency/scale on metrics that lack them
|
|
18
|
-
* - idempotent : re-running is a no-op
|
|
19
|
-
* - value-safe : never touches metric.value/breakdown/etc.; no displayed number changes
|
|
20
|
-
* (metric.currency := snapshot.currency, which read paths already fall back to today)
|
|
21
|
-
*
|
|
22
|
-
* Usage:
|
|
23
|
-
* node scripts/backfill-metric-currency-scale.js <customerId> [--config <path>] [--dry-run]
|
|
24
|
-
* node scripts/backfill-metric-currency-scale.js --all-customers --config <path> [--dry-run]
|
|
25
|
-
* DB_URI="mongodb://..." node scripts/backfill-metric-currency-scale.js <customerId> --dry-run
|
|
26
|
-
*
|
|
27
|
-
* ALWAYS run with --dry-run first and inspect the output.
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
// Valid metric scales (must match the enum in lib/Financials.js). A corrupt legacy snapshot scale
|
|
31
|
-
// is sanitized to 1 rather than propagated into a metric (which would fail a future validated write).
|
|
32
|
-
const VALID_METRIC_SCALES = [1, 1000, 1000000, 1000000000, 1000000000000];
|
|
33
|
-
function sanitizeScale(scale) {
|
|
34
|
-
return VALID_METRIC_SCALES.includes(scale) ? scale : 1;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Pure function: compute the $set backfill for a single financials document.
|
|
39
|
-
* Returns { updates: {dotPath: value}, changes: [string] }. Empty updates => nothing to do.
|
|
40
|
-
* No DB access, so this is directly unit-testable.
|
|
41
|
-
*/
|
|
42
|
-
function computeBackfillUpdates(financial) {
|
|
43
|
-
const updates = {};
|
|
44
|
-
const changes = [];
|
|
45
|
-
if (!financial || !Array.isArray(financial.snapshots)) return { updates, changes };
|
|
46
|
-
|
|
47
|
-
financial.snapshots.forEach((snapshot, i) => {
|
|
48
|
-
if (!snapshot) return;
|
|
49
|
-
|
|
50
|
-
const currency = snapshot.currency || 'USD';
|
|
51
|
-
const scale = sanitizeScale(snapshot.scale);
|
|
52
|
-
|
|
53
|
-
// Normalize a null/absent/invalid snapshot-level scale to 1 so no null scales remain anywhere
|
|
54
|
-
// (a null scale already means "unscaled" = 1, so this is a value-safe cleanup).
|
|
55
|
-
if (!VALID_METRIC_SCALES.includes(snapshot.scale)) {
|
|
56
|
-
updates[`snapshots.${i}.scale`] = 1;
|
|
57
|
-
changes.push(` - snapshot ${i}: scale ${JSON.stringify(snapshot.scale)} -> 1`);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (!Array.isArray(snapshot.metrics)) return;
|
|
61
|
-
snapshot.metrics.forEach((metric, j) => {
|
|
62
|
-
if (!metric) return;
|
|
63
|
-
|
|
64
|
-
if (!('currency' in metric)) {
|
|
65
|
-
updates[`snapshots.${i}.metrics.${j}.currency`] = currency;
|
|
66
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): currency := ${currency}`);
|
|
67
|
-
}
|
|
68
|
-
if (!('scale' in metric)) {
|
|
69
|
-
updates[`snapshots.${i}.metrics.${j}.scale`] = scale;
|
|
70
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): scale := ${scale}`);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// Backfill the prior-state sub-doc too, so undo restores currency/scale rather than unsetting them.
|
|
74
|
-
const prev = metric.previous;
|
|
75
|
-
if (prev && typeof prev === 'object') {
|
|
76
|
-
if (!('currency' in prev)) {
|
|
77
|
-
updates[`snapshots.${i}.metrics.${j}.previous.currency`] = currency;
|
|
78
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.currency := ${currency}`);
|
|
79
|
-
}
|
|
80
|
-
if (!('scale' in prev)) {
|
|
81
|
-
updates[`snapshots.${i}.metrics.${j}.previous.scale`] = scale;
|
|
82
|
-
changes.push(` - snapshot ${i} metric ${j} (${metric.name}): previous.scale := ${scale}`);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
return { updates, changes };
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
module.exports = { computeBackfillUpdates, sanitizeScale };
|
|
92
|
-
|
|
93
|
-
// ---------------------------------------------------------------------------
|
|
94
|
-
// DB runner (only when executed directly, so the pure logic above stays testable)
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
|
-
if (require.main === module) {
|
|
97
|
-
const mongoose = require('mongoose');
|
|
98
|
-
const path = require('path');
|
|
99
|
-
|
|
100
|
-
const args = process.argv.slice(2);
|
|
101
|
-
const DRY_RUN = args.includes('--dry-run');
|
|
102
|
-
const ALL_CUSTOMERS = args.includes('--all-customers');
|
|
103
|
-
const CUSTOMER_ID = (args[0] && !args[0].startsWith('--')) ? args[0] : null;
|
|
104
|
-
|
|
105
|
-
let configPath = null;
|
|
106
|
-
let dbUri = null;
|
|
107
|
-
|
|
108
|
-
const configIndex = args.indexOf('--config');
|
|
109
|
-
if (configIndex !== -1 && args[configIndex + 1]) configPath = path.resolve(args[configIndex + 1]);
|
|
110
|
-
const dbUriIndex = args.indexOf('--db-uri');
|
|
111
|
-
if (dbUriIndex !== -1 && args[dbUriIndex + 1]) dbUri = args[dbUriIndex + 1];
|
|
112
|
-
if (!dbUri && process.env.DB_URI) dbUri = process.env.DB_URI;
|
|
113
|
-
|
|
114
|
-
if (!CUSTOMER_ID && !ALL_CUSTOMERS) {
|
|
115
|
-
console.error('Error: provide a <customerId> or --all-customers');
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
|
118
|
-
if (configPath) {
|
|
119
|
-
console.log(`Loading config from: ${configPath}`);
|
|
120
|
-
dbUri = require(configPath).db.uri;
|
|
121
|
-
}
|
|
122
|
-
if (!dbUri) {
|
|
123
|
-
console.error('Error: --config, --db-uri, or DB_URI is required');
|
|
124
|
-
process.exit(1);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
console.log('Backfill Configuration:');
|
|
128
|
-
console.log(' Scope:', ALL_CUSTOMERS ? 'ALL customers' : `customer ${CUSTOMER_ID}`);
|
|
129
|
-
console.log(' Database URI:', dbUri.replace(/\/\/[^:]+:[^@]+@/, '//*****:*****@'));
|
|
130
|
-
console.log(' Dry Run:', DRY_RUN);
|
|
131
|
-
console.log('');
|
|
132
|
-
|
|
133
|
-
mongoose.connect(dbUri);
|
|
134
|
-
const db = mongoose.connection;
|
|
135
|
-
db.on('error', (err) => { console.error('MongoDB connection error:', err); process.exit(1); });
|
|
136
|
-
db.once('open', async () => {
|
|
137
|
-
console.log('Connected to MongoDB');
|
|
138
|
-
try {
|
|
139
|
-
await run(db);
|
|
140
|
-
console.log('\nBackfill completed successfully!');
|
|
141
|
-
process.exit(0);
|
|
142
|
-
} catch (err) {
|
|
143
|
-
console.error('\nBackfill failed:', err);
|
|
144
|
-
process.exit(1);
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
async function run(db) {
|
|
149
|
-
const Financials = db.collection('financials');
|
|
150
|
-
const query = ALL_CUSTOMERS ? {} : { customer: new mongoose.Types.ObjectId(CUSTOMER_ID) };
|
|
151
|
-
const financials = await Financials.find(query).toArray();
|
|
152
|
-
console.log(`Found ${financials.length} financial document(s)\n`);
|
|
153
|
-
|
|
154
|
-
// Detailed per-field logging is far too noisy at prod scale (100k+ writes), so print full
|
|
155
|
-
// detail only for the first few changed docs as a sample (or all docs with --verbose), and
|
|
156
|
-
// otherwise track category totals derived from the update key suffixes.
|
|
157
|
-
const VERBOSE = process.argv.includes('--verbose');
|
|
158
|
-
const SAMPLE_DOCS = 3;
|
|
159
|
-
let changed = 0, unchanged = 0, fieldWrites = 0;
|
|
160
|
-
let metricCurrency = 0, metricScale = 0, prevWrites = 0, snapshotScaleFixes = 0;
|
|
161
|
-
|
|
162
|
-
for (const financial of financials) {
|
|
163
|
-
const { updates, changes } = computeBackfillUpdates(financial);
|
|
164
|
-
const keys = Object.keys(updates);
|
|
165
|
-
if (keys.length === 0) { unchanged++; continue; }
|
|
166
|
-
fieldWrites += keys.length;
|
|
167
|
-
keys.forEach((k) => {
|
|
168
|
-
if (/\.metrics\.\d+\.currency$/.test(k)) metricCurrency++;
|
|
169
|
-
else if (/\.metrics\.\d+\.scale$/.test(k)) metricScale++;
|
|
170
|
-
else if (/\.previous\./.test(k)) prevWrites++;
|
|
171
|
-
else if (/^snapshots\.\d+\.scale$/.test(k)) snapshotScaleFixes++;
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
if (!DRY_RUN) await Financials.updateOne({ _id: financial._id }, { $set: updates });
|
|
175
|
-
|
|
176
|
-
if (VERBOSE || changed < SAMPLE_DOCS) {
|
|
177
|
-
console.log(`${DRY_RUN ? '[DRY RUN] would update' : 'updated'} ${financial._id} (${keys.length} writes)${(!VERBOSE && changed === SAMPLE_DOCS - 1) ? ' [sample; further per-doc detail suppressed]' : ''}`);
|
|
178
|
-
changes.forEach((c) => console.log(c));
|
|
179
|
-
console.log('');
|
|
180
|
-
}
|
|
181
|
-
changed++;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
console.log('=== Backfill Summary ===');
|
|
185
|
-
console.log(`Documents ${DRY_RUN ? 'that would change' : 'changed'}: ${changed} / ${financials.length} (already complete: ${unchanged})`);
|
|
186
|
-
console.log(`Total field writes: ${fieldWrites}`);
|
|
187
|
-
console.log(` metric currency: ${metricCurrency}`);
|
|
188
|
-
console.log(` metric scale: ${metricScale}`);
|
|
189
|
-
console.log(` metric.previous: ${prevWrites}`);
|
|
190
|
-
console.log(` snapshot null-scale->1: ${snapshotScaleFixes}`);
|
|
191
|
-
if (DRY_RUN) console.log('\nThis was a dry run. No changes were made.');
|
|
192
|
-
}
|
|
193
|
-
}
|