@rtbnext/core 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +11 -0
- package/dist/abstract/Cache.d.ts +9 -0
- package/dist/abstract/Cache.js +19 -0
- package/dist/abstract/Index.d.ts +22 -0
- package/dist/abstract/Index.js +81 -0
- package/dist/abstract/Job.d.ts +19 -0
- package/dist/abstract/Job.js +49 -0
- package/dist/abstract/Snapshot.d.ts +22 -0
- package/dist/abstract/Snapshot.js +78 -0
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +25 -0
- package/dist/bin/cron.d.ts +2 -0
- package/dist/bin/cron.js +4 -0
- package/dist/core/Config.d.ts +30 -0
- package/dist/core/Config.js +66 -0
- package/dist/core/Cron.d.ts +12 -0
- package/dist/core/Cron.js +52 -0
- package/dist/core/Fetch.d.ts +28 -0
- package/dist/core/Fetch.js +172 -0
- package/dist/core/Logger.d.ts +30 -0
- package/dist/core/Logger.js +92 -0
- package/dist/core/Queue.d.ts +37 -0
- package/dist/core/Queue.js +136 -0
- package/dist/core/Storage.d.ts +28 -0
- package/dist/core/Storage.js +166 -0
- package/dist/core/Utils.d.ts +33 -0
- package/dist/core/Utils.js +167 -0
- package/dist/interfaces/cache.d.ts +6 -0
- package/dist/interfaces/config.d.ts +21 -0
- package/dist/interfaces/cron.d.ts +3 -0
- package/dist/interfaces/fetch.d.ts +13 -0
- package/dist/interfaces/filter.d.ts +12 -0
- package/dist/interfaces/index.d.ts +30 -0
- package/dist/interfaces/job.d.ts +9 -0
- package/dist/interfaces/list.d.ts +9 -0
- package/dist/interfaces/logger.d.ts +20 -0
- package/dist/interfaces/mover.d.ts +7 -0
- package/dist/interfaces/parser.d.ts +68 -0
- package/dist/interfaces/profile.d.ts +30 -0
- package/dist/interfaces/queue.d.ts +17 -0
- package/dist/interfaces/snapshot.d.ts +16 -0
- package/dist/interfaces/stats.d.ts +45 -0
- package/dist/interfaces/storage.d.ts +16 -0
- package/dist/job/Alias.d.ts +8 -0
- package/dist/job/Alias.js +42 -0
- package/dist/job/Annual.d.ts +8 -0
- package/dist/job/Annual.js +41 -0
- package/dist/job/List.d.ts +11 -0
- package/dist/job/List.js +101 -0
- package/dist/job/Merge.d.ts +10 -0
- package/dist/job/Merge.js +59 -0
- package/dist/job/Move.d.ts +7 -0
- package/dist/job/Move.js +33 -0
- package/dist/job/Performance.d.ts +8 -0
- package/dist/job/Performance.js +27 -0
- package/dist/job/Profile.d.ts +11 -0
- package/dist/job/Profile.js +76 -0
- package/dist/job/Queue.d.ts +8 -0
- package/dist/job/Queue.js +54 -0
- package/dist/job/RTB.d.ts +12 -0
- package/dist/job/RTB.js +121 -0
- package/dist/job/Stats.d.ts +11 -0
- package/dist/job/Stats.js +46 -0
- package/dist/job/Top10.d.ts +9 -0
- package/dist/job/Top10.js +48 -0
- package/dist/job/Wiki.d.ts +9 -0
- package/dist/job/Wiki.js +40 -0
- package/dist/job/index.d.ts +26 -0
- package/dist/job/index.js +26 -0
- package/dist/lib/const.d.ts +31 -0
- package/dist/lib/const.js +74 -0
- package/dist/lib/list.d.ts +90 -0
- package/dist/lib/list.js +72 -0
- package/dist/lib/regex.d.ts +7 -0
- package/dist/lib/regex.js +7 -0
- package/dist/model/Filter.d.ts +28 -0
- package/dist/model/Filter.js +122 -0
- package/dist/model/List.d.ts +12 -0
- package/dist/model/List.js +43 -0
- package/dist/model/ListIndex.d.ts +8 -0
- package/dist/model/ListIndex.js +10 -0
- package/dist/model/Mover.d.ts +15 -0
- package/dist/model/Mover.js +74 -0
- package/dist/model/Profile.d.ts +49 -0
- package/dist/model/Profile.js +181 -0
- package/dist/model/ProfileIndex.d.ts +20 -0
- package/dist/model/ProfileIndex.js +140 -0
- package/dist/model/Stats.d.ts +56 -0
- package/dist/model/Stats.js +435 -0
- package/dist/parser/BillionairesListParser.d.ts +3 -0
- package/dist/parser/BillionairesListParser.js +2 -0
- package/dist/parser/ListParser.d.ts +7 -0
- package/dist/parser/ListParser.js +11 -0
- package/dist/parser/Parser.d.ts +43 -0
- package/dist/parser/Parser.js +146 -0
- package/dist/parser/PersonListParser.d.ts +29 -0
- package/dist/parser/PersonListParser.js +111 -0
- package/dist/parser/ProfileParser.d.ts +44 -0
- package/dist/parser/ProfileParser.js +193 -0
- package/dist/parser/RTBListParser.d.ts +15 -0
- package/dist/parser/RTBListParser.js +91 -0
- package/dist/types/annual.d.ts +7 -0
- package/dist/types/config.d.ts +35 -0
- package/dist/types/fetch.d.ts +3 -0
- package/dist/types/generic.d.ts +10 -0
- package/dist/types/job.d.ts +71 -0
- package/dist/types/list.d.ts +49 -0
- package/dist/types/parser.d.ts +7 -0
- package/dist/types/profile.d.ts +9 -0
- package/dist/types/queue.d.ts +15 -0
- package/dist/types/response.d.ts +183 -0
- package/dist/types/storage.d.ts +3 -0
- package/dist/types/wiki.d.ts +1 -0
- package/dist/utils/Annual.d.ts +7 -0
- package/dist/utils/Annual.js +99 -0
- package/dist/utils/Performance.d.ts +8 -0
- package/dist/utils/Performance.js +39 -0
- package/dist/utils/ProfileManager.d.ts +24 -0
- package/dist/utils/ProfileManager.js +60 -0
- package/dist/utils/ProfileMerger.d.ts +11 -0
- package/dist/utils/ProfileMerger.js +67 -0
- package/dist/utils/Ranking.d.ts +11 -0
- package/dist/utils/Ranking.js +77 -0
- package/dist/utils/Wiki.d.ts +11 -0
- package/dist/utils/Wiki.js +168 -0
- package/package.json +45 -0
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
import { readdirSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { log } from '../core/Logger.js';
|
|
4
|
+
import { Storage } from '../core/Storage.js';
|
|
5
|
+
import { Utils } from '../core/Utils.js';
|
|
6
|
+
import { Percentiles, StatsGroup, WealthSpread } from '../lib/const.js';
|
|
7
|
+
import { Parser } from '../parser/Parser.js';
|
|
8
|
+
export class Stats {
|
|
9
|
+
static storage = Storage.getInstance();
|
|
10
|
+
static instance;
|
|
11
|
+
constructor() {
|
|
12
|
+
this.initDB();
|
|
13
|
+
}
|
|
14
|
+
initDB() {
|
|
15
|
+
log.debug('Initializing stats storage paths');
|
|
16
|
+
StatsGroup.forEach(group => Stats.storage.ensurePath(this.resolvePath(group), true));
|
|
17
|
+
}
|
|
18
|
+
// --- helper ---
|
|
19
|
+
resolvePath(path) {
|
|
20
|
+
return join('stats', path);
|
|
21
|
+
}
|
|
22
|
+
prepStats(data) {
|
|
23
|
+
return { ...Utils.metaData(), ...data };
|
|
24
|
+
}
|
|
25
|
+
getStats(path, format) {
|
|
26
|
+
return (
|
|
27
|
+
Stats.storage[format === 'csv' ? 'readCSV' : 'readJSON'](this.resolvePath(path)) ?? (format === 'csv' ? [] : {})
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
saveStats(path, format, data) {
|
|
31
|
+
return (
|
|
32
|
+
log.catch(
|
|
33
|
+
() => Stats.storage[format === 'csv' ? 'writeCSV' : 'writeJSON'](this.resolvePath(path), data),
|
|
34
|
+
`Failed to save stats to ${path}`
|
|
35
|
+
) ?? false
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
// --- getter ---
|
|
39
|
+
getGlobalStats() {
|
|
40
|
+
return this.getStats('global.json', 'json');
|
|
41
|
+
}
|
|
42
|
+
getProfileStats() {
|
|
43
|
+
return this.getStats('profile.json', 'json');
|
|
44
|
+
}
|
|
45
|
+
getWealthStats() {
|
|
46
|
+
return this.getStats('wealth.json', 'json');
|
|
47
|
+
}
|
|
48
|
+
getScatter() {
|
|
49
|
+
return this.getStats('scatter.json', 'json');
|
|
50
|
+
}
|
|
51
|
+
getTop10() {
|
|
52
|
+
return this.getStats('top10.json', 'json');
|
|
53
|
+
}
|
|
54
|
+
getDBStats() {
|
|
55
|
+
return this.getStats('db.json', 'json');
|
|
56
|
+
}
|
|
57
|
+
getHistory() {
|
|
58
|
+
return this.getStats('history.csv', 'csv');
|
|
59
|
+
}
|
|
60
|
+
// --- grouped stats getter ---
|
|
61
|
+
getGroupedStatsIndex(group) {
|
|
62
|
+
return this.getStats(`${group}/index.json`, 'json');
|
|
63
|
+
}
|
|
64
|
+
getGroupedStatsHistory(group, key) {
|
|
65
|
+
return this.getStats(`${group}/${key}.csv`, 'csv');
|
|
66
|
+
}
|
|
67
|
+
getGroupedStats(group) {
|
|
68
|
+
const index = this.getGroupedStatsIndex(group);
|
|
69
|
+
const history = Object.fromEntries(Object.keys(index.items).map(k => [k, this.getGroupedStatsHistory(group, k)]));
|
|
70
|
+
return { index, history };
|
|
71
|
+
}
|
|
72
|
+
// --- setter ---
|
|
73
|
+
setGlobalStats(data) {
|
|
74
|
+
return this.saveStats(
|
|
75
|
+
'global.json',
|
|
76
|
+
'json',
|
|
77
|
+
this.prepStats(
|
|
78
|
+
Parser.container({
|
|
79
|
+
date: { value: data.date, type: 'date', args: ['ymd'] },
|
|
80
|
+
count: { value: data.count, type: 'number' },
|
|
81
|
+
total: { value: data.total, type: 'money' },
|
|
82
|
+
woman: { value: data.woman, type: 'number' },
|
|
83
|
+
quota: { value: data.quota, type: 'pct' },
|
|
84
|
+
today: {
|
|
85
|
+
value: Parser.container({
|
|
86
|
+
value: { value: data.today?.value, type: 'money' },
|
|
87
|
+
percent: { value: data.today?.percent, type: 'pct' }
|
|
88
|
+
}),
|
|
89
|
+
type: 'container'
|
|
90
|
+
},
|
|
91
|
+
ytd: {
|
|
92
|
+
value: Parser.container({
|
|
93
|
+
value: { value: data.ytd?.value, type: 'money' },
|
|
94
|
+
percent: { value: data.ytd?.percent, type: 'pct' }
|
|
95
|
+
}),
|
|
96
|
+
type: 'container'
|
|
97
|
+
},
|
|
98
|
+
stats: {
|
|
99
|
+
value: Parser.container({
|
|
100
|
+
profiles: { value: data.stats?.profiles, type: 'number' },
|
|
101
|
+
days: { value: data.stats?.days, type: 'number' }
|
|
102
|
+
}),
|
|
103
|
+
type: 'container'
|
|
104
|
+
}
|
|
105
|
+
})
|
|
106
|
+
)
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
setProfileStats(data) {
|
|
110
|
+
return this.saveStats(
|
|
111
|
+
'profile.json',
|
|
112
|
+
'json',
|
|
113
|
+
this.prepStats({
|
|
114
|
+
...Parser.container({
|
|
115
|
+
gender: { value: data.gender, type: 'obj', args: ['number'] },
|
|
116
|
+
maritalStatus: { value: data.maritalStatus, type: 'obj', args: ['number'] },
|
|
117
|
+
children: {
|
|
118
|
+
value: Parser.container({
|
|
119
|
+
full: { value: data.children?.full, type: 'obj', args: ['number'] },
|
|
120
|
+
short: { value: data.children?.short, type: 'obj', args: ['number'] }
|
|
121
|
+
}),
|
|
122
|
+
type: 'container'
|
|
123
|
+
},
|
|
124
|
+
selfMade: { value: data.selfMade, type: 'obj', args: ['number'] },
|
|
125
|
+
philanthropyScore: { value: data.philanthropyScore, type: 'obj', args: ['number'] }
|
|
126
|
+
}),
|
|
127
|
+
agePyramid: Object.fromEntries(
|
|
128
|
+
Object.entries(data.agePyramid).map(([gender, item]) => [
|
|
129
|
+
gender,
|
|
130
|
+
Parser.container({
|
|
131
|
+
count: { value: item.count, type: 'number' },
|
|
132
|
+
decades: { value: item.decades, type: 'obj', args: ['number'] },
|
|
133
|
+
max: { value: item.max, type: 'number' },
|
|
134
|
+
min: { value: item.min, type: 'number' },
|
|
135
|
+
mean: { value: item.mean, type: 'number' }
|
|
136
|
+
})
|
|
137
|
+
])
|
|
138
|
+
)
|
|
139
|
+
})
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
setWealthStats(data) {
|
|
143
|
+
return this.saveStats(
|
|
144
|
+
'wealth.json',
|
|
145
|
+
'json',
|
|
146
|
+
this.prepStats(
|
|
147
|
+
Parser.container({
|
|
148
|
+
percentiles: { value: data.percentiles, type: 'obj', args: ['money'] },
|
|
149
|
+
quartiles: { value: data.quartiles, type: 'list', args: ['money'] },
|
|
150
|
+
total: { value: data.total, type: 'money' },
|
|
151
|
+
max: { value: data.max, type: 'money' },
|
|
152
|
+
min: { value: data.min, type: 'money' },
|
|
153
|
+
mean: { value: data.mean, type: 'money' },
|
|
154
|
+
median: { value: data.median, type: 'money' },
|
|
155
|
+
stdDev: { value: data.stdDev, type: 'money' },
|
|
156
|
+
decades: { value: data.decades, type: 'obj', args: ['money'] },
|
|
157
|
+
gender: { value: data.gender, type: 'obj', args: ['money'] },
|
|
158
|
+
spread: { value: data.spread, type: 'obj', args: ['number'] }
|
|
159
|
+
})
|
|
160
|
+
)
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
setScatter(items) {
|
|
164
|
+
return this.saveStats('scatter.json', 'json', this.prepStats({ count: Parser.number(items.length), items }));
|
|
165
|
+
}
|
|
166
|
+
setTop10(data) {
|
|
167
|
+
return this.saveStats('top10.json', 'json', this.prepStats({ data }));
|
|
168
|
+
}
|
|
169
|
+
updateTop10(key, list) {
|
|
170
|
+
return this.setTop10({ ...this.getTop10().entries, [key]: list });
|
|
171
|
+
}
|
|
172
|
+
setDBStats(data) {
|
|
173
|
+
return this.saveStats(
|
|
174
|
+
'db.json',
|
|
175
|
+
'json',
|
|
176
|
+
this.prepStats(
|
|
177
|
+
Parser.container({ files: { value: data.files, type: 'number' }, size: { value: data.size, type: 'number' } })
|
|
178
|
+
)
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
// --- grouped stats setter ---
|
|
182
|
+
setGroupedStats(group, raw) {
|
|
183
|
+
return (
|
|
184
|
+
log.catch(() => {
|
|
185
|
+
const items = Object.fromEntries(
|
|
186
|
+
Object.entries(raw).map(([key, item]) => {
|
|
187
|
+
item = {
|
|
188
|
+
first: item.first,
|
|
189
|
+
...Parser.container({
|
|
190
|
+
date: { value: item.date, type: 'date', args: ['ymd'] },
|
|
191
|
+
count: { value: item.count, type: 'number' },
|
|
192
|
+
total: { value: item.total, type: 'money' },
|
|
193
|
+
woman: { value: item.woman, type: 'number' },
|
|
194
|
+
quota: { value: item.quota, type: 'pct' },
|
|
195
|
+
today: {
|
|
196
|
+
value: Parser.container({
|
|
197
|
+
value: { value: item.today?.value, type: 'money' },
|
|
198
|
+
percent: { value: item.today?.percent, type: 'pct' }
|
|
199
|
+
}),
|
|
200
|
+
type: 'container'
|
|
201
|
+
},
|
|
202
|
+
ytd: {
|
|
203
|
+
value: Parser.container({
|
|
204
|
+
value: { value: item.ytd?.value, type: 'money' },
|
|
205
|
+
percent: { value: item.ytd?.percent, type: 'pct' }
|
|
206
|
+
}),
|
|
207
|
+
type: 'container'
|
|
208
|
+
}
|
|
209
|
+
})
|
|
210
|
+
};
|
|
211
|
+
Stats.storage.datedCSV(
|
|
212
|
+
this.resolvePath(`${group}/${key}.csv`),
|
|
213
|
+
[
|
|
214
|
+
item.date,
|
|
215
|
+
item.count,
|
|
216
|
+
item.total,
|
|
217
|
+
item.woman,
|
|
218
|
+
item.quota,
|
|
219
|
+
item.today?.value ?? 0,
|
|
220
|
+
item.today?.percent ?? 0
|
|
221
|
+
],
|
|
222
|
+
true
|
|
223
|
+
);
|
|
224
|
+
return [key, item];
|
|
225
|
+
})
|
|
226
|
+
);
|
|
227
|
+
this.saveStats(`${group}/index.json`, 'json', this.prepStats({ items }));
|
|
228
|
+
}, `Failed to set grouped stats for group ${group}`) ?? false
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
// --- update history (add new line) ---
|
|
232
|
+
updateHistory(data) {
|
|
233
|
+
return (
|
|
234
|
+
log.catch(
|
|
235
|
+
() =>
|
|
236
|
+
Stats.storage.datedCSV(
|
|
237
|
+
this.resolvePath('history.csv'),
|
|
238
|
+
[
|
|
239
|
+
Parser.date(data.date, 'ymd'),
|
|
240
|
+
Parser.number(data.count),
|
|
241
|
+
Parser.money(data.total),
|
|
242
|
+
Parser.number(data.woman),
|
|
243
|
+
Parser.pct(data.quota),
|
|
244
|
+
Parser.money(data.today?.value),
|
|
245
|
+
Parser.pct(data.today?.percent)
|
|
246
|
+
],
|
|
247
|
+
true
|
|
248
|
+
),
|
|
249
|
+
'Failed to update history'
|
|
250
|
+
) ?? false
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
// --- generate wealth stats ---
|
|
254
|
+
generateWealthStats(scatter) {
|
|
255
|
+
return (
|
|
256
|
+
log.catch(() => {
|
|
257
|
+
log.debug('Generating wealth stats ...');
|
|
258
|
+
if (!scatter || !scatter.length) throw new Error('No scatter data provided');
|
|
259
|
+
scatter.sort((a, b) => a.networth - b.networth);
|
|
260
|
+
const count = scatter.length;
|
|
261
|
+
const total = scatter.reduce((acc, i) => acc + i.networth, 0);
|
|
262
|
+
const medianIndex = Math.floor(count / 2);
|
|
263
|
+
const median =
|
|
264
|
+
count % 2 === 0
|
|
265
|
+
? (scatter[medianIndex - 1].networth + scatter[medianIndex].networth) / 2
|
|
266
|
+
: scatter[medianIndex].networth;
|
|
267
|
+
const mean = total / count;
|
|
268
|
+
const variance =
|
|
269
|
+
scatter.reduce((acc, i) => {
|
|
270
|
+
const diff = i.networth - mean;
|
|
271
|
+
return acc + diff * diff;
|
|
272
|
+
}, 0) / count;
|
|
273
|
+
const stdDev = Math.sqrt(variance);
|
|
274
|
+
const percentiles = {};
|
|
275
|
+
Percentiles.forEach(p => {
|
|
276
|
+
const idx = Math.ceil((parseInt(p) / 100) * count) - 1;
|
|
277
|
+
percentiles[p] = scatter[idx].networth;
|
|
278
|
+
});
|
|
279
|
+
const quartiles = [
|
|
280
|
+
scatter[Math.floor(count * 0.25)].networth,
|
|
281
|
+
scatter[Math.floor(count * 0.5)].networth,
|
|
282
|
+
scatter[Math.floor(count * 0.75)].networth
|
|
283
|
+
];
|
|
284
|
+
const decades = {};
|
|
285
|
+
const gender = {};
|
|
286
|
+
const spread = {};
|
|
287
|
+
scatter.forEach(item => {
|
|
288
|
+
const { gender: g, age, networth } = item;
|
|
289
|
+
const decade = Math.max(30, Math.min(90, Math.floor(age / 10) * 10));
|
|
290
|
+
decades[decade] = (decades[decade] ?? 0) + networth;
|
|
291
|
+
gender[g] = (gender[g] ?? 0) + networth;
|
|
292
|
+
WealthSpread.forEach(n => {
|
|
293
|
+
if (networth >= Number(n) * 1000) spread[n] = (spread[n] ?? 0) + 1;
|
|
294
|
+
});
|
|
295
|
+
});
|
|
296
|
+
return this.setWealthStats({
|
|
297
|
+
total,
|
|
298
|
+
median,
|
|
299
|
+
mean,
|
|
300
|
+
stdDev,
|
|
301
|
+
percentiles,
|
|
302
|
+
quartiles,
|
|
303
|
+
decades,
|
|
304
|
+
gender,
|
|
305
|
+
spread,
|
|
306
|
+
max: scatter.at(-1).networth,
|
|
307
|
+
min: scatter[0].networth
|
|
308
|
+
});
|
|
309
|
+
}, 'Failed to generate wealth stats') ?? false
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
// --- generate top 10 entry ---
|
|
313
|
+
generateTop10Entry(snapshot) {
|
|
314
|
+
return (
|
|
315
|
+
log.catch(() => {
|
|
316
|
+
const [year, month] = snapshot.date.split('-', 2).map(Number);
|
|
317
|
+
const key = `${year}-${String(month).padStart(2, '0')}`;
|
|
318
|
+
const prev = `${month === 1 ? year - 1 : year}-${String(month === 1 ? 12 : month - 1).padStart(2, '0')}`;
|
|
319
|
+
log.debug(`Generating top 10 entry for ${key} ...`);
|
|
320
|
+
const { entries = {} } = this.getTop10();
|
|
321
|
+
const last = entries[prev];
|
|
322
|
+
const top10 = [];
|
|
323
|
+
for (const { uri, rank, networth } of snapshot.items.slice(0, 10)) {
|
|
324
|
+
const prevItem = last?.find(i => i.uri === uri);
|
|
325
|
+
top10.push({
|
|
326
|
+
uri,
|
|
327
|
+
rank,
|
|
328
|
+
networth,
|
|
329
|
+
flag: !last
|
|
330
|
+
? 'unknown'
|
|
331
|
+
: prevItem
|
|
332
|
+
? rank < prevItem.rank
|
|
333
|
+
? 'up'
|
|
334
|
+
: rank > prevItem.rank
|
|
335
|
+
? 'down'
|
|
336
|
+
: 'unchanged'
|
|
337
|
+
: Object.entries(entries).some(([k, l]) => k !== key && l.some(i => i.uri === uri))
|
|
338
|
+
? 'returned'
|
|
339
|
+
: 'new'
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
return this.updateTop10(key, top10);
|
|
343
|
+
}, 'Failed to generate top 10 entry') ?? false
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
// --- generate DB stats ---
|
|
347
|
+
generateDBStats() {
|
|
348
|
+
return (
|
|
349
|
+
log.catch(() => {
|
|
350
|
+
log.debug('Generating DB stats ...');
|
|
351
|
+
const stats = { files: 0, size: 0 };
|
|
352
|
+
const scan = path => {
|
|
353
|
+
readdirSync(path, { recursive: true }).forEach(p => {
|
|
354
|
+
if (p === '.' || p === '..' || typeof p !== 'string') return;
|
|
355
|
+
const fullPath = join(path, p);
|
|
356
|
+
const stat = Stats.storage.stat(fullPath);
|
|
357
|
+
if (stat) stat.isDirectory() ? scan(fullPath) : (stats.files++, (stats.size += stat.size));
|
|
358
|
+
});
|
|
359
|
+
};
|
|
360
|
+
scan(Stats.storage.root);
|
|
361
|
+
return this.setDBStats(stats);
|
|
362
|
+
}, 'Failed to generate DB stats') ?? false
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
// --- instantiate ---
|
|
366
|
+
static getInstance() {
|
|
367
|
+
return (Stats.instance ??= new Stats());
|
|
368
|
+
}
|
|
369
|
+
// --- aggregate stats data ---
|
|
370
|
+
static aggregate(data, date, stats) {
|
|
371
|
+
return log.catch(() => {
|
|
372
|
+
const { uri, info, realtime, realtime: { rank, networth } = {} } = data;
|
|
373
|
+
const age = Parser.age(info.birthDate),
|
|
374
|
+
decade = Parser.ageDecade(info.birthDate);
|
|
375
|
+
const item = { uri, name: info.name.shortName };
|
|
376
|
+
const set = (path, n) => Utils.update('set', stats, path, n);
|
|
377
|
+
const inc = (path, n) => Utils.update('inc', stats, path, n);
|
|
378
|
+
const max = (path, n) => Utils.update('max', stats, path, n);
|
|
379
|
+
const min = (path, n) => Utils.update('min', stats, path, n);
|
|
380
|
+
const srt = n =>
|
|
381
|
+
n >= 10
|
|
382
|
+
? 'over-10'
|
|
383
|
+
: n >= 5
|
|
384
|
+
? '5-to-10'
|
|
385
|
+
: n === 4
|
|
386
|
+
? 'four'
|
|
387
|
+
: n === 3
|
|
388
|
+
? 'three'
|
|
389
|
+
: n === 2
|
|
390
|
+
? 'two'
|
|
391
|
+
: n === 1
|
|
392
|
+
? 'one'
|
|
393
|
+
: 'none';
|
|
394
|
+
if (info.gender) inc(`profile.gender.${info.gender}`);
|
|
395
|
+
if (info.maritalStatus) inc(`profile.maritalStatus.${info.maritalStatus}`);
|
|
396
|
+
if (info.selfMade?.rank) inc(`profile.selfMade.${info.selfMade.rank}`);
|
|
397
|
+
if (info.philanthropyScore) inc(`profile.philanthropyScore.${info.philanthropyScore}`);
|
|
398
|
+
if (info.gender && age && decade) {
|
|
399
|
+
inc(`profile.agePyramid.${info.gender}.count`);
|
|
400
|
+
inc(`profile.agePyramid.${info.gender}.decades.${decade}`);
|
|
401
|
+
inc(`profile.agePyramid.${info.gender}.total`, age);
|
|
402
|
+
max(`profile.agePyramid.${info.gender}.max`, age);
|
|
403
|
+
min(`profile.agePyramid.${info.gender}.min`, age);
|
|
404
|
+
set(
|
|
405
|
+
`profile.agePyramid.${info.gender}.mean`,
|
|
406
|
+
stats.profile.agePyramid[info.gender].total / stats.profile.agePyramid[info.gender].count
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
if (info.children) {
|
|
410
|
+
inc(`profile.children.full.${info.children}`);
|
|
411
|
+
inc(`profile.children.short.${srt(info.children)}`);
|
|
412
|
+
} else {
|
|
413
|
+
inc('profile.children.short.none');
|
|
414
|
+
}
|
|
415
|
+
if (!networth || !rank || realtime?.date !== date) return stats;
|
|
416
|
+
if (info.gender && age && networth) (stats.scatter ??= []).push({ ...item, gender: info.gender, age, networth });
|
|
417
|
+
let k;
|
|
418
|
+
StatsGroup.forEach(key => {
|
|
419
|
+
if (key in info && info[key] && (k = info[key])) {
|
|
420
|
+
set(`groups.${key}.${k}.date`, date);
|
|
421
|
+
inc(`groups.${key}.${k}.count`);
|
|
422
|
+
inc(`groups.${key}.${k}.total`, networth);
|
|
423
|
+
inc(`groups.${key}.${k}.woman`, +(info.gender === 'f'));
|
|
424
|
+
set(`groups.${key}.${k}.quota`, (stats.groups[key][k].woman / stats.groups[key][k].count) * 100);
|
|
425
|
+
inc(`groups.${key}.${k}.today.value`, realtime.today?.value ?? 0);
|
|
426
|
+
inc(`groups.${key}.${k}.today.percent`, realtime.today?.percent ?? 0);
|
|
427
|
+
inc(`groups.${key}.${k}.ytd.value`, realtime.ytd?.value ?? 0);
|
|
428
|
+
inc(`groups.${key}.${k}.ytd.percent`, realtime.ytd?.percent ?? 0);
|
|
429
|
+
if (rank < (stats?.groups?.[key]?.[k]?.first?.rank ?? Infinity))
|
|
430
|
+
set(`groups.${key}.${k}.first`, { ...item, rank, networth });
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
}, 'Failed to aggregate stats data');
|
|
434
|
+
}
|
|
435
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Cache } from '../abstract/Cache.js';
|
|
2
|
+
import type { IListParser } from '../interfaces/parser';
|
|
3
|
+
export declare class ListParser<T extends object> extends Cache implements IListParser<T> {
|
|
4
|
+
protected readonly raw: T;
|
|
5
|
+
constructor(raw: T);
|
|
6
|
+
rawData(): T;
|
|
7
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { TGender, TIndustry, TMaritalStatus } from '@rtbnext/schema/src/base/const';
|
|
2
|
+
import type { TLocation } from '@rtbnext/schema/src/base/generic';
|
|
3
|
+
import type { Primitive } from 'devtypes/types/primitive';
|
|
4
|
+
import type { TParserContainer, TParserDateType, TParserMethod } from '../types/parser';
|
|
5
|
+
export declare class Parser {
|
|
6
|
+
static primitive(value: unknown, safe?: boolean): Primitive;
|
|
7
|
+
static string(value: unknown): string;
|
|
8
|
+
static safeStr(value: unknown, allowedTags?: string[]): string;
|
|
9
|
+
static boolean(value: unknown): boolean;
|
|
10
|
+
static number(value: unknown, digits?: number): number;
|
|
11
|
+
static money(value: unknown): number;
|
|
12
|
+
static pct(value: unknown, digits?: number): number;
|
|
13
|
+
static date(value?: any, format?: TParserDateType): string | undefined;
|
|
14
|
+
static json(value: unknown): any;
|
|
15
|
+
static decodeURI(value: unknown): string;
|
|
16
|
+
static encodeURI(value: unknown): string;
|
|
17
|
+
static age(value: any): number | undefined;
|
|
18
|
+
static ageDecade(value: any, min?: number, max?: number): number | undefined;
|
|
19
|
+
static gender(value: unknown): TGender | undefined;
|
|
20
|
+
static maritalStatus(value: unknown): TMaritalStatus | undefined;
|
|
21
|
+
static industry(value: unknown): TIndustry;
|
|
22
|
+
static country(value: unknown): string | undefined;
|
|
23
|
+
static state(value: unknown): string | undefined;
|
|
24
|
+
static latLng(lat: unknown, lng: unknown): [number, number] | undefined;
|
|
25
|
+
static location(value: { country: unknown; state?: unknown; city?: unknown }): TLocation | undefined;
|
|
26
|
+
static strict<T = unknown>(value: unknown, method: TParserMethod, ...args: any[]): T | undefined;
|
|
27
|
+
static list<T extends string | (string | number | undefined)[]>(
|
|
28
|
+
value: T | T[],
|
|
29
|
+
type?: TParserMethod,
|
|
30
|
+
delimiter?: string,
|
|
31
|
+
strict?: boolean,
|
|
32
|
+
...args: any[]
|
|
33
|
+
): T[];
|
|
34
|
+
static obj<T = unknown>(value: T, type?: TParserMethod, strict?: boolean, ...args: any[]): T;
|
|
35
|
+
static map<T extends Primitive, L extends readonly T[] | Record<string | number, T>>(
|
|
36
|
+
value: any,
|
|
37
|
+
list: L,
|
|
38
|
+
fb?: T | undefined,
|
|
39
|
+
exactMatch?: boolean,
|
|
40
|
+
useKey?: boolean
|
|
41
|
+
): T | undefined;
|
|
42
|
+
static container<T = unknown>(obj: { [K in keyof T]: TParserContainer }): T;
|
|
43
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import usStates from '@isodb/us-states';
|
|
2
|
+
import countries from 'i18n-iso-countries';
|
|
3
|
+
import { Gender, IndustryResolver, MaritalStatusResolver } from '../lib/const.js';
|
|
4
|
+
import { REGEX_SPACES } from '../lib/regex.js';
|
|
5
|
+
export class Parser {
|
|
6
|
+
static primitive(value, safe = true) {
|
|
7
|
+
return value === null || value === undefined
|
|
8
|
+
? value
|
|
9
|
+
: typeof value === 'boolean'
|
|
10
|
+
? value
|
|
11
|
+
: !isNaN(Number(value)) && value !== ''
|
|
12
|
+
? Parser.number(value)
|
|
13
|
+
: safe
|
|
14
|
+
? Parser.safeStr(value)
|
|
15
|
+
: Parser.string(value);
|
|
16
|
+
}
|
|
17
|
+
static string(value) {
|
|
18
|
+
return String(value).trim().replace(REGEX_SPACES, ' ');
|
|
19
|
+
}
|
|
20
|
+
static safeStr(value, allowedTags) {
|
|
21
|
+
return String(value)
|
|
22
|
+
.replace(
|
|
23
|
+
new RegExp(allowedTags?.length ? `<\\/?(?!(${allowedTags.join('|')})\\b)(\\w+)([^>]*)>` : '<[^>]*>', 'gi'),
|
|
24
|
+
''
|
|
25
|
+
)
|
|
26
|
+
.replace(REGEX_SPACES, ' ')
|
|
27
|
+
.trim();
|
|
28
|
+
}
|
|
29
|
+
static boolean(value) {
|
|
30
|
+
return (
|
|
31
|
+
value !== null &&
|
|
32
|
+
value !== undefined &&
|
|
33
|
+
(typeof value === 'boolean' ? value : ['1', 'true', 'yes', 'y'].includes(Parser.string(value).toLowerCase()))
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
static number(value, digits = 0) {
|
|
37
|
+
return Number(Number(value).toFixed(digits));
|
|
38
|
+
}
|
|
39
|
+
static money(value) {
|
|
40
|
+
return Parser.number(value, 3);
|
|
41
|
+
}
|
|
42
|
+
static pct(value, digits = 3) {
|
|
43
|
+
return Parser.number(value, digits);
|
|
44
|
+
}
|
|
45
|
+
static date(value, format = 'ymd') {
|
|
46
|
+
try {
|
|
47
|
+
value = (value ? new Date(value) : new Date()).toISOString();
|
|
48
|
+
} catch {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
return format === 'iso' ? value : value.split('T')[0].split('-').slice(0, format.length).join('-');
|
|
52
|
+
}
|
|
53
|
+
static json(value) {
|
|
54
|
+
if (typeof value === 'object') return value;
|
|
55
|
+
try {
|
|
56
|
+
return JSON.parse(String(value));
|
|
57
|
+
} catch (err) {
|
|
58
|
+
throw new SyntaxError(`Invalid JSON string: ${value}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// --- URI component ---
|
|
62
|
+
static decodeURI(value) {
|
|
63
|
+
return decodeURIComponent(Parser.string(value));
|
|
64
|
+
}
|
|
65
|
+
static encodeURI(value) {
|
|
66
|
+
return encodeURIComponent(Parser.string(value));
|
|
67
|
+
}
|
|
68
|
+
// --- profile ---
|
|
69
|
+
static age(value) {
|
|
70
|
+
const date = new Date(value);
|
|
71
|
+
return isNaN(date.getTime()) ? undefined : new Date(Date.now() - date.getTime()).getUTCFullYear() - 1970;
|
|
72
|
+
}
|
|
73
|
+
static ageDecade(value, min = 30, max = 90) {
|
|
74
|
+
const age = Parser.age(value);
|
|
75
|
+
return age === undefined ? undefined : Math.max(min, Math.min(max, Math.floor(age / 10) * 10));
|
|
76
|
+
}
|
|
77
|
+
static gender(value) {
|
|
78
|
+
return Parser.map(value, Gender);
|
|
79
|
+
}
|
|
80
|
+
static maritalStatus(value) {
|
|
81
|
+
return Parser.map(value, MaritalStatusResolver);
|
|
82
|
+
}
|
|
83
|
+
static industry(value) {
|
|
84
|
+
return Parser.map(value, IndustryResolver, 'diversified');
|
|
85
|
+
}
|
|
86
|
+
// --- location ---
|
|
87
|
+
static country(value) {
|
|
88
|
+
const code = countries.getAlpha2Code(Parser.string(value), 'en');
|
|
89
|
+
return code ? code.toUpperCase() : undefined;
|
|
90
|
+
}
|
|
91
|
+
static state(value) {
|
|
92
|
+
return usStates.byName(Parser.string(value))?.code;
|
|
93
|
+
}
|
|
94
|
+
static latLng(lat, lng) {
|
|
95
|
+
const latitude = Parser.number(lat, 6),
|
|
96
|
+
longitude = Parser.number(lng, 6);
|
|
97
|
+
return isNaN(latitude) || isNaN(longitude) ? undefined : [latitude, longitude];
|
|
98
|
+
}
|
|
99
|
+
static location(value) {
|
|
100
|
+
const country = Parser.country(value.country);
|
|
101
|
+
return country
|
|
102
|
+
? { country, state: Parser.state(value.state), city: Parser.strict(value.city, 'string') }
|
|
103
|
+
: undefined;
|
|
104
|
+
}
|
|
105
|
+
// --- helper ---
|
|
106
|
+
static strict(value, method, ...args) {
|
|
107
|
+
return value === null || value === undefined ? undefined : Parser[method](value, ...args);
|
|
108
|
+
}
|
|
109
|
+
static list(value, type = 'primitive', delimiter = ',', strict = true, ...args) {
|
|
110
|
+
return (Array.isArray(value) ? value : value.split(delimiter))
|
|
111
|
+
.map(v => (strict ? Parser.strict(v, type, ...(args ?? [])) : Parser[type](v, ...(args ?? []))))
|
|
112
|
+
.filter(Boolean);
|
|
113
|
+
}
|
|
114
|
+
static obj(value, type = 'primitive', strict = true, ...args) {
|
|
115
|
+
if (typeof value !== 'object' || value === null) return {};
|
|
116
|
+
return Object.fromEntries(
|
|
117
|
+
Object.entries(value).map(([k, v]) => [
|
|
118
|
+
k,
|
|
119
|
+
strict ? Parser.strict(v, type, ...(args ?? [])) : Parser[type](v, ...(args ?? []))
|
|
120
|
+
])
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
static map(value, list, fb = undefined, exactMatch = false, useKey) {
|
|
124
|
+
if (useKey === undefined) useKey = !Array.isArray(list);
|
|
125
|
+
value = Parser.string(value).toLowerCase();
|
|
126
|
+
return (
|
|
127
|
+
Object.entries(list).find(([k, v]) => {
|
|
128
|
+
const test = Parser.string(useKey ? k : v).toLowerCase();
|
|
129
|
+
return exactMatch ? value === test : (value.includes(test) ?? test.includes(value));
|
|
130
|
+
})?.[1] ?? fb
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
// --- container ---
|
|
134
|
+
static container(obj) {
|
|
135
|
+
return Object.fromEntries(
|
|
136
|
+
Object.entries(obj).map(([key, { value, type, strict = true, args }]) => [
|
|
137
|
+
key,
|
|
138
|
+
type === 'container'
|
|
139
|
+
? value
|
|
140
|
+
: strict
|
|
141
|
+
? Parser.strict(value, type, ...(args ?? []))
|
|
142
|
+
: Parser[type](value, ...(args ?? []))
|
|
143
|
+
])
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { TLocation, TOrganization, TSelfMade } from '@rtbnext/schema/src/base/generic';
|
|
2
|
+
import type { TProfileBio, TProfileInfo, TProfileName } from '@rtbnext/schema/src/model/profile';
|
|
3
|
+
import type { TGenericStats } from '@rtbnext/schema/src/model/stats';
|
|
4
|
+
import type { IPersonListParser } from '../interfaces/parser';
|
|
5
|
+
import { ListParser } from './ListParser.js';
|
|
6
|
+
import type { TPreparedList } from '../types/list';
|
|
7
|
+
import type { TListResponse, TPersonListEntry, TResponse } from '../types/response';
|
|
8
|
+
export declare class PersonListParser extends ListParser<TPersonListEntry> implements IPersonListParser {
|
|
9
|
+
uri(): string;
|
|
10
|
+
id(): string;
|
|
11
|
+
date(): string;
|
|
12
|
+
year(): number;
|
|
13
|
+
rank(): number | undefined;
|
|
14
|
+
networth(): number | undefined;
|
|
15
|
+
dropOff(): boolean | undefined;
|
|
16
|
+
name(): { name: TProfileName; family: boolean };
|
|
17
|
+
info(): TProfileInfo;
|
|
18
|
+
residence(): TLocation | undefined;
|
|
19
|
+
selfMade(): TSelfMade | undefined;
|
|
20
|
+
philanthropyScore(): number | undefined;
|
|
21
|
+
organization(): TOrganization | undefined;
|
|
22
|
+
bio(): TProfileBio;
|
|
23
|
+
age(): number | undefined;
|
|
24
|
+
static prepareList(
|
|
25
|
+
res: TResponse<TListResponse<TPersonListEntry>>,
|
|
26
|
+
filter?: (item: TPersonListEntry) => boolean
|
|
27
|
+
): TPreparedList<TPersonListEntry>;
|
|
28
|
+
static stats(data: Partial<TGenericStats>): TGenericStats;
|
|
29
|
+
}
|