@rtbnext/core 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +11 -0
- package/dist/abstract/Cache.d.ts +9 -0
- package/dist/abstract/Cache.js +19 -0
- package/dist/abstract/Index.d.ts +22 -0
- package/dist/abstract/Index.js +81 -0
- package/dist/abstract/Job.d.ts +19 -0
- package/dist/abstract/Job.js +49 -0
- package/dist/abstract/Snapshot.d.ts +22 -0
- package/dist/abstract/Snapshot.js +78 -0
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +25 -0
- package/dist/bin/cron.d.ts +2 -0
- package/dist/bin/cron.js +4 -0
- package/dist/core/Config.d.ts +30 -0
- package/dist/core/Config.js +66 -0
- package/dist/core/Cron.d.ts +12 -0
- package/dist/core/Cron.js +52 -0
- package/dist/core/Fetch.d.ts +28 -0
- package/dist/core/Fetch.js +172 -0
- package/dist/core/Logger.d.ts +30 -0
- package/dist/core/Logger.js +92 -0
- package/dist/core/Queue.d.ts +37 -0
- package/dist/core/Queue.js +136 -0
- package/dist/core/Storage.d.ts +28 -0
- package/dist/core/Storage.js +166 -0
- package/dist/core/Utils.d.ts +33 -0
- package/dist/core/Utils.js +167 -0
- package/dist/interfaces/cache.d.ts +6 -0
- package/dist/interfaces/config.d.ts +21 -0
- package/dist/interfaces/cron.d.ts +3 -0
- package/dist/interfaces/fetch.d.ts +13 -0
- package/dist/interfaces/filter.d.ts +12 -0
- package/dist/interfaces/index.d.ts +30 -0
- package/dist/interfaces/job.d.ts +9 -0
- package/dist/interfaces/list.d.ts +9 -0
- package/dist/interfaces/logger.d.ts +20 -0
- package/dist/interfaces/mover.d.ts +7 -0
- package/dist/interfaces/parser.d.ts +68 -0
- package/dist/interfaces/profile.d.ts +30 -0
- package/dist/interfaces/queue.d.ts +17 -0
- package/dist/interfaces/snapshot.d.ts +16 -0
- package/dist/interfaces/stats.d.ts +45 -0
- package/dist/interfaces/storage.d.ts +16 -0
- package/dist/job/Alias.d.ts +8 -0
- package/dist/job/Alias.js +42 -0
- package/dist/job/Annual.d.ts +8 -0
- package/dist/job/Annual.js +41 -0
- package/dist/job/List.d.ts +11 -0
- package/dist/job/List.js +101 -0
- package/dist/job/Merge.d.ts +10 -0
- package/dist/job/Merge.js +59 -0
- package/dist/job/Move.d.ts +7 -0
- package/dist/job/Move.js +33 -0
- package/dist/job/Performance.d.ts +8 -0
- package/dist/job/Performance.js +27 -0
- package/dist/job/Profile.d.ts +11 -0
- package/dist/job/Profile.js +76 -0
- package/dist/job/Queue.d.ts +8 -0
- package/dist/job/Queue.js +54 -0
- package/dist/job/RTB.d.ts +12 -0
- package/dist/job/RTB.js +121 -0
- package/dist/job/Stats.d.ts +11 -0
- package/dist/job/Stats.js +46 -0
- package/dist/job/Top10.d.ts +9 -0
- package/dist/job/Top10.js +48 -0
- package/dist/job/Wiki.d.ts +9 -0
- package/dist/job/Wiki.js +40 -0
- package/dist/job/index.d.ts +26 -0
- package/dist/job/index.js +26 -0
- package/dist/lib/const.d.ts +31 -0
- package/dist/lib/const.js +74 -0
- package/dist/lib/list.d.ts +90 -0
- package/dist/lib/list.js +72 -0
- package/dist/lib/regex.d.ts +7 -0
- package/dist/lib/regex.js +7 -0
- package/dist/model/Filter.d.ts +28 -0
- package/dist/model/Filter.js +122 -0
- package/dist/model/List.d.ts +12 -0
- package/dist/model/List.js +43 -0
- package/dist/model/ListIndex.d.ts +8 -0
- package/dist/model/ListIndex.js +10 -0
- package/dist/model/Mover.d.ts +15 -0
- package/dist/model/Mover.js +74 -0
- package/dist/model/Profile.d.ts +49 -0
- package/dist/model/Profile.js +181 -0
- package/dist/model/ProfileIndex.d.ts +20 -0
- package/dist/model/ProfileIndex.js +140 -0
- package/dist/model/Stats.d.ts +56 -0
- package/dist/model/Stats.js +435 -0
- package/dist/parser/BillionairesListParser.d.ts +3 -0
- package/dist/parser/BillionairesListParser.js +2 -0
- package/dist/parser/ListParser.d.ts +7 -0
- package/dist/parser/ListParser.js +11 -0
- package/dist/parser/Parser.d.ts +43 -0
- package/dist/parser/Parser.js +146 -0
- package/dist/parser/PersonListParser.d.ts +29 -0
- package/dist/parser/PersonListParser.js +111 -0
- package/dist/parser/ProfileParser.d.ts +44 -0
- package/dist/parser/ProfileParser.js +193 -0
- package/dist/parser/RTBListParser.d.ts +15 -0
- package/dist/parser/RTBListParser.js +91 -0
- package/dist/types/annual.d.ts +7 -0
- package/dist/types/config.d.ts +35 -0
- package/dist/types/fetch.d.ts +3 -0
- package/dist/types/generic.d.ts +10 -0
- package/dist/types/job.d.ts +71 -0
- package/dist/types/list.d.ts +49 -0
- package/dist/types/parser.d.ts +7 -0
- package/dist/types/profile.d.ts +9 -0
- package/dist/types/queue.d.ts +15 -0
- package/dist/types/response.d.ts +183 -0
- package/dist/types/storage.d.ts +3 -0
- package/dist/types/wiki.d.ts +1 -0
- package/dist/utils/Annual.d.ts +7 -0
- package/dist/utils/Annual.js +99 -0
- package/dist/utils/Performance.d.ts +8 -0
- package/dist/utils/Performance.js +39 -0
- package/dist/utils/ProfileManager.d.ts +24 -0
- package/dist/utils/ProfileManager.js +60 -0
- package/dist/utils/ProfileMerger.d.ts +11 -0
- package/dist/utils/ProfileMerger.js +67 -0
- package/dist/utils/Ranking.d.ts +11 -0
- package/dist/utils/Ranking.js +77 -0
- package/dist/utils/Wiki.d.ts +11 -0
- package/dist/utils/Wiki.js +168 -0
- package/package.json +45 -0
package/dist/job/List.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Fetch } from '../core/Fetch.js';
|
|
3
|
+
import { ListQueue, ProfileQueue } from '../core/Queue.js';
|
|
4
|
+
import { getListConfigByUri } from '../lib/list.js';
|
|
5
|
+
import { List } from '../model/List.js';
|
|
6
|
+
import { Profile } from '../model/Profile.js';
|
|
7
|
+
import { Parser } from '../parser/Parser.js';
|
|
8
|
+
import { ProfileManager } from '../utils/ProfileManager.js';
|
|
9
|
+
export class ListJob extends Job {
|
|
10
|
+
static fetch = Fetch.getInstance();
|
|
11
|
+
static profileQueue = ProfileQueue.getInstance();
|
|
12
|
+
static queue = ListQueue.getInstance();
|
|
13
|
+
constructor(options = {}) {
|
|
14
|
+
super(options, 'list');
|
|
15
|
+
}
|
|
16
|
+
// --- job runner ---
|
|
17
|
+
async run() {
|
|
18
|
+
await this.protect(async () => {
|
|
19
|
+
const method = this.options.profileUpdate ? 'updateData' : 'createOnly';
|
|
20
|
+
const { uri, args } = this.options.list
|
|
21
|
+
? { uri: this.options.list, args: this.options }
|
|
22
|
+
: ListJob.queue.next()[0];
|
|
23
|
+
// --- if no URI is provided, exit the job ---
|
|
24
|
+
if (!uri) return;
|
|
25
|
+
// --- check if the list already exists for the specified year ---
|
|
26
|
+
let list = List.get(uri);
|
|
27
|
+
if (list && args.year && !this.options.override && list.datesInYear(args.year).length)
|
|
28
|
+
throw new Error(`List with URI ${uri} already exists for year ${args.year}`);
|
|
29
|
+
// --- fetch raw list data from Forbes ---
|
|
30
|
+
const res = await ListJob.fetch.list(uri, args.year ?? '0');
|
|
31
|
+
if (!res?.success || !res.data) throw new Error('Request failed');
|
|
32
|
+
const { parser, indexItem, listItem } = getListConfigByUri(uri);
|
|
33
|
+
const th = Date.now() - Job.config.queue.tsThreshold;
|
|
34
|
+
const { entries } = parser.prepareList(res);
|
|
35
|
+
// --- determine list date ---
|
|
36
|
+
const d = new Date(entries[0].date ?? entries[0].timestamp);
|
|
37
|
+
if (Number.isNaN(d.getTime())) throw new Error(`Failed to determine date for ${uri} list`);
|
|
38
|
+
if (args.year && d.getFullYear() !== +args.year)
|
|
39
|
+
throw new Error(`List year ${args.year} does not match data year ${d.getFullYear()}`);
|
|
40
|
+
this.log(`Processing ${uri} list for year ${args.year ?? '-'} (${entries.length} items)`);
|
|
41
|
+
// --- process list data ---
|
|
42
|
+
let count = 0,
|
|
43
|
+
total = 0,
|
|
44
|
+
woman = 0,
|
|
45
|
+
{ name, desc } = args;
|
|
46
|
+
const date = Parser.date(d, 'ymd');
|
|
47
|
+
const items = [];
|
|
48
|
+
const queue = [];
|
|
49
|
+
for (const raw of Object.values(entries)) {
|
|
50
|
+
name ??= Parser.string(raw.name);
|
|
51
|
+
desc ??= Parser.string(raw.listDescription);
|
|
52
|
+
const parsed = new parser(raw);
|
|
53
|
+
const uri = parsed.uri();
|
|
54
|
+
const id = parsed.id();
|
|
55
|
+
let profileData = Profile.factory({ uri, id, info: parsed.info(), bio: parsed.bio() });
|
|
56
|
+
// --- process profile using ProfileManager ---
|
|
57
|
+
const { profile, action } = ProfileManager.process(uri, id, profileData, method);
|
|
58
|
+
if (!profile) {
|
|
59
|
+
this.log(`Failed to process profile for ${uri}`);
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
ProfileManager.updateQueue(queue, profile, action, th);
|
|
63
|
+
profileData = profile.getData();
|
|
64
|
+
// --- push list item ---
|
|
65
|
+
items.push(listItem({ parsed, profileData }));
|
|
66
|
+
count++;
|
|
67
|
+
total += parsed.networth() ?? 0;
|
|
68
|
+
woman += +(profileData.info?.gender === 'f');
|
|
69
|
+
}
|
|
70
|
+
// --- create list (if not exists) ---
|
|
71
|
+
if (!name || !desc) throw new Error(`Failed to determine name or description for ${uri} list`);
|
|
72
|
+
list ??= List.create(uri, indexItem(uri, { name, desc }));
|
|
73
|
+
if (!list) throw new Error(`Failed to create or retrieve ${uri} list`);
|
|
74
|
+
this.log(`Saving ${uri} list for year ${args.year ?? '-'} (${count} items)`);
|
|
75
|
+
// --- create stats ---
|
|
76
|
+
const stats = parser.stats({ date, count, total, woman });
|
|
77
|
+
// --- save data ---
|
|
78
|
+
list.saveSnapshot({ date, count, items, stats }, this.options.override);
|
|
79
|
+
ListJob.profileQueue.addMany(queue);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
// --- command definition ---
|
|
83
|
+
static command = {
|
|
84
|
+
id: 'list',
|
|
85
|
+
desc: 'Fetch and process Forbes lists',
|
|
86
|
+
options: [
|
|
87
|
+
{ name: '-l, --list <URI>', desc: 'Specify the list URI to process' },
|
|
88
|
+
{ name: '-y, --year <YYYY>', desc: 'Specify the year to process the list for' },
|
|
89
|
+
{ name: '--name <NAME>', desc: 'Specify a name for the list' },
|
|
90
|
+
{ name: '--desc <DESC>', desc: 'Specify a description for the list' },
|
|
91
|
+
{ name: '-o, --override', desc: 'If set, existing lists will be overridden with new data' },
|
|
92
|
+
{ name: '-u, --update', desc: 'If set, profile data will be updated when processing lists' }
|
|
93
|
+
]
|
|
94
|
+
};
|
|
95
|
+
// --- cron job definition ---
|
|
96
|
+
static cron = [
|
|
97
|
+
{
|
|
98
|
+
cronexpr: '*/15 1 * * *' // run every 15 minutes between 1:00 and 1:59 AM
|
|
99
|
+
}
|
|
100
|
+
];
|
|
101
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TMergeJobOptions } from '../types/job';
|
|
3
|
+
export declare class MergeJob extends Job<TMergeJobOptions> {
|
|
4
|
+
constructor(options: TMergeJobOptions);
|
|
5
|
+
private listMergeable;
|
|
6
|
+
private isMergeable;
|
|
7
|
+
private merge;
|
|
8
|
+
run(): Promise<void>;
|
|
9
|
+
static readonly command: TCommandJob;
|
|
10
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Profile } from '../model/Profile.js';
|
|
3
|
+
import { Parser } from '../parser/Parser.js';
|
|
4
|
+
import { ProfileMerger } from '../utils/ProfileMerger.js';
|
|
5
|
+
export class MergeJob extends Job {
|
|
6
|
+
constructor(options) {
|
|
7
|
+
super(options, 'merge');
|
|
8
|
+
}
|
|
9
|
+
// --- job runner ---
|
|
10
|
+
listMergeable(uriLike) {
|
|
11
|
+
for (const [uri, list] of Object.entries(ProfileMerger.listCandidates(...uriLike))) {
|
|
12
|
+
console.log(`Candidates for ${uri}:`);
|
|
13
|
+
if (!list.length) console.log(' - None');
|
|
14
|
+
for (const candidate of list) console.log(` - ${candidate}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
isMergeable(target, source) {
|
|
18
|
+
const test = ProfileMerger.mergeableProfiles(target.getData(), source.getData());
|
|
19
|
+
if (test) console.log(`Profiles ${target.getUri()} and ${source.getUri()} are mergeable.`);
|
|
20
|
+
else console.log(`Profiles ${target.getUri()} and ${source.getUri()} are NOT mergeable.`);
|
|
21
|
+
}
|
|
22
|
+
merge(target, source, force, makeAlias) {
|
|
23
|
+
this.log(`Merging profile ${source.getUri()} into ${target.getUri()}`);
|
|
24
|
+
const res = ProfileMerger.mergeProfiles(target, source, force, makeAlias);
|
|
25
|
+
if (!res) throw new Error('Failed to merge profiles.');
|
|
26
|
+
this.log('Merge completed successfully.');
|
|
27
|
+
}
|
|
28
|
+
async run() {
|
|
29
|
+
await this.protect(async () => {
|
|
30
|
+
const { list, source, target, dryRun, force, makeAlias } = this.options;
|
|
31
|
+
if (list?.length) this.listMergeable(list);
|
|
32
|
+
else if (!source || !target) throw new Error('Invalid arguments for merge job');
|
|
33
|
+
else {
|
|
34
|
+
const src = Profile.get(source),
|
|
35
|
+
tgt = Profile.get(target);
|
|
36
|
+
if (!src || !tgt) throw new Error('One or both profiles not found');
|
|
37
|
+
if (dryRun) this.isMergeable(tgt, src);
|
|
38
|
+
else this.merge(tgt, src, Parser.boolean(force), Parser.boolean(makeAlias));
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// --- command definition ---
|
|
43
|
+
static command = {
|
|
44
|
+
id: 'merge',
|
|
45
|
+
desc: 'Merge two profiles safely',
|
|
46
|
+
options: [
|
|
47
|
+
{
|
|
48
|
+
name: '-l, --list <URIs>',
|
|
49
|
+
desc: 'List merge candidates for the given profile URIs (comma-separated)',
|
|
50
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
51
|
+
},
|
|
52
|
+
{ name: '-s, --source <URI>', desc: 'Source profile URI to merge from' },
|
|
53
|
+
{ name: '-t, --target <URI>', desc: 'Target profile URI to merge into' },
|
|
54
|
+
{ name: '--dry-run', desc: 'Perform a dry run of the merge without making any changes' },
|
|
55
|
+
{ name: '--force', desc: 'Force the merge even if there are potential conflicts' },
|
|
56
|
+
{ name: '-a, --make-alias', desc: 'Whether to create an alias from the old URI to the new one after merge' }
|
|
57
|
+
]
|
|
58
|
+
};
|
|
59
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TMoveJobOptions } from '../types/job';
|
|
3
|
+
export declare class MoveJob extends Job<TMoveJobOptions> {
|
|
4
|
+
constructor(options: TMoveJobOptions);
|
|
5
|
+
run(): Promise<void>;
|
|
6
|
+
static readonly command: TCommandJob;
|
|
7
|
+
}
|
package/dist/job/Move.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Utils } from '../core/Utils.js';
|
|
3
|
+
import { Profile } from '../model/Profile.js';
|
|
4
|
+
export class MoveJob extends Job {
|
|
5
|
+
constructor(options) {
|
|
6
|
+
super(options, 'move');
|
|
7
|
+
}
|
|
8
|
+
// --- job runner ---
|
|
9
|
+
async run() {
|
|
10
|
+
await this.protect(async () => {
|
|
11
|
+
const makeAlias = !!this.options.makeAlias;
|
|
12
|
+
const source = Utils.sanitize(this.options.source);
|
|
13
|
+
const target = Utils.sanitize(this.options.target);
|
|
14
|
+
if (!source || !target) throw new Error('Invalid from/to profile names');
|
|
15
|
+
const profile = Profile.find(source);
|
|
16
|
+
if (!profile) throw new Error(`Profile ${source} not found`);
|
|
17
|
+
this.log(`Moving profile from ${source} to ${target} ...`);
|
|
18
|
+
const res = profile.move(target, makeAlias);
|
|
19
|
+
if (!res) throw new Error(`Failed to move profile from ${source} to ${target}`);
|
|
20
|
+
this.log(`Profile moved successfully to ${target}${makeAlias ? ' with alias' : ''}`);
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
// --- command definition ---
|
|
24
|
+
static command = {
|
|
25
|
+
id: 'move',
|
|
26
|
+
desc: 'Move a profile from one URI to another',
|
|
27
|
+
options: [
|
|
28
|
+
{ name: '-s, --source <URI>', desc: 'Source profile URI to move from', required: true },
|
|
29
|
+
{ name: '-t, --target <URI>', desc: 'Target profile URI to move to', required: true },
|
|
30
|
+
{ name: '-a, --make-alias', desc: 'Whether to create an alias from the old URI to the new one' }
|
|
31
|
+
]
|
|
32
|
+
};
|
|
33
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TJobClsOptions } from '../types/job';
|
|
3
|
+
export declare class PerformanceJob extends Job {
|
|
4
|
+
private static readonly index;
|
|
5
|
+
constructor(options?: TJobClsOptions);
|
|
6
|
+
run(): Promise<void>;
|
|
7
|
+
static readonly command: TCommandJob;
|
|
8
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Profile } from '../model/Profile.js';
|
|
3
|
+
import { ProfileIndex } from '../model/ProfileIndex.js';
|
|
4
|
+
import { Performance } from '../utils/Performance.js';
|
|
5
|
+
export class PerformanceJob extends Job {
|
|
6
|
+
static index = ProfileIndex.getInstance();
|
|
7
|
+
constructor(options = {}) {
|
|
8
|
+
super(options, 'performance');
|
|
9
|
+
}
|
|
10
|
+
// --- job runner ---
|
|
11
|
+
async run() {
|
|
12
|
+
await this.protect(async () => {
|
|
13
|
+
for (const item of PerformanceJob.index.values) {
|
|
14
|
+
const profile = Profile.getByItem(item);
|
|
15
|
+
if (!profile) {
|
|
16
|
+
this.log(`Profile not found for ${item.uri}`, undefined, 'warn');
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
const performance = Performance.generateProfilePerformance(profile.getHistory());
|
|
20
|
+
profile.updateData({ performance });
|
|
21
|
+
profile.save();
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
// --- command definition ---
|
|
26
|
+
static command = { id: 'performance', desc: 'Generate profile performance data' };
|
|
27
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TCronJob, TProfileJobOptions } from '../types/job';
|
|
3
|
+
export declare class ProfileJob extends Job<TProfileJobOptions> {
|
|
4
|
+
private static readonly fetch;
|
|
5
|
+
private static readonly queue;
|
|
6
|
+
private static readonly index;
|
|
7
|
+
constructor(options?: TProfileJobOptions);
|
|
8
|
+
run(): Promise<void>;
|
|
9
|
+
static readonly command: TCommandJob;
|
|
10
|
+
static readonly cron: TCronJob<TProfileJobOptions>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Fetch } from '../core/Fetch.js';
|
|
3
|
+
import { ProfileQueue } from '../core/Queue.js';
|
|
4
|
+
import { Profile } from '../model/Profile.js';
|
|
5
|
+
import { ProfileIndex } from '../model/ProfileIndex.js';
|
|
6
|
+
import { Parser } from '../parser/Parser.js';
|
|
7
|
+
import { ProfileParser } from '../parser/ProfileParser.js';
|
|
8
|
+
import { ProfileManager } from '../utils/ProfileManager.js';
|
|
9
|
+
import { Ranking } from '../utils/Ranking.js';
|
|
10
|
+
import { Wiki } from '../utils/Wiki.js';
|
|
11
|
+
export class ProfileJob extends Job {
|
|
12
|
+
static fetch = Fetch.getInstance();
|
|
13
|
+
static queue = ProfileQueue.getInstance();
|
|
14
|
+
static index = ProfileIndex.getInstance();
|
|
15
|
+
constructor(options = {}) {
|
|
16
|
+
super(options, 'profile');
|
|
17
|
+
}
|
|
18
|
+
// --- job runner ---
|
|
19
|
+
async run() {
|
|
20
|
+
await this.protect(async () => {
|
|
21
|
+
const method = this.options.replace ? 'setData' : 'updateData';
|
|
22
|
+
const batch = this.options.profiles?.length
|
|
23
|
+
? this.options.profiles
|
|
24
|
+
: ProfileJob.queue.nextUri(Job.config.fetch.rateLimit.batchSize);
|
|
25
|
+
for (const raw of await ProfileJob.fetch.profile(...batch)) {
|
|
26
|
+
if (!raw?.success || !raw.data) {
|
|
27
|
+
this.log('Request failed', raw, 'warn');
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
// --- parse raw profile data ---
|
|
31
|
+
const parser = new ProfileParser(raw.data);
|
|
32
|
+
const uri = parser.uri();
|
|
33
|
+
const id = parser.id();
|
|
34
|
+
const profileData = Profile.factory({
|
|
35
|
+
uri,
|
|
36
|
+
id,
|
|
37
|
+
info: parser.info(),
|
|
38
|
+
bio: parser.bio(),
|
|
39
|
+
related: parser.related(),
|
|
40
|
+
media: parser.media()
|
|
41
|
+
});
|
|
42
|
+
// --- enrich profile data with ranking and wiki ---
|
|
43
|
+
if (!Parser.boolean(this.options.skipRanking))
|
|
44
|
+
profileData.ranking = Ranking.generateProfileRanking(parser.sortedLists(), profileData.ranking);
|
|
45
|
+
if (!Parser.boolean(this.options.skipWiki)) profileData.wiki = await Wiki.fromProfileData(profileData);
|
|
46
|
+
// --- process profile using ProfileManager ---
|
|
47
|
+
const { action, success } = ProfileManager.process(uri, id, profileData, method, true, true);
|
|
48
|
+
if (!success) this.log(`Failed to process profile with uri ${uri}`, profileData, 'warn');
|
|
49
|
+
// --- add profile aliases ---
|
|
50
|
+
ProfileJob.index.addAliases(uri, ...parser.aliases());
|
|
51
|
+
this.log(`Profile with uri ${uri} processed in ${action} mode`);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
// --- command definition ---
|
|
56
|
+
static command = {
|
|
57
|
+
id: 'profile',
|
|
58
|
+
desc: 'Fetch and update Forbes profiles',
|
|
59
|
+
options: [
|
|
60
|
+
{
|
|
61
|
+
name: '-p, --profiles <URIs>',
|
|
62
|
+
desc: 'Process specific profiles by URI (comma-separated)',
|
|
63
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
64
|
+
},
|
|
65
|
+
{ name: '--replace', desc: 'Replace existing profile data' },
|
|
66
|
+
{ name: '--skip-ranking', desc: 'Skip ranking generation' },
|
|
67
|
+
{ name: '--skip-wiki', desc: 'Skip wiki data enrichment' }
|
|
68
|
+
]
|
|
69
|
+
};
|
|
70
|
+
// --- cron job definition ---
|
|
71
|
+
static cron = [
|
|
72
|
+
{
|
|
73
|
+
cronexpr: '*/5 2-23 * * *' // run every 5 minutes between 2:00 and 23:59
|
|
74
|
+
}
|
|
75
|
+
];
|
|
76
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TQueueJobOptions } from '../types/job';
|
|
3
|
+
export declare class QueueJob extends Job<TQueueJobOptions> {
|
|
4
|
+
private static readonly queues;
|
|
5
|
+
constructor(options: TQueueJobOptions);
|
|
6
|
+
run(): Promise<void>;
|
|
7
|
+
static readonly command: TCommandJob;
|
|
8
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { ListQueue, ProfileQueue } from '../core/Queue.js';
|
|
3
|
+
import { Parser } from '../parser/Parser.js';
|
|
4
|
+
export class QueueJob extends Job {
|
|
5
|
+
static queues = { list: ListQueue.getInstance(), profile: ProfileQueue.getInstance() };
|
|
6
|
+
constructor(options) {
|
|
7
|
+
super(options, 'queue');
|
|
8
|
+
}
|
|
9
|
+
// --- job runner ---
|
|
10
|
+
async run() {
|
|
11
|
+
await this.protect(async () => {
|
|
12
|
+
const { type, clear, add, remove, prio, args } = this.options;
|
|
13
|
+
const queue = QueueJob.queues[type];
|
|
14
|
+
if (!queue) throw new Error(`Unknown queue type: ${type}`);
|
|
15
|
+
if (clear) queue.clear();
|
|
16
|
+
if (add?.length) queue.addMany(add.map(uriLike => ({ uriLike, prio, args })));
|
|
17
|
+
if (remove?.length) queue.remove(...remove);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
// --- command definition ---
|
|
21
|
+
static command = {
|
|
22
|
+
id: 'queue',
|
|
23
|
+
desc: 'Managing list and profile queues',
|
|
24
|
+
options: [
|
|
25
|
+
{
|
|
26
|
+
name: '-t, --type <TYPE>',
|
|
27
|
+
desc: 'Which queue to use (available: list, profile)',
|
|
28
|
+
parser: v => v.toLowerCase(),
|
|
29
|
+
required: true
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
name: '--add <URIs>',
|
|
33
|
+
desc: 'Add items to the queue (comma-separated)',
|
|
34
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: '--remove <URIs>',
|
|
38
|
+
desc: 'Remove items from the queue (comma-separated)',
|
|
39
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
name: '--prio <NUMBER>',
|
|
43
|
+
desc: 'Set priority of added items (higher number = higher priority)',
|
|
44
|
+
parser: v => Parser.strict(v, 'number')
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
name: '--args <JSON>',
|
|
48
|
+
desc: 'Additional queue item arguments as JSON string',
|
|
49
|
+
parser: v => Parser.strict(v, 'json')
|
|
50
|
+
},
|
|
51
|
+
{ name: '-c, --clear', desc: 'Clear the queue before adding new items' }
|
|
52
|
+
]
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TCronJob, TJobClsOptions } from '../types/job';
|
|
3
|
+
export declare class RTBJob extends Job {
|
|
4
|
+
private static readonly fetch;
|
|
5
|
+
private static readonly mover;
|
|
6
|
+
private static readonly queue;
|
|
7
|
+
private static readonly stats;
|
|
8
|
+
constructor(options?: TJobClsOptions);
|
|
9
|
+
run(): Promise<void>;
|
|
10
|
+
static readonly command: TCommandJob;
|
|
11
|
+
static readonly cron: TCronJob;
|
|
12
|
+
}
|
package/dist/job/RTB.js
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Fetch } from '../core/Fetch.js';
|
|
3
|
+
import { ProfileQueue } from '../core/Queue.js';
|
|
4
|
+
import { LISTS } from '../lib/list.js';
|
|
5
|
+
import { List } from '../model/List.js';
|
|
6
|
+
import { Mover } from '../model/Mover.js';
|
|
7
|
+
import { Profile } from '../model/Profile.js';
|
|
8
|
+
import { ProfileIndex } from '../model/ProfileIndex.js';
|
|
9
|
+
import { Stats } from '../model/Stats.js';
|
|
10
|
+
import { Parser } from '../parser/Parser.js';
|
|
11
|
+
import { Performance } from '../utils/Performance.js';
|
|
12
|
+
import { ProfileManager } from '../utils/ProfileManager.js';
|
|
13
|
+
export class RTBJob extends Job {
|
|
14
|
+
static fetch = Fetch.getInstance();
|
|
15
|
+
static mover = Mover.getInstance();
|
|
16
|
+
static queue = ProfileQueue.getInstance();
|
|
17
|
+
static stats = Stats.getInstance();
|
|
18
|
+
constructor(options = {}) {
|
|
19
|
+
super(options, 'rtb');
|
|
20
|
+
}
|
|
21
|
+
// --- job runner ---
|
|
22
|
+
async run() {
|
|
23
|
+
await this.protect(async () => {
|
|
24
|
+
const res = await RTBJob.fetch.list('rtb', '0');
|
|
25
|
+
if (!res?.success || !res.data) throw new Error('Request failed');
|
|
26
|
+
const date = Parser.date(undefined, 'ymd');
|
|
27
|
+
const ts = new Date(date).getTime();
|
|
28
|
+
// --- if RTB list is already up to date, exit the job ---
|
|
29
|
+
if (RTBJob.stats.getGlobalStats().date === date) {
|
|
30
|
+
this.log('RTB list is already up to date');
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const { parser, indexItem, listItem } = LISTS.rtb;
|
|
34
|
+
const th = Date.now() - Job.config.queue.tsThreshold;
|
|
35
|
+
const { entries } = parser.prepareList(res);
|
|
36
|
+
this.log(`Processing RTB list dated ${date} (${entries.length} items)`);
|
|
37
|
+
// --- process list data ---
|
|
38
|
+
let count = 0,
|
|
39
|
+
total = 0,
|
|
40
|
+
woman = 0;
|
|
41
|
+
const items = [];
|
|
42
|
+
const mover = Mover.factory(date);
|
|
43
|
+
const queue = [];
|
|
44
|
+
for (const [i, raw] of Object.entries(entries)) {
|
|
45
|
+
raw.date = ts;
|
|
46
|
+
// --- parse raw list data ---
|
|
47
|
+
const parsed = new parser(raw);
|
|
48
|
+
const uri = parsed.uri();
|
|
49
|
+
const id = parsed.id();
|
|
50
|
+
const rank = parsed.rank();
|
|
51
|
+
const networth = parsed.networth();
|
|
52
|
+
if (!rank || !networth) {
|
|
53
|
+
this.log(`Skipping invalid RTB entry for ${uri}`);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
let profileData = Profile.factory({ uri, id, info: parsed.info(), bio: parsed.bio(), assets: parsed.assets() });
|
|
57
|
+
// --- process profile using ProfileManager ---
|
|
58
|
+
const { profile, action } = ProfileManager.process(uri, id, profileData, 'updateData');
|
|
59
|
+
if (!profile) {
|
|
60
|
+
this.log(`Failed to process profile for ${uri}`);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
ProfileManager.updateQueue(queue, profile, action, th);
|
|
64
|
+
profileData = profile.getData();
|
|
65
|
+
// --- process realtime data ---
|
|
66
|
+
const prev = entries[Number(i) - 1]?.uri;
|
|
67
|
+
const next = entries[Number(i) + 1]?.uri;
|
|
68
|
+
const realtime = parsed.realtime(profileData, prev, next);
|
|
69
|
+
const { flag, rankDiff } = parsed.rankDiff(profileData);
|
|
70
|
+
const { value = 0, percent = 0 } = realtime?.today ?? {};
|
|
71
|
+
// --- update profile data ---
|
|
72
|
+
profile.addHistory([date, rank, networth, value, percent]);
|
|
73
|
+
const performance = Performance.generateProfilePerformance(profile.getHistory());
|
|
74
|
+
profile.updateData({ realtime, performance });
|
|
75
|
+
profile.save();
|
|
76
|
+
profileData = profile.getData();
|
|
77
|
+
const name = profileData.info.name.shortName;
|
|
78
|
+
// --- skip profiles if their networth is less than $1B ---
|
|
79
|
+
if (networth < 1000 && networth - value < 1000) continue;
|
|
80
|
+
// --- aggregate mover data ---
|
|
81
|
+
Mover.aggregate(realtime, uri, name, mover, total);
|
|
82
|
+
// --- push list item ---
|
|
83
|
+
items.push(listItem({ parsed, profileData, flag, rankDiff, realtime }));
|
|
84
|
+
count++;
|
|
85
|
+
total += networth;
|
|
86
|
+
woman += +(profileData.info?.gender === 'f');
|
|
87
|
+
}
|
|
88
|
+
// --- create "rtb" list ---
|
|
89
|
+
const list = List.get('rtb') || List.create('rtb', indexItem());
|
|
90
|
+
if (!list) throw new Error('Failed to create or retrieve RTB list');
|
|
91
|
+
this.log(`Saving RTB list dated ${date} (${count} items)`);
|
|
92
|
+
// --- create stats ---
|
|
93
|
+
const stats = parser.stats({
|
|
94
|
+
date,
|
|
95
|
+
count,
|
|
96
|
+
total,
|
|
97
|
+
woman,
|
|
98
|
+
today: { value: mover.today.total.value, percent: mover.today.total.percent },
|
|
99
|
+
ytd: { value: mover.ytd.total.value, percent: mover.ytd.total.percent }
|
|
100
|
+
});
|
|
101
|
+
const globalStats = {
|
|
102
|
+
...stats,
|
|
103
|
+
stats: { profiles: ProfileIndex.getInstance().size, days: list.getDates().length }
|
|
104
|
+
};
|
|
105
|
+
// --- save data ---
|
|
106
|
+
list.saveSnapshot({ date, count, items, stats });
|
|
107
|
+
RTBJob.mover.saveSnapshot(mover);
|
|
108
|
+
RTBJob.queue.addMany(queue);
|
|
109
|
+
RTBJob.stats.setGlobalStats(globalStats);
|
|
110
|
+
RTBJob.stats.updateHistory(globalStats);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
// --- command definition ---
|
|
114
|
+
static command = { id: 'rtb', desc: 'Proceed daily real-time billionaires list' };
|
|
115
|
+
// --- cron job definition ---
|
|
116
|
+
static cron = [
|
|
117
|
+
{
|
|
118
|
+
cronexpr: '15 0 * * *' // run at 0:15 AM on every day
|
|
119
|
+
}
|
|
120
|
+
];
|
|
121
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TCronJob, TJobClsOptions } from '../types/job';
|
|
3
|
+
export declare class StatsJob extends Job {
|
|
4
|
+
private static readonly filter;
|
|
5
|
+
private static readonly index;
|
|
6
|
+
private static readonly stats;
|
|
7
|
+
constructor(options?: TJobClsOptions);
|
|
8
|
+
run(): Promise<void>;
|
|
9
|
+
static readonly command: TCommandJob;
|
|
10
|
+
static readonly cron: TCronJob;
|
|
11
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { StatsGroup } from '../lib/const.js';
|
|
3
|
+
import { Filter } from '../model/Filter.js';
|
|
4
|
+
import { Profile } from '../model/Profile.js';
|
|
5
|
+
import { ProfileIndex } from '../model/ProfileIndex.js';
|
|
6
|
+
import { Stats } from '../model/Stats.js';
|
|
7
|
+
export class StatsJob extends Job {
|
|
8
|
+
static filter = Filter.getInstance();
|
|
9
|
+
static index = ProfileIndex.getInstance();
|
|
10
|
+
static stats = Stats.getInstance();
|
|
11
|
+
constructor(options = {}) {
|
|
12
|
+
super(options, 'stats');
|
|
13
|
+
}
|
|
14
|
+
// --- job runner ---
|
|
15
|
+
async run() {
|
|
16
|
+
await this.protect(async () => {
|
|
17
|
+
const date = StatsJob.stats.getGlobalStats().date;
|
|
18
|
+
if (!date || !StatsJob.index.size) throw new Error('No data available');
|
|
19
|
+
this.log(`Generating stats for ${date} with ${StatsJob.index.size} profiles`);
|
|
20
|
+
const filter = {},
|
|
21
|
+
stats = {};
|
|
22
|
+
for (const item of StatsJob.index.values) {
|
|
23
|
+
const profile = Profile.getByItem(item);
|
|
24
|
+
if (!profile) continue;
|
|
25
|
+
const data = profile.getData();
|
|
26
|
+
Stats.aggregate(data, date, stats);
|
|
27
|
+
Filter.aggregate(data, filter);
|
|
28
|
+
}
|
|
29
|
+
this.log(`Saving stats for ${date}`);
|
|
30
|
+
StatsJob.filter.save(filter);
|
|
31
|
+
StatsJob.stats.setProfileStats(stats.profile);
|
|
32
|
+
StatsJob.stats.generateWealthStats(stats.scatter);
|
|
33
|
+
StatsGroup.forEach(g => StatsJob.stats.setGroupedStats(g, stats.groups[g]));
|
|
34
|
+
StatsJob.stats.setScatter(stats.scatter);
|
|
35
|
+
StatsJob.stats.generateDBStats();
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
// --- command definition ---
|
|
39
|
+
static command = { id: 'stats', desc: 'Generate stats and filtered lists' };
|
|
40
|
+
// --- cron job definition ---
|
|
41
|
+
static cron = [
|
|
42
|
+
{
|
|
43
|
+
cronexpr: '30 0 * * *' // run at 0:30 AM on every day
|
|
44
|
+
}
|
|
45
|
+
];
|
|
46
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TCronJob, TTop10JobOptions } from '../types/job';
|
|
3
|
+
export declare class Top10Job extends Job<TTop10JobOptions> {
|
|
4
|
+
private static readonly stats;
|
|
5
|
+
constructor(options: TTop10JobOptions);
|
|
6
|
+
run(): Promise<void>;
|
|
7
|
+
static readonly command: TCommandJob;
|
|
8
|
+
static readonly cron: TCronJob<TTop10JobOptions>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Utils } from '../core/Utils.js';
|
|
3
|
+
import { List } from '../model/List.js';
|
|
4
|
+
import { Stats } from '../model/Stats.js';
|
|
5
|
+
import { Parser } from '../parser/Parser.js';
|
|
6
|
+
export class Top10Job extends Job {
|
|
7
|
+
static stats = Stats.getInstance();
|
|
8
|
+
constructor(options) {
|
|
9
|
+
super(options, 'top10');
|
|
10
|
+
}
|
|
11
|
+
// --- job runner ---
|
|
12
|
+
async run() {
|
|
13
|
+
await this.protect(async () => {
|
|
14
|
+
const list = List.get('rtb');
|
|
15
|
+
if (!list) throw new Error('Real-time billionaires list not found');
|
|
16
|
+
const [year, month] = this.options.date ?? Utils.date('ym').split('-', 2);
|
|
17
|
+
const date = Parser.date(Utils.lastMonthDay(month, year), 'ymd');
|
|
18
|
+
const snapshot = list.getSnapshot(date, false);
|
|
19
|
+
if (!snapshot) throw new Error(`No snapshot found for ${month}/${year}`);
|
|
20
|
+
Top10Job.stats.generateTop10Entry(snapshot);
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
// --- command definition ---
|
|
24
|
+
static command = {
|
|
25
|
+
id: 'top10',
|
|
26
|
+
desc: 'Generate monthly top 10 ranking',
|
|
27
|
+
options: [
|
|
28
|
+
{
|
|
29
|
+
name: '-d, --date <YYYY-MM>',
|
|
30
|
+
desc: 'Specify the date for the top 10 ranking',
|
|
31
|
+
parser: v => {
|
|
32
|
+
if (!/^\d{4}-\d{2}$/.test(v)) throw new Error(`Invalid date format "${v}"; use YYYY-MM.`);
|
|
33
|
+
return v.split('-', 2).map(v => Number(v));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
};
|
|
38
|
+
// --- cron job definition ---
|
|
39
|
+
static cron = [
|
|
40
|
+
{
|
|
41
|
+
cronexpr: '35 0 1 * *', // run at 1:35 AM on the first day of every month
|
|
42
|
+
options: date => {
|
|
43
|
+
date.setMonth(date.getMonth() - 1);
|
|
44
|
+
return { date: [date.getFullYear(), date.getMonth() + 1] };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
];
|
|
48
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TWikiJobOptions } from '../types/job';
|
|
3
|
+
export declare class WikiJob extends Job<TWikiJobOptions> {
|
|
4
|
+
constructor(options: TWikiJobOptions);
|
|
5
|
+
private update;
|
|
6
|
+
private assign;
|
|
7
|
+
run(): Promise<void>;
|
|
8
|
+
static readonly command: TCommandJob;
|
|
9
|
+
}
|