@rtbnext/core 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +11 -0
- package/dist/abstract/Cache.d.ts +9 -0
- package/dist/abstract/Cache.js +19 -0
- package/dist/abstract/Index.d.ts +22 -0
- package/dist/abstract/Index.js +81 -0
- package/dist/abstract/Job.d.ts +19 -0
- package/dist/abstract/Job.js +49 -0
- package/dist/abstract/Snapshot.d.ts +22 -0
- package/dist/abstract/Snapshot.js +78 -0
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +25 -0
- package/dist/bin/cron.d.ts +2 -0
- package/dist/bin/cron.js +4 -0
- package/dist/core/Config.d.ts +30 -0
- package/dist/core/Config.js +66 -0
- package/dist/core/Cron.d.ts +12 -0
- package/dist/core/Cron.js +52 -0
- package/dist/core/Fetch.d.ts +28 -0
- package/dist/core/Fetch.js +172 -0
- package/dist/core/Logger.d.ts +30 -0
- package/dist/core/Logger.js +92 -0
- package/dist/core/Queue.d.ts +37 -0
- package/dist/core/Queue.js +136 -0
- package/dist/core/Storage.d.ts +28 -0
- package/dist/core/Storage.js +166 -0
- package/dist/core/Utils.d.ts +33 -0
- package/dist/core/Utils.js +167 -0
- package/dist/interfaces/cache.d.ts +6 -0
- package/dist/interfaces/config.d.ts +21 -0
- package/dist/interfaces/cron.d.ts +3 -0
- package/dist/interfaces/fetch.d.ts +13 -0
- package/dist/interfaces/filter.d.ts +12 -0
- package/dist/interfaces/index.d.ts +30 -0
- package/dist/interfaces/job.d.ts +9 -0
- package/dist/interfaces/list.d.ts +9 -0
- package/dist/interfaces/logger.d.ts +20 -0
- package/dist/interfaces/mover.d.ts +7 -0
- package/dist/interfaces/parser.d.ts +68 -0
- package/dist/interfaces/profile.d.ts +30 -0
- package/dist/interfaces/queue.d.ts +17 -0
- package/dist/interfaces/snapshot.d.ts +16 -0
- package/dist/interfaces/stats.d.ts +45 -0
- package/dist/interfaces/storage.d.ts +16 -0
- package/dist/job/Alias.d.ts +8 -0
- package/dist/job/Alias.js +42 -0
- package/dist/job/Annual.d.ts +8 -0
- package/dist/job/Annual.js +41 -0
- package/dist/job/List.d.ts +11 -0
- package/dist/job/List.js +101 -0
- package/dist/job/Merge.d.ts +10 -0
- package/dist/job/Merge.js +59 -0
- package/dist/job/Move.d.ts +7 -0
- package/dist/job/Move.js +33 -0
- package/dist/job/Performance.d.ts +8 -0
- package/dist/job/Performance.js +27 -0
- package/dist/job/Profile.d.ts +11 -0
- package/dist/job/Profile.js +76 -0
- package/dist/job/Queue.d.ts +8 -0
- package/dist/job/Queue.js +54 -0
- package/dist/job/RTB.d.ts +12 -0
- package/dist/job/RTB.js +121 -0
- package/dist/job/Stats.d.ts +11 -0
- package/dist/job/Stats.js +46 -0
- package/dist/job/Top10.d.ts +9 -0
- package/dist/job/Top10.js +48 -0
- package/dist/job/Wiki.d.ts +9 -0
- package/dist/job/Wiki.js +40 -0
- package/dist/job/index.d.ts +26 -0
- package/dist/job/index.js +26 -0
- package/dist/lib/const.d.ts +31 -0
- package/dist/lib/const.js +74 -0
- package/dist/lib/list.d.ts +90 -0
- package/dist/lib/list.js +72 -0
- package/dist/lib/regex.d.ts +7 -0
- package/dist/lib/regex.js +7 -0
- package/dist/model/Filter.d.ts +28 -0
- package/dist/model/Filter.js +122 -0
- package/dist/model/List.d.ts +12 -0
- package/dist/model/List.js +43 -0
- package/dist/model/ListIndex.d.ts +8 -0
- package/dist/model/ListIndex.js +10 -0
- package/dist/model/Mover.d.ts +15 -0
- package/dist/model/Mover.js +74 -0
- package/dist/model/Profile.d.ts +49 -0
- package/dist/model/Profile.js +181 -0
- package/dist/model/ProfileIndex.d.ts +20 -0
- package/dist/model/ProfileIndex.js +140 -0
- package/dist/model/Stats.d.ts +56 -0
- package/dist/model/Stats.js +435 -0
- package/dist/parser/BillionairesListParser.d.ts +3 -0
- package/dist/parser/BillionairesListParser.js +2 -0
- package/dist/parser/ListParser.d.ts +7 -0
- package/dist/parser/ListParser.js +11 -0
- package/dist/parser/Parser.d.ts +43 -0
- package/dist/parser/Parser.js +146 -0
- package/dist/parser/PersonListParser.d.ts +29 -0
- package/dist/parser/PersonListParser.js +111 -0
- package/dist/parser/ProfileParser.d.ts +44 -0
- package/dist/parser/ProfileParser.js +193 -0
- package/dist/parser/RTBListParser.d.ts +15 -0
- package/dist/parser/RTBListParser.js +91 -0
- package/dist/types/annual.d.ts +7 -0
- package/dist/types/config.d.ts +35 -0
- package/dist/types/fetch.d.ts +3 -0
- package/dist/types/generic.d.ts +10 -0
- package/dist/types/job.d.ts +71 -0
- package/dist/types/list.d.ts +49 -0
- package/dist/types/parser.d.ts +7 -0
- package/dist/types/profile.d.ts +9 -0
- package/dist/types/queue.d.ts +15 -0
- package/dist/types/response.d.ts +183 -0
- package/dist/types/storage.d.ts +3 -0
- package/dist/types/wiki.d.ts +1 -0
- package/dist/utils/Annual.d.ts +7 -0
- package/dist/utils/Annual.js +99 -0
- package/dist/utils/Performance.d.ts +8 -0
- package/dist/utils/Performance.js +39 -0
- package/dist/utils/ProfileManager.d.ts +24 -0
- package/dist/utils/ProfileManager.js +60 -0
- package/dist/utils/ProfileMerger.d.ts +11 -0
- package/dist/utils/ProfileMerger.js +67 -0
- package/dist/utils/Ranking.d.ts +11 -0
- package/dist/utils/Ranking.js +77 -0
- package/dist/utils/Wiki.d.ts +11 -0
- package/dist/utils/Wiki.js +168 -0
- package/package.json +45 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { Merger } from '@komed3/deepmerge';
|
|
2
|
+
import { sha256 } from 'js-sha256';
|
|
3
|
+
import { hrtime } from 'node:process';
|
|
4
|
+
import { REGEX_DIACRITICS, REGEX_NOALNUM } from '../lib/regex.js';
|
|
5
|
+
import { Parser } from '../parser/Parser.js';
|
|
6
|
+
export class Utils {
|
|
7
|
+
static mergeInstances = {};
|
|
8
|
+
// --- sanitize IDs and URIs ---
|
|
9
|
+
static sanitize(value, delimiter = '-') {
|
|
10
|
+
return Parser.string(value)
|
|
11
|
+
.toLowerCase()
|
|
12
|
+
.replace(REGEX_NOALNUM, delimiter)
|
|
13
|
+
.replace(new RegExp(`[${delimiter}]{2,}`, 'g'), delimiter);
|
|
14
|
+
}
|
|
15
|
+
// --- hashing ---
|
|
16
|
+
static hash(value) {
|
|
17
|
+
return sha256(Parser.string(value).split('/').pop() ?? '');
|
|
18
|
+
}
|
|
19
|
+
static verifyHash(value, hash) {
|
|
20
|
+
return value === hash || Utils.hash(value) === hash;
|
|
21
|
+
}
|
|
22
|
+
// --- measurement ---
|
|
23
|
+
static async measure(fn) {
|
|
24
|
+
if (typeof fn !== 'function') throw new TypeError('Parameter must be a function');
|
|
25
|
+
const now = hrtime.bigint();
|
|
26
|
+
const diff = () => Number(hrtime.bigint() - now) / 1e6;
|
|
27
|
+
try {
|
|
28
|
+
return { result: await fn(), ms: diff() };
|
|
29
|
+
} catch (err) {
|
|
30
|
+
throw Object.assign(err ?? {}, { ms: diff() });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// --- date ---
|
|
34
|
+
static date(format = 'ymd') {
|
|
35
|
+
return Parser.date(undefined, format);
|
|
36
|
+
}
|
|
37
|
+
static lastMonthDay(month, year) {
|
|
38
|
+
const date = new Date(Number(year ?? new Date().getFullYear()), Number(month), 0);
|
|
39
|
+
date.setHours(23, 59, 59, 999);
|
|
40
|
+
return date;
|
|
41
|
+
}
|
|
42
|
+
// --- meta data ---
|
|
43
|
+
static metaData(obj) {
|
|
44
|
+
return { $metadata: { schemaVersion: 2, lastModified: Utils.date('iso'), ...obj } };
|
|
45
|
+
}
|
|
46
|
+
// --- aggregate from object arrays ---
|
|
47
|
+
static aggregate(arr, key, aggregator = 'first') {
|
|
48
|
+
const values = arr.map(item => item[key]).filter(v => v !== undefined);
|
|
49
|
+
if (!values.length) return undefined;
|
|
50
|
+
if (typeof aggregator === 'function') return aggregator(values);
|
|
51
|
+
const isNum = v => typeof v === 'number' && !Number.isNaN(v);
|
|
52
|
+
const sum = (acc, val) => (acc === undefined || !isNum(val) ? undefined : acc + val);
|
|
53
|
+
switch (aggregator) {
|
|
54
|
+
case 'all':
|
|
55
|
+
return values;
|
|
56
|
+
case 'first':
|
|
57
|
+
return values[0];
|
|
58
|
+
case 'last':
|
|
59
|
+
return values.at(-1);
|
|
60
|
+
case 'sum':
|
|
61
|
+
return values.reduce(sum, 0);
|
|
62
|
+
case 'min':
|
|
63
|
+
return values.reduce((acc, val) => (acc === undefined || val < acc ? val : acc), Infinity);
|
|
64
|
+
case 'max':
|
|
65
|
+
return values.reduce((acc, val) => (acc === undefined || val > acc ? val : acc), -Infinity);
|
|
66
|
+
case 'mean':
|
|
67
|
+
const s = values.reduce(sum, 0);
|
|
68
|
+
return s === undefined ? undefined : s / values.length;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// --- deep object updates ---
|
|
72
|
+
static update(operator, obj, path, n) {
|
|
73
|
+
return path
|
|
74
|
+
.split('.')
|
|
75
|
+
.reduce(
|
|
76
|
+
(curr, p, i, arr) => (
|
|
77
|
+
i < arr.length - 1 && (curr[p] ??= {}),
|
|
78
|
+
i === arr.length - 1
|
|
79
|
+
? operator === 'set'
|
|
80
|
+
? (curr[p] = n)
|
|
81
|
+
: operator === 'inc'
|
|
82
|
+
? (curr[p] = (curr[p] ?? 0) + (n ?? 1))
|
|
83
|
+
: operator === 'max'
|
|
84
|
+
? (curr[p] = Math.max(curr[p] ?? -Infinity, n))
|
|
85
|
+
: operator === 'min'
|
|
86
|
+
? (curr[p] = Math.min(curr[p] ?? Infinity, n))
|
|
87
|
+
: operator === 'append'
|
|
88
|
+
? (curr[p] = [...(curr[p] ?? []), n])
|
|
89
|
+
: operator === 'prepend'
|
|
90
|
+
? (curr[p] = [n, ...(curr[p] ?? [])])
|
|
91
|
+
: operator(curr[p], p)
|
|
92
|
+
: (curr = curr[p]),
|
|
93
|
+
curr
|
|
94
|
+
),
|
|
95
|
+
obj
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
// --- sorting arrays and objects ---
|
|
99
|
+
static sort(value, compare, objCompare) {
|
|
100
|
+
compare ??= (a, b) => (a > b ? 1 : a < b ? -1 : 0);
|
|
101
|
+
objCompare ??= (a, b) => compare(a[0], b[0]);
|
|
102
|
+
return Array.isArray(value)
|
|
103
|
+
? [...value].sort(compare)
|
|
104
|
+
: value instanceof Set
|
|
105
|
+
? new Set([...value].sort(compare))
|
|
106
|
+
: value instanceof Map
|
|
107
|
+
? new Map([...value.entries()].sort(objCompare))
|
|
108
|
+
: typeof value === 'object'
|
|
109
|
+
? Object.fromEntries(Object.entries(value).sort(objCompare))
|
|
110
|
+
: [...value].sort(compare);
|
|
111
|
+
}
|
|
112
|
+
static sortKeysDeep(value, exclude = new Set()) {
|
|
113
|
+
if (value === null || typeof value !== 'object') return value;
|
|
114
|
+
if (Array.isArray(value)) return value.map(v => Utils.sortKeysDeep(v, exclude));
|
|
115
|
+
return Object.keys(value)
|
|
116
|
+
.sort((a, b) => (exclude.has(a) || exclude.has(b) ? 0 : a.localeCompare(b)))
|
|
117
|
+
.reduce((acc, k) => {
|
|
118
|
+
acc[k] = Utils.sortKeysDeep(value[k], exclude);
|
|
119
|
+
return acc;
|
|
120
|
+
}, {});
|
|
121
|
+
}
|
|
122
|
+
// --- merging ---
|
|
123
|
+
static merge(mode, ...objects) {
|
|
124
|
+
return (Utils.mergeInstances[mode] ??= new Merger({ arrayMode: mode })).merge(
|
|
125
|
+
objects[0] ?? {},
|
|
126
|
+
...objects.slice(1)
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
static unique(arr) {
|
|
130
|
+
return [...new Set(arr.map(item => JSON.stringify(item)))].map(item => JSON.parse(item));
|
|
131
|
+
}
|
|
132
|
+
static mergeArray(target, source, mode = 'unique' /* ArrayMode.Unique */) {
|
|
133
|
+
switch (mode) {
|
|
134
|
+
case 'replace' /* ArrayMode.Replace */:
|
|
135
|
+
return source;
|
|
136
|
+
case 'keep' /* ArrayMode.Keep */:
|
|
137
|
+
return target;
|
|
138
|
+
case 'concat' /* ArrayMode.Concat */:
|
|
139
|
+
return [...target, ...source];
|
|
140
|
+
case 'unique' /* ArrayMode.Unique */:
|
|
141
|
+
return Utils.unique([...target, ...source]);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// --- queries ---
|
|
145
|
+
static queryStr(query) {
|
|
146
|
+
return new URLSearchParams(query).toString();
|
|
147
|
+
}
|
|
148
|
+
// --- search index ---
|
|
149
|
+
static buildSearchText(value, minLength = 4) {
|
|
150
|
+
return [
|
|
151
|
+
...new Set(
|
|
152
|
+
String(value)
|
|
153
|
+
.normalize('NFD')
|
|
154
|
+
.replace(REGEX_DIACRITICS, '')
|
|
155
|
+
.toLowerCase()
|
|
156
|
+
.replace(REGEX_NOALNUM, ' ')
|
|
157
|
+
.split(' ')
|
|
158
|
+
.filter(w => w.length >= minLength)
|
|
159
|
+
.filter(Boolean)
|
|
160
|
+
)
|
|
161
|
+
].join(' ');
|
|
162
|
+
}
|
|
163
|
+
static tokenSearch(text, tokens, looseMatch = false) {
|
|
164
|
+
if (!text || !tokens.length) return false;
|
|
165
|
+
return tokens[looseMatch ? 'some' : 'every'](t => text.includes(t));
|
|
166
|
+
}
|
|
167
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
TConfigObject,
|
|
3
|
+
TCronConfig,
|
|
4
|
+
TFetchConfig,
|
|
5
|
+
TJobConfig,
|
|
6
|
+
TLoggingConfig,
|
|
7
|
+
TQueueConfig,
|
|
8
|
+
TStorageConfig
|
|
9
|
+
} from '../types/config';
|
|
10
|
+
|
|
11
|
+
export interface IConfig {
|
|
12
|
+
root: string;
|
|
13
|
+
environment: string;
|
|
14
|
+
config: TConfigObject;
|
|
15
|
+
logging: TLoggingConfig;
|
|
16
|
+
job: TJobConfig;
|
|
17
|
+
cron: TCronConfig;
|
|
18
|
+
storage: TStorageConfig;
|
|
19
|
+
fetch: TFetchConfig;
|
|
20
|
+
queue: TQueueConfig;
|
|
21
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { TFetchMethod, THeader } from '../types/fetch';
|
|
2
|
+
import type { TListResponse, TProfileResponse, TResponse } from '../types/response';
|
|
3
|
+
|
|
4
|
+
export interface IFetch {
|
|
5
|
+
single<T>(url: string, method: TFetchMethod = 'get', header?: THeader): Promise<TResponse<T>>;
|
|
6
|
+
batch<T>(urls: string[], method: TFetchMethod = 'get', header?: THeader): Promise<TResponse<T>[]>;
|
|
7
|
+
wayback<T>(url: string, ts: unknown): Promise<TResponse<T>>;
|
|
8
|
+
list<T extends object>(uriLike: string, year: string): Promise<TResponse<TListResponse<T>>>;
|
|
9
|
+
profile(...uriLike: string[]): Promise<TResponse<TProfileResponse>[]>;
|
|
10
|
+
wikidata<T>(sparql: string): Promise<TResponse<T>>;
|
|
11
|
+
wikipedia<T>(query: Record<string, unknown>, lang: string = 'en'): Promise<TResponse<T>>;
|
|
12
|
+
commons<T>(query: Record<string, unknown>): Promise<TResponse<T>>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { TFilterGroup, TFilterSpecial } from '@rtbnext/schema/src/base/const';
|
|
2
|
+
import type { TFilter, TFilterList } from '@rtbnext/schema/src/model/filter';
|
|
3
|
+
|
|
4
|
+
export interface IFilter {
|
|
5
|
+
resolvePath(path: string): string | false;
|
|
6
|
+
getFilter(group: TFilterGroup, key: string): TFilter | undefined;
|
|
7
|
+
getFilterByPath(path: string): TFilter | undefined;
|
|
8
|
+
getGroup(group: TFilterGroup): Record<string, TFilter> | undefined;
|
|
9
|
+
getSpecial(special: TFilterSpecial): TFilter | undefined;
|
|
10
|
+
has(path: string, uriLike: string): boolean;
|
|
11
|
+
save(collection: Partial<TFilterList>): void;
|
|
12
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { TIndex } from '@rtbnext/schema/src/base/generic';
|
|
2
|
+
import type { TListIndex, TListIndexItem } from '@rtbnext/schema/src/model/list';
|
|
3
|
+
import type { TProfileIndex, TProfileIndexItem } from '@rtbnext/schema/src/model/profile';
|
|
4
|
+
|
|
5
|
+
export interface IIndex<I extends TIndex, T extends Map<string, I>> {
|
|
6
|
+
readonly size: number;
|
|
7
|
+
readonly values: IterableIterator<I>;
|
|
8
|
+
readonly keys: IterableIterator<string>;
|
|
9
|
+
getIndex(): T;
|
|
10
|
+
has(uriLike: string): boolean;
|
|
11
|
+
get(uriLike: string): I | undefined;
|
|
12
|
+
update(uriLike: string, data: Partial<I>, allowUpdate: boolean = true, save: boolean = true): I | false;
|
|
13
|
+
delta(items: Array<{ uriLike: string; data: Partial<I> }>, allowUpdate: boolean = true): number;
|
|
14
|
+
add(uriLike: string, data: I): I | false;
|
|
15
|
+
delete(uriLike: string): void;
|
|
16
|
+
search(query: string, looseMatch: boolean = false): T;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface IProfileIndex extends IIndex<TProfileIndexItem, TProfileIndex> {
|
|
20
|
+
find(uriLike: string): TProfileIndex;
|
|
21
|
+
move(from: string, to: string, makeAlias: boolean = true): TProfileIndexItem | false;
|
|
22
|
+
syncFromData(data: TProfileData): TProfileIndexItem | false;
|
|
23
|
+
hasAlias(aliasLike: string, uriLike?: string): string | false;
|
|
24
|
+
isAliasAvailable(aliasLike: string): boolean;
|
|
25
|
+
removeAlias(aliasLike: string): boolean;
|
|
26
|
+
addAliases(uriLike: string, ...aliases: string[]): TProfileIndexItem | false;
|
|
27
|
+
rmvAliases(uriLike: string, ...aliases: string[]): TProfileIndexItem | false;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface IListIndex extends IIndex<TListIndexItem, TListIndex> {}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { TListSnapshot } from '@rtbnext/schema/src/model/list';
|
|
2
|
+
|
|
3
|
+
import type { ISnapshot } from './snapshot';
|
|
4
|
+
import type { TListSnapshotData } from '../types/generic';
|
|
5
|
+
|
|
6
|
+
export interface IList extends ISnapshot<TListSnapshot> {
|
|
7
|
+
getUri(): string;
|
|
8
|
+
saveSnapshot(snapshot: TListSnapshotData, force: boolean = false): boolean;
|
|
9
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export interface ILogger {
|
|
2
|
+
error(msg: string, err?: Error): void;
|
|
3
|
+
errMsg(err: unknown, msg?: string): void;
|
|
4
|
+
exit(msg: string, err?: Error): never;
|
|
5
|
+
warn(msg: string, meta?: unknown): void;
|
|
6
|
+
info(msg: string, meta?: unknown): void;
|
|
7
|
+
debug(msg: string, meta?: unknown): void;
|
|
8
|
+
catch<F extends (...args: any[]) => any, R = ReturnType<F>>(
|
|
9
|
+
fn: F,
|
|
10
|
+
msg: string,
|
|
11
|
+
level: TLoggingLevel = 'error'
|
|
12
|
+
): R | undefined;
|
|
13
|
+
catchAsync<F extends (...args: any[]) => Promise<any>, R = Awaited<ReturnType<F>>>(
|
|
14
|
+
fn: F,
|
|
15
|
+
msg: string,
|
|
16
|
+
level: TLoggingLevel = 'error'
|
|
17
|
+
): Promise<R | undefined>;
|
|
18
|
+
getLogFile(date: string): string | undefined;
|
|
19
|
+
getCurrentLogFile(): string | undefined;
|
|
20
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { TAsset, TRealtime } from '@rtbnext/schema/src/base/assets';
|
|
2
|
+
import type { TChangeFlag } from '@rtbnext/schema/src/base/const';
|
|
3
|
+
import type {
|
|
4
|
+
TEducation,
|
|
5
|
+
TImage,
|
|
6
|
+
TLocation,
|
|
7
|
+
TOrganization,
|
|
8
|
+
TRelation,
|
|
9
|
+
TSelfMade
|
|
10
|
+
} from '@rtbnext/schema/src/base/generic';
|
|
11
|
+
import type { TProfileBio, TProfileFlags, TProfileInfo, TProfileName } from '@rtbnext/schema/src/model/profile';
|
|
12
|
+
|
|
13
|
+
import type { ICache } from '@/type/cache';
|
|
14
|
+
import type { TPersonListEntry, TProfileResponse } from '../types/response';
|
|
15
|
+
|
|
16
|
+
export interface IProfileParser extends ICache {
|
|
17
|
+
rawData(): TProfileResponse['person'];
|
|
18
|
+
sortedLists(): TProfileResponse['person']['personLists'];
|
|
19
|
+
uri(): string;
|
|
20
|
+
id(): string;
|
|
21
|
+
aliases(): string[];
|
|
22
|
+
name(): { name: TProfileName; family: boolean };
|
|
23
|
+
flags(): TProfileFlags;
|
|
24
|
+
info(): TProfileInfo;
|
|
25
|
+
citizenship(): string | undefined;
|
|
26
|
+
residence(): TLocation | undefined;
|
|
27
|
+
birthPlace(): TLocation | undefined;
|
|
28
|
+
education(): TEducation[];
|
|
29
|
+
selfMade(): TSelfMade;
|
|
30
|
+
philanthropyScore(): number | undefined;
|
|
31
|
+
organization(): TOrganization | undefined;
|
|
32
|
+
bio(): TProfileBio;
|
|
33
|
+
cv(): string[];
|
|
34
|
+
facts(): string[];
|
|
35
|
+
quotes(): string[];
|
|
36
|
+
related(): TRelation[];
|
|
37
|
+
media(): TImage[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface IListParser<T extends object> extends ICache {
|
|
41
|
+
rawData(): T;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface IPersonListParser extends IListParser<TPersonListEntry> {
|
|
45
|
+
uri(): string;
|
|
46
|
+
id(): string;
|
|
47
|
+
date(): string;
|
|
48
|
+
year(): number;
|
|
49
|
+
rank(): number | undefined;
|
|
50
|
+
networth(): number | undefined;
|
|
51
|
+
dropOff(): boolean | undefined;
|
|
52
|
+
name(): { name: TProfileName; family: boolean };
|
|
53
|
+
info(): Partial<TProfileInfo>;
|
|
54
|
+
residence(): TLocation | undefined;
|
|
55
|
+
selfMade(): TSelfMade | undefined;
|
|
56
|
+
philanthropyScore(): number | undefined;
|
|
57
|
+
organization(): TOrganization | undefined;
|
|
58
|
+
bio(): TProfileBio;
|
|
59
|
+
age(): number | undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface IRTBListParser extends IPersonListParser {
|
|
63
|
+
assets(): TAsset[];
|
|
64
|
+
realtime(data?: Partial<TProfileData>, prev?: string, next?: string): TRealtime | undefined;
|
|
65
|
+
rankDiff(data?: Partial<TProfileData>): { flag: TChangeFlag; rankDiff?: number };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface IBillionairesListParser extends IPersonListParser {}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ArrayMode } from '@komed3/deepmerge';
|
|
2
|
+
import type {
|
|
3
|
+
TProfileData,
|
|
4
|
+
TProfileHistory,
|
|
5
|
+
TProfileHistoryItem,
|
|
6
|
+
TProfileMetaData
|
|
7
|
+
} from '@rtbnext/schema/src/model/profile';
|
|
8
|
+
|
|
9
|
+
export interface IProfile {
|
|
10
|
+
getUri(): string;
|
|
11
|
+
getMeta(): TProfileMetaData['$metadata'];
|
|
12
|
+
schemaVersion(): number;
|
|
13
|
+
lastModified(): string;
|
|
14
|
+
lastModifiedTime(): number;
|
|
15
|
+
lastLookup(): string | undefined;
|
|
16
|
+
lastLookupTime(): number | undefined;
|
|
17
|
+
verify(id: string): boolean;
|
|
18
|
+
touch(): void;
|
|
19
|
+
touchLookup(): void;
|
|
20
|
+
needSave(): boolean;
|
|
21
|
+
save(syncIndex: boolean = true): void;
|
|
22
|
+
getData(): TProfileData;
|
|
23
|
+
setData(data: TProfileData): void;
|
|
24
|
+
updateData(data: Partial<TProfileData>, mode: ArrayMode = ArrayMode.Replace): void;
|
|
25
|
+
getHistory(): TProfileHistory;
|
|
26
|
+
setHistory(history: TProfileHistory): void;
|
|
27
|
+
addHistory(row: TProfileHistoryItem): void;
|
|
28
|
+
mergeHistory(history: TProfileHistory): void;
|
|
29
|
+
move(uriLike: string, makeAlias: boolean = true): boolean;
|
|
30
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { TQueueItem, TQueueOptions } from '../types/queue';
|
|
2
|
+
|
|
3
|
+
export interface IQueue {
|
|
4
|
+
readonly size: number;
|
|
5
|
+
getQueue(): TQueueItem[];
|
|
6
|
+
getByKey(key: string): TQueueItem | undefined;
|
|
7
|
+
hasKey(key: string): boolean;
|
|
8
|
+
getByUri(uriLike: string): TQueueItem[];
|
|
9
|
+
hasUri(uriLike: string): boolean;
|
|
10
|
+
clear(): void;
|
|
11
|
+
add(opt: TQueueOptions, save: boolean = true): boolean;
|
|
12
|
+
addMany(items: TQueueOptions[]): number;
|
|
13
|
+
removeByKey(key: string): boolean;
|
|
14
|
+
remove(...uriLike: string[]): number;
|
|
15
|
+
next(n: number = 1): TQueueItem[];
|
|
16
|
+
nextUri(n: number = 1): string[];
|
|
17
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { TSnapshot } from '@rtbnext/schema/src/base/generic';
|
|
2
|
+
|
|
3
|
+
export interface ISnapshot<T extends TSnapshot> {
|
|
4
|
+
getDates(): string[];
|
|
5
|
+
hasDate(dateLike: string): boolean;
|
|
6
|
+
firstDate(): string | undefined;
|
|
7
|
+
latestDate(): string | undefined;
|
|
8
|
+
nearestDate(dateLike: string): string | undefined;
|
|
9
|
+
datesInRange(from: string, to: string): string[];
|
|
10
|
+
datesInYear(year: string | number): string[];
|
|
11
|
+
firstInYear(year: string | number): string | undefined;
|
|
12
|
+
latestInYear(year: string | number): string | undefined;
|
|
13
|
+
getSnapshot(dateLike: string, exactMatch: boolean = true): T | false;
|
|
14
|
+
getLatest(): T | false;
|
|
15
|
+
saveSnapshot(snapshot: T, force: boolean = false): boolean;
|
|
16
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { TStatsGroup as TStatsGroupType } from '@rtbnext/schema/src/base/const';
|
|
2
|
+
import type { TListSnapshot } from '@rtbnext/schema/src/model/list';
|
|
3
|
+
import type {
|
|
4
|
+
TDBStats,
|
|
5
|
+
TDBStatsData,
|
|
6
|
+
TGlobalStats,
|
|
7
|
+
TGlobalStatsData,
|
|
8
|
+
THistory,
|
|
9
|
+
TProfileStats,
|
|
10
|
+
TProfileStatsData,
|
|
11
|
+
TScatter,
|
|
12
|
+
TScatterItem,
|
|
13
|
+
TStatsGroup,
|
|
14
|
+
TStatsGroupItem,
|
|
15
|
+
TTop10,
|
|
16
|
+
TTop10Data,
|
|
17
|
+
TTop10List,
|
|
18
|
+
TWealthStats,
|
|
19
|
+
TWealthStatsData
|
|
20
|
+
} from '@rtbnext/schema/src/model/stats';
|
|
21
|
+
|
|
22
|
+
export interface IStats {
|
|
23
|
+
getGlobalStats(): TGlobalStats;
|
|
24
|
+
getProfileStats(): TProfileStats;
|
|
25
|
+
getWealthStats(): TWealthStats;
|
|
26
|
+
getScatter(): TScatter;
|
|
27
|
+
getTop10(): TTop10;
|
|
28
|
+
getDBStats(): TDBStats;
|
|
29
|
+
getHistory(): THistory;
|
|
30
|
+
getGroupedStatsIndex(group: TStatsGroupType): TStatsGroup<string>['index'];
|
|
31
|
+
getGroupedStatsHistory(group: TStatsGroupType, key: string): THistory;
|
|
32
|
+
getGroupedStats(group: TStatsGroupType): TStatsGroup<string>;
|
|
33
|
+
setGlobalStats(data: TGlobalStatsData): boolean;
|
|
34
|
+
setProfileStats(data: TProfileStatsData): boolean;
|
|
35
|
+
setWealthStats(data: TWealthStatsData): boolean;
|
|
36
|
+
setScatter(data: TScatterItem[]): boolean;
|
|
37
|
+
setTop10(data: TTop10Data): boolean;
|
|
38
|
+
updateTop10(key: string, list: TTop10List): boolean;
|
|
39
|
+
setDBStats(data: TDBStatsData): boolean;
|
|
40
|
+
setGroupedStats<T extends string = string>(group: TStatsGroupType, raw: Record<T, TStatsGroupItem>): boolean;
|
|
41
|
+
updateHistory(data: Partial<TGlobalStats>): boolean;
|
|
42
|
+
generateWealthStats(scatter: TScatterItem[]): boolean;
|
|
43
|
+
generateTop10Entry(snapshot: TListSnapshot): boolean;
|
|
44
|
+
generateDBStats(): boolean;
|
|
45
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface IStorage {
|
|
2
|
+
readonly root: string;
|
|
3
|
+
exists(path: string): boolean;
|
|
4
|
+
assertPath(path: string): void | never;
|
|
5
|
+
ensurePath(path: string, isDir: boolean = false): void;
|
|
6
|
+
stat(path: string): Stats | false;
|
|
7
|
+
scanDir(path: string, ext: string[] = ['json', 'csv']): string[];
|
|
8
|
+
readJSON<T extends object>(path: string): T | false;
|
|
9
|
+
writeJSON<T extends object>(path: string, content: T): boolean;
|
|
10
|
+
readCSV<T extends any[]>(path: string): T | false;
|
|
11
|
+
writeCSV<T extends any[]>(path: string, content: T): boolean;
|
|
12
|
+
appendCSV<T extends any[]>(path: string, content: T, nl: boolean = true): boolean;
|
|
13
|
+
datedCSV<T extends any[]>(path: string, content: T, force: boolean = false): boolean;
|
|
14
|
+
remove(path: string, force: boolean = true): boolean;
|
|
15
|
+
move(from: string, to: string, force: boolean = false): boolean;
|
|
16
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TAliasJobOptions, TCommandJob } from '../types/job';
|
|
3
|
+
export declare class AliasJob extends Job<TAliasJobOptions> {
|
|
4
|
+
private static readonly index;
|
|
5
|
+
constructor(options: TAliasJobOptions);
|
|
6
|
+
run(): Promise<void>;
|
|
7
|
+
static readonly command: TCommandJob;
|
|
8
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { ProfileIndex } from '../model/ProfileIndex.js';
|
|
3
|
+
import { Parser } from '../parser/Parser.js';
|
|
4
|
+
export class AliasJob extends Job {
|
|
5
|
+
static index = ProfileIndex.getInstance();
|
|
6
|
+
constructor(options) {
|
|
7
|
+
super(options, 'alias');
|
|
8
|
+
}
|
|
9
|
+
// --- job runner ---
|
|
10
|
+
async run() {
|
|
11
|
+
await this.protect(async () => {
|
|
12
|
+
for (const a of this.options.removeGlobal ?? []) AliasJob.index.removeAlias(a);
|
|
13
|
+
if (this.options.profile) {
|
|
14
|
+
this.options.remove?.length && AliasJob.index.rmvAliases(this.options.profile, ...this.options.remove);
|
|
15
|
+
this.options.add?.length && AliasJob.index.addAliases(this.options.profile, ...this.options.add);
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
// --- command definition ---
|
|
20
|
+
static command = {
|
|
21
|
+
id: 'alias',
|
|
22
|
+
desc: 'Manage profile aliases',
|
|
23
|
+
options: [
|
|
24
|
+
{ name: '-p, --profile <URI>', desc: 'The profile URI whose aliases will be edited' },
|
|
25
|
+
{
|
|
26
|
+
name: '--add <URIs>',
|
|
27
|
+
desc: 'Aliases to be added to the profile index item (comma-separated)',
|
|
28
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
name: '--remove <URIs>',
|
|
32
|
+
desc: 'Aliases to be removed from the profile index item (comma-separated)',
|
|
33
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
name: '-g, --remove-global <URIs>',
|
|
37
|
+
desc: 'Aliases to remove from any profile index item (comma-separated)',
|
|
38
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
};
|
|
42
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TAnnualJobOptions, TCommandJob, TCronJob } from '../types/job';
|
|
3
|
+
export declare class AnnualJob extends Job<TAnnualJobOptions> {
|
|
4
|
+
constructor(options: TAnnualJobOptions);
|
|
5
|
+
run(): Promise<void>;
|
|
6
|
+
static readonly command: TCommandJob;
|
|
7
|
+
static readonly cron: TCronJob<TAnnualJobOptions>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import { Parser } from '../parser/Parser.js';
|
|
3
|
+
import { Annual } from '../utils/Annual.js';
|
|
4
|
+
export class AnnualJob extends Job {
|
|
5
|
+
constructor(options) {
|
|
6
|
+
super(options, 'annual');
|
|
7
|
+
}
|
|
8
|
+
// --- job runner ---
|
|
9
|
+
async run() {
|
|
10
|
+
await this.protect(async () => {
|
|
11
|
+
if (this.options.profiles?.length)
|
|
12
|
+
for (const uri of this.options.profiles) Annual.generate(uri, this.options.year);
|
|
13
|
+
else Annual.generateAll(this.options.year);
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
// --- command definition ---
|
|
17
|
+
static command = {
|
|
18
|
+
id: 'annual',
|
|
19
|
+
desc: 'Generate annual records for profiles',
|
|
20
|
+
options: [
|
|
21
|
+
{
|
|
22
|
+
name: '-y, --year <YEAR>',
|
|
23
|
+
desc: 'The year for which to generate annual records',
|
|
24
|
+
parser: v => Parser.number(v),
|
|
25
|
+
required: true
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: '-p, --profiles <URIs>',
|
|
29
|
+
desc: 'Process specific profiles by URI (comma-separated)',
|
|
30
|
+
parser: v => Parser.list(v, 'string', ',')
|
|
31
|
+
}
|
|
32
|
+
]
|
|
33
|
+
};
|
|
34
|
+
// --- cron job definition ---
|
|
35
|
+
static cron = [
|
|
36
|
+
{
|
|
37
|
+
cronexpr: '40 0 1 1 *', // run at 0:40 AM on January 1st every year
|
|
38
|
+
options: date => ({ year: date.getFullYear() - 1 })
|
|
39
|
+
}
|
|
40
|
+
];
|
|
41
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Job } from '../abstract/Job.js';
|
|
2
|
+
import type { TCommandJob, TCronJob, TListJobOptions } from '../types/job';
|
|
3
|
+
export declare class ListJob extends Job<TListJobOptions> {
|
|
4
|
+
private static readonly fetch;
|
|
5
|
+
private static readonly profileQueue;
|
|
6
|
+
private static readonly queue;
|
|
7
|
+
constructor(options?: TListJobOptions);
|
|
8
|
+
run(): Promise<void>;
|
|
9
|
+
static readonly command: TCommandJob;
|
|
10
|
+
static readonly cron: TCronJob;
|
|
11
|
+
}
|