@rtbnext/core 2.0.0-alpha.1

package/dist/types/response.d.ts

@@ -0,0 +1,183 @@
+export type TResponse<T> = {
+  success: boolean;
+  data?: T;
+  error?: string;
+  statusCode?: number;
+  duration: number;
+  retries: number;
+};
+
+export type TWaybackResponse = {
+  archived_snapshots: { closest?: { status: string; available: boolean; url: string; timestamp: string } };
+};
+
+export type TProfileResponse = {
+  person: {
+    naturalId: string;
+    name: string;
+    listImages?: Array<{
+      image: string;
+      uri: string;
+      caption: string;
+      description: string;
+      title: string;
+      credit: string;
+    }>;
+    geoLocation?: { longitude: number; latitude: number };
+    relatedEntities?: Array<{ name: string; type: string; uri?: string; relationshipType?: string }>;
+    uri: string;
+    dropOff: boolean;
+    uris: string[];
+    industries: string[];
+    embargo?: boolean;
+    firstName?: string;
+    lastName?: string;
+    birthDate?: number;
+    finalWorth: number;
+    finalWorthDate: number;
+    stateProvince?: string;
+    city?: string;
+    numberOfChildren?: number;
+    maritalStatus?: 'M' | 'F';
+    source: string;
+    title?: string;
+    organization?: string;
+    asianFormat?: 'N' | 'Y';
+    personLists: Array<{
+      listUri: string;
+      name: string;
+      listDescription?: string;
+      rank?: number;
+      finalWorth?: number;
+      timestamp: number | string;
+      financialAssets?: Array<{
+        exchange: string;
+        ticker: string;
+        companyName: string;
+        numberOfShares?: number;
+        sharePrice: number;
+        exchangeRate?: number;
+        currentPrice?: number;
+      }>;
+      date?: number | string;
+      estWorthPrev?: number;
+      privateAssetsWorth?: number;
+      familyList?: boolean;
+      archivedWorth?: number;
+      bios?: string[];
+      abouts?: string[];
+      philanthropyScore?: number;
+    }>;
+    educations?: Array<{ school: string; degree: string }>;
+    selfMade?: boolean;
+    gender: string;
+    countryOfCitizenship: string;
+    countryOfResidence?: string;
+    birthCity?: string;
+    birthState?: string;
+    birthCountry?: string;
+    deceased: boolean;
+    selfMadeType?: string;
+    selfMadeRank?: number;
+    quote?: string;
+  };
+};
+
+export type TPersonListEntry = {
+  naturalId: string;
+  name: string;
+  listDescription?: string;
+  timestamp: number;
+  date?: number;
+  year: number;
+  listUri: string;
+  uri: string;
+  position?: number;
+  rank?: number;
+  previousRank?: number;
+  finalWorth?: number;
+  estWorthPrev?: number;
+  privateAssetsWorth?: number;
+  archivedWorth?: number;
+  change?: number;
+  changePercent?: number;
+  person?: { name?: string; uri?: string };
+  personName: string;
+  lastName?: string;
+  birthDate?: number;
+  gender?: 'M' | 'F';
+  countryOfCitizenship?: string;
+  country?: string;
+  state?: string;
+  city?: string;
+  source?: string;
+  industries?: string[];
+  selfMadeRank?: number;
+  philanthropyScore?: number;
+  organization?: string;
+  title?: string;
+  bios?: string[];
+  abouts?: string[];
+  financialAssets?: Array<{
+    exchange: string;
+    ticker: string;
+    companyName: string;
+    numberOfShares?: number;
+    sharePrice?: number;
+    currencyCode?: string;
+    exchangeRate?: number;
+    currentPrice?: number;
+  }>;
+  csfDisplayFields: string[];
+};
+
+export type TListResponse<T extends object> = { personList: { count: number; personsLists: Array<T> } };
+
+export type TWikidataProp =
+  'gender' | 'birthdate' | 'article' | 'image' | 'iso2' | 'occupation' | 'employer' | 'ownerOf' | 'netWorth';
+
+export type TWikidataResponseItem = { item: { value: string }; itemLabel: { value: string; xmlLang: string } } & {
+  [K in TWikidataProp]?: { value: string };
+};
+
+export type TWikidataResponse = { results: { bindings: TWikidataResponseItem[] } };
+
+export type TWikipediaResponse = {
+  query: {
+    pages: Array<{
+      pageid: number;
+      title: string;
+      extract?: string;
+      touched: string;
+      lastrevid: number;
+      pageimage?: string;
+      pageprops?: { [K in 'defaultsort' | 'wikibase-shortdesc' | 'wikibase_item']?: string };
+    }>;
+  };
+};
+
+export type TCommonsResponse = {
+  query: {
+    pages: Array<{
+      imageinfo?: Array<{
+        url: string;
+        descriptionurl: string;
+        thumburl?: string;
+        responsiveUrls?: Record<string, string>;
+        extmetadata?: {
+          [
+            K in
+              | 'Artist'
+              | 'Attribution'
+              | 'Credit'
+              | 'DateTime'
+              | 'DateTimeOriginal'
+              | 'ImageDescription'
+              | 'LicenseShortName'
+              | 'UsageTerms'
+          ]?: { value?: string };
+        };
+      }>;
+    }>;
+  };
+};

package/dist/types/storage.d.ts

@@ -0,0 +1,3 @@
+export type TStorageRWType = 'raw' | 'json' | 'csv';
+
+export type TStorageWOptions = { append: boolean; nl: boolean };

package/dist/types/wiki.d.ts

@@ -0,0 +1 @@
+export type TWikidata = { qid: string; confidence: number; article?: string; image?: string };

package/dist/utils/Annual.d.ts

@@ -0,0 +1,7 @@
+export declare class Annual {
+  private static readonly handler;
+  private static aggregate;
+  private static record;
+  static generate(uriLike: string, year: number): boolean;
+  static generateAll(year: number): number;
+}

package/dist/utils/Annual.js

@@ -0,0 +1,99 @@
+import { log } from '../core/Logger.js';
+import { Profile } from '../model/Profile.js';
+import { ProfileIndex } from '../model/ProfileIndex.js';
+import { Parser } from '../parser/Parser.js';
+export class Annual {
+  static handler = {
+    rank: {
+      sort: (a, b) => a - b,
+      parse: v => Parser.number(v, 0),
+      flag: raw => {
+        if (raw.networth?.length && raw.networth[0] < 1000) return 'dropoff';
+        if (raw.prevRank === undefined) return raw.hadBefore ? 'returned' : 'new';
+        const diff = raw.prevRank - raw.rank[0];
+        return diff > 0 ? 'up' : diff < 0 ? 'down' : 'unchanged';
+      }
+    },
+    networth: {
+      sort: (a, b) => b - a,
+      parse: Parser.money,
+      flag: raw => {
+        if (raw.prevNetworth === undefined) return 'unknown';
+        const diff = raw.networth[0] - raw.prevNetworth;
+        return diff > 0 ? 'up' : diff < 0 ? 'down' : 'unchanged';
+      }
+    }
+  };
+  static aggregate(history, year) {
+    const rank = [],
+      networth = [];
+    let prevRank = undefined;
+    let prevNetworth = undefined;
+    let hadBefore = false;
+    for (let i = history.length - 1; i >= 0; i--) {
+      const [date, rnk, ntw] = history[i];
+      const y = +date.slice(0, 4);
+      if (y > year) continue;
+      if (y === year - 1) {
+        prevRank ??= rnk;
+        prevNetworth ??= ntw;
+      }
+      if (y < year) hadBefore = true;
+      if (hadBefore) break;
+      if (y !== year) continue;
+      rank.push(rnk);
+      networth.push(ntw);
+    }
+    return { rank, networth, prevRank, prevNetworth, hadBefore };
+  }
+  static record(raw, type) {
+    const data = raw[type],
+      len = data?.length ?? 0;
+    if (len === 0) return;
+    const { sort, parse, flag } = Annual.handler[type];
+    const first = data[len - 1],
+      last = data[0];
+    const sorted = [...data].sort(sort);
+    const max = sorted[0],
+      min = sorted[len - 1];
+    const mean = sorted.reduce((sum, v) => sum + v, 0) / len;
+    const idx = Math.floor(len / 2);
+    const median = len % 2 === 0 ? (sorted[idx - 1] + sorted[idx]) / 2 : sorted[idx];
+    return {
+      first: parse(first),
+      last: parse(last),
+      max: parse(max),
+      min: parse(min),
+      diff: parse(sort(first, last)),
+      flag: flag(raw),
+      mean: parse(mean),
+      median: parse(median),
+      range: parse(Math.abs(max - min)),
+      stdDev: parse(Math.sqrt(sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / len))
+    };
+  }
+  static generate(uriLike, year) {
+    return (
+      log.catch(() => {
+        const profile = Profile.get(uriLike);
+        if (!profile) throw new Error(`Profile not found for ${uriLike}`);
+        const history = profile.getHistory();
+        if (!history?.length) throw new Error(`No history found for ${uriLike}`);
+        const raw = Annual.aggregate(history, year);
+        const item = { year, rank: Annual.record(raw, 'rank'), networth: Annual.record(raw, 'networth') };
+        const annual = profile.getData().annual ?? [];
+        const idx = annual.findIndex(a => a.year === year);
+        if (idx >= 0) annual[idx] = item;
+        else annual.push(item);
+        profile.updateData({ annual });
+        profile.save();
+        return true;
+      }, `Failed to generate annual report for ${uriLike} @ ${year}`) ?? false
+    );
+  }
+  static generateAll(year) {
+    let count = 0;
+    for (const uri of ProfileIndex.getInstance().keys) if (Annual.generate(uri, year)) count++;
+    return count;
+  }
+}

package/dist/utils/Performance.d.ts

@@ -0,0 +1,8 @@
+import type { TExtrema, TPerformance, TReturns } from '@rtbnext/schema/src/base/assets';
+import type { TProfileHistory } from '@rtbnext/schema/src/model/profile';
+export declare class Performance {
+  private static readonly RETURNS;
+  static getProfileExtrema(history: TProfileHistory): TExtrema;
+  static generateProfileReturns(history: TProfileHistory): TReturns;
+  static generateProfilePerformance(history: TProfileHistory): TPerformance;
+}

package/dist/utils/Performance.js

@@ -0,0 +1,39 @@
+import { Parser } from '../parser/Parser.js';
+export class Performance {
+  static RETURNS = { week: 7, month: 30, quarter: 90, halfYear: 182, year: 365, twoYear: 730, fiveYear: 1825 };
+  // --- extrema ---
+  static getProfileExtrema(history) {
+    const map = ([date, rank, networth]) => ({ date, rank: Parser.number(rank), networth: Parser.money(networth) });
+    const [low, high] = history.reduce(
+      ([l, h], row) => [!l || row[2] < l[2] ? row : l, !h || row[2] > h[2] ? row : h],
+      []
+    );
+    return { low: low && map(low), high: high && map(high) };
+  }
+  // --- returns ---
+  static generateProfileReturns(history) {
+    const latest = history.at(-1);
+    if (!latest) return {};
+    const result = {};
+    const now = Date.parse(latest[0]);
+    const networth = latest[2];
+    const returns = Object.entries(Performance.RETURNS);
+    let remaining = Object.keys(returns).length;
+    for (const item of history.toReversed()) {
+      const days = (now - Date.parse(item[0])) / 86400000;
+      for (const [key, target] of returns) {
+        if (result[key] || days < target) continue;
+        result[key] = {
+          value: Parser.money(networth - item[2]),
+          percent: Parser.pct((networth / (item[2] ?? 1)) * 100)
+        };
+        if (--remaining === 0) return result;
+      }
+    }
+    return result;
+  }
+  // --- performance ---
+  static generateProfilePerformance(history) {
+    return { extrema: Performance.getProfileExtrema(history), returns: Performance.generateProfileReturns(history) };
+  }
+}

package/dist/utils/ProfileManager.d.ts

@@ -0,0 +1,24 @@
+import type { TProfileData } from '@rtbnext/schema/src/model/profile';
+import type { IProfile } from '../interfaces/profile';
+import type {
+  TProfileUpdateMode,
+  TProfileLookupResult,
+  TProfileOperation,
+  TProfileProcessResult
+} from '../types/profile';
+import type { TQueueOptions } from '../types/queue';
+export declare class ProfileManager {
+  private static readonly index;
+  private static execute;
+  static lookup(uriLike: string, id?: string, profileData?: Partial<TProfileData>): TProfileLookupResult;
+  static determineAction(lookup: TProfileLookupResult): TProfileOperation;
+  static process(
+    uriLike: string,
+    id: string,
+    profileData: Partial<TProfileData>,
+    mode?: TProfileUpdateMode,
+    makeAlias?: boolean,
+    touchLookup?: boolean
+  ): TProfileProcessResult;
+  static updateQueue(queue: TQueueOptions[], profile: IProfile, action: TProfileOperation, th?: number): void;
+}

package/dist/utils/ProfileManager.js

@@ -0,0 +1,60 @@
+import { Profile } from '../model/Profile.js';
+import { ProfileIndex } from '../model/ProfileIndex.js';
+import { ProfileMerger } from './ProfileMerger.js';
+export class ProfileManager {
+  static index = ProfileIndex.getInstance();
+  static execute(profile, action, uriLike, profileData, mode = 'updateData', makeAlias = true, touchLookup = false) {
+    if (!profile) return Profile.create(uriLike, profileData);
+    if (touchLookup) profile.touchLookup();
+    switch (action) {
+      case 'update':
+        if (mode !== 'createOnly') {
+          profile[mode](profileData);
+          profile.save();
+        }
+        break;
+      case 'move':
+        if (mode !== 'createOnly') {
+          profile[mode](profileData);
+          profile.move(uriLike, makeAlias);
+        } else if (makeAlias) {
+          this.index.addAliases(profile.getUri(), uriLike);
+        }
+        break;
+    }
+    return profile;
+  }
+  // --- lookup profile by URI and ID, or find a similar matching profile ---
+  static lookup(uriLike, id, profileData) {
+    let profile = Profile.find(uriLike);
+    const isExisting = profile && profile.verify(id ?? '');
+    const isSimilar = !isExisting && !!(profile = ProfileMerger.findMatching(profileData ?? {})[0]);
+    return { profile, isExisting, isSimilar };
+  }
+  // --- determine action based on profile lookup ---
+  static determineAction(lookup) {
+    return lookup.isExisting ? 'update' : lookup.isSimilar ? 'move' : 'create';
+  }
+  // --- perform profile operation ---
+  static process(uriLike, id, profileData, mode = 'updateData', makeAlias = true, touchLookup = false) {
+    const lookup = this.lookup(uriLike, id, profileData);
+    const action = this.determineAction(lookup);
+    const profile = this.execute(lookup.profile, action, uriLike, profileData, mode, makeAlias, touchLookup);
+    return { profile, action, success: !!profile };
+  }
+  // --- add queue item based on action ---
+  static updateQueue(queue, profile, action, th) {
+    const uriLike = profile.getUri();
+    switch (action) {
+      case 'update':
+        if (!th || (profile.lastLookupTime() ?? 0) < th) queue.push({ uriLike });
+        break;
+      case 'move':
+        queue.push({ uriLike, prio: 5 });
+        break;
+      case 'create':
+        queue.push({ uriLike, prio: 10 });
+        break;
+    }
+  }
+}

package/dist/utils/ProfileMerger.d.ts

@@ -0,0 +1,11 @@
+import type { TProfileData } from '@rtbnext/schema/src/model/profile';
+import type { IProfile } from '../interfaces/profile';
+export declare class ProfileMerger {
+  private static readonly cmp;
+  private static readonly index;
+  private static similarURIs;
+  static mergeableProfiles(target: Partial<TProfileData>, source: Partial<TProfileData>): boolean;
+  static findMatching(data: Partial<TProfileData>): IProfile[];
+  static mergeProfiles(target: IProfile, source: IProfile, force?: boolean, makeAlias?: boolean): boolean;
+  static listCandidates(...uriLike: string[]): Record<string, string[]>;
+}

package/dist/utils/ProfileMerger.js

@@ -0,0 +1,67 @@
+import { CmpStrAsync } from 'cmpstr';
+import { REGEX_URI_CLEANUP } from '../lib/regex.js';
+import { Profile } from '../model/Profile.js';
+import { ProfileIndex } from '../model/ProfileIndex.js';
+CmpStrAsync.filter.add('input', 'normalizeUri', uri => uri.replace(REGEX_URI_CLEANUP, ''));
+export class ProfileMerger {
+  static cmp = CmpStrAsync.create({ metric: 'dice', safeEmpty: true });
+  static index = ProfileIndex.getInstance();
+  // --- helper ---
+  static similarURIs(uri) {
+    const revUri = uri.split('-').reverse().join('-');
+    const keys = [...ProfileMerger.index.keys];
+    const res = new Set([
+      ...ProfileMerger.cmp.match(keys, uri, 0.9).map(i => i.source),
+      ...ProfileMerger.cmp.match(keys, revUri, 0.8).map(i => i.source)
+    ]);
+    res.delete(uri);
+    return [...res];
+  }
+  // --- check mergeable profiles ---
+  static mergeableProfiles(target, source) {
+    if (target.id === source.id) return true;
+    for (const match of ['gender', 'birthDate', 'birthPlace', 'citizenship', 'industry'])
+      if (
+        target.info &&
+        match in target.info &&
+        source.info &&
+        match in source.info &&
+        JSON.stringify(target.info[match]) !== JSON.stringify(source.info[match])
+      )
+        return false;
+    return true;
+  }
+  // --- find matching profiles ---
+  static findMatching(data) {
+    if (!data.id || !data.uri) return [];
+    const res = [];
+    for (const uri of ProfileMerger.similarURIs(data.uri)) {
+      const profile = Profile.get(uri);
+      if (profile && ProfileMerger.mergeableProfiles(profile.getData(), data)) res.push(profile);
+    }
+    return res;
+  }
+  // --- merge profiles ---
+  static mergeProfiles(target, source, force = false, makeAlias = true) {
+    if (!force && !ProfileMerger.mergeableProfiles(target.getData(), source.getData())) return false;
+    target.updateData(source.getData(), 'unique' /* ArrayMode.Unique */);
+    target.mergeHistory(source.getHistory());
+    target.save();
+    return (
+      Profile.delete(source.getUri()) &&
+      (!makeAlias ? true : !!ProfileMerger.index.addAliases(target.getUri(), source.getUri()))
+    );
+  }
+  // --- list matching candidates ---
+  static listCandidates(...uriLike) {
+    if (!uriLike.length) return {};
+    const res = {};
+    for (const uri of uriLike) {
+      const profile = Profile.get(uri);
+      if (!profile) continue;
+      const matches = this.findMatching(profile.getData());
+      res[profile.getUri()] = matches.map(m => m.getUri());
+    }
+    return res;
+  }
+}

package/dist/utils/Ranking.d.ts

@@ -0,0 +1,11 @@
+import type { TRanking } from '@rtbnext/schema/src/base/assets';
+import type { TProfileResponse } from '../types/response';
+export declare class Ranking {
+  private static readonly queue;
+  static generateProfileRanking(
+    sortedLists: TProfileResponse['person']['personLists'],
+    rankingData?: TRanking[],
+    history?: boolean,
+    addQueue?: boolean
+  ): TRanking[];
+}

package/dist/utils/Ranking.js

@@ -0,0 +1,77 @@
+import { ListQueue } from '../core/Queue.js';
+import { List } from '../model/List.js';
+import { Parser } from '../parser/Parser.js';
+export class Ranking {
+  static queue = ListQueue.getInstance();
+  static generateProfileRanking(sortedLists, rankingData = [], history = true, addQueue = true) {
+    const lists = new Map(rankingData.map(r => [r.list, r]));
+    const entries = new Map();
+    const names = new Map();
+    const queue = [];
+    // --- prep new entries from sorted lists ---
+    for (const { listUri, name, listDescription, date, timestamp, rank, finalWorth } of sortedLists) {
+      if (['rtb', 'rtrl'].includes(listUri)) continue;
+      const item = Parser.container({
+        date: { value: date ?? timestamp, type: 'date', args: ['ymd'], strict: false },
+        rank: { value: rank, type: 'number' },
+        networth: { value: finalWorth, type: 'money' }
+      });
+      if (!entries.has(listUri)) entries.set(listUri, []);
+      entries.get(listUri).push(item);
+      names.set(listUri, { name, desc: listDescription });
+    }
+    // --- merge existing and new entries ---
+    const allListUris = new Set([...lists.keys(), ...entries.keys()]);
+    const result = [];
+    // --- generate final rankings ---
+    for (const listUri of allListUris) {
+      const existing = lists.get(listUri);
+      const news = entries.get(listUri) ?? [];
+      const allItems = [];
+      // --- add existing entry ---
+      if (existing) {
+        allItems.push({
+          date: existing.date,
+          rank: existing.rank,
+          networth: existing.networth,
+          prev: existing.prev,
+          next: existing.next
+        });
+        if (existing.history) allItems.push(...existing.history);
+      }
+      // --- add new entries and sort by date ---
+      allItems.push(...news);
+      allItems.sort((a, b) => b.date.localeCompare(a.date));
+      const main = allItems[0];
+      let historyItems = [];
+      // --- prepare history ---
+      if (history) historyItems = allItems.slice(1);
+      else if (existing?.history) historyItems = [...existing.history];
+      historyItems.sort((a, b) => b.date.localeCompare(a.date));
+      // --- create final ranking entry ---
+      const name = existing?.name ?? names.get(listUri)?.name ?? listUri;
+      const ranking = {
+        list: listUri,
+        name,
+        date: main.date,
+        rank: main.rank,
+        prev: main.prev,
+        next: main.next,
+        networth: main.networth,
+        history: historyItems
+      };
+      result.push(ranking);
+      // --- queue list for future processing if needed ---
+      if (addQueue && main.rank && main.networth) {
+        const list = List.get(listUri);
+        if (!list || !list.hasDate(main.date))
+          queue.push({
+            uriLike: listUri,
+            args: { name, desc: names.get(listUri)?.desc, year: main.date.split('-')[0] }
+          });
+      }
+    }
+    if (addQueue && queue.length) this.queue.addMany(queue);
+    return result;
+  }
+}

package/dist/utils/Wiki.d.ts

@@ -0,0 +1,11 @@
+import type { TImage, TWiki } from '@rtbnext/schema/src/base/generic';
+import type { TProfileData } from '@rtbnext/schema/src/model/profile';
+import type { TWikidata } from '../types/wiki';
+export declare class Wiki {
+  private static readonly fetch;
+  private static scoreWDItem;
+  static queryWikidata(data: Partial<TProfileData>): Promise<TWikidata | undefined>;
+  static queryCommonsImage(title: string): Promise<TImage | undefined>;
+  static queryWikiPage(title: string, qid?: string, image?: TImage, confidence?: number): Promise<TWiki | undefined>;
+  static fromProfileData(data: Partial<TProfileData>): Promise<TWiki | undefined>;
+}