@rtbnext/core 2.0.0-alpha.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rtbnext by Paul Köhler (komed3)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,11 @@
+# @rtbnext/core
+
+Internal core package for the rtbnext project.
+
+Implements the [rtbnext database](#) model and provides services for accessing data via the [rtbnext API](#). It is based on and integrates the [@rtbnext/schema](https://github.com/rtbnext/schema).
+
+It serves as the backbone across all services in the project.
+
+**Copyright © 2026 rtbnext**  
+Created and maintained by Paul Köhler (komed3).  
+Licensed under the MIT License.

package/dist/abstract/Cache.d.ts

@@ -0,0 +1,9 @@
+import type { ICache } from '../interfaces/cache';
+export declare abstract class Cache implements ICache {
+  protected cachedData: Map<string, unknown>;
+  protected cache<T = unknown>(key: string, fn: () => T): T;
+  get size(): number;
+  get<T = unknown>(key: string): T | undefined;
+  has(key: string): boolean;
+  clear(): void;
+}

package/dist/abstract/Cache.js

@@ -0,0 +1,19 @@
+export class Cache {
+  cachedData = new Map();
+  cache(key, fn) {
+    if (!this.cachedData.has(key)) this.cachedData.set(key, fn());
+    return this.cachedData.get(key);
+  }
+  get size() {
+    return this.cachedData.size;
+  }
+  get(key) {
+    return this.cachedData.get(key);
+  }
+  has(key) {
+    return this.cachedData.has(key);
+  }
+  clear() {
+    this.cachedData.clear();
+  }
+}

package/dist/abstract/Index.d.ts

@@ -0,0 +1,22 @@
+import type { TIndex } from '@rtbnext/schema/src/base/generic';
+import type { IIndex } from '../interfaces/index';
+export declare abstract class Index<I extends TIndex, T extends Map<string, I>> implements IIndex<I, T> {
+  protected static readonly storage: import('../interfaces/storage').IStorage;
+  protected readonly type: 'profile' | 'list';
+  protected readonly path: string;
+  protected index: T;
+  protected constructor(type: 'profile' | 'list', path: string);
+  protected loadIndex(): T;
+  protected saveIndex(): void;
+  get size(): number;
+  get values(): IterableIterator<I>;
+  get keys(): IterableIterator<string>;
+  getIndex(): T;
+  has(uriLike: string): boolean;
+  get(uriLike: string): I | undefined;
+  update(uriLike: string, data: Partial<I>, allowUpdate?: boolean, save?: boolean): I | false;
+  delta(items: Array<{ uriLike: string; data: Partial<I> }>, allowUpdate?: boolean): number;
+  add(uriLike: string, data: I): I | false;
+  delete(uriLike: string): void;
+  search(query: string, looseMatch?: boolean): T;
+}

package/dist/abstract/Index.js

@@ -0,0 +1,81 @@
+import { log } from '../core/Logger.js';
+import { Storage } from '../core/Storage.js';
+import { Utils } from '../core/Utils.js';
+export class Index {
+  static storage = Storage.getInstance();
+  type;
+  path;
+  index;
+  constructor(type, path) {
+    this.type = type;
+    this.path = path;
+    Index.storage.ensurePath(this.path);
+    this.index = this.loadIndex();
+  }
+  // --- load & save index ---
+  loadIndex() {
+    const raw = Index.storage.readJSON(this.path) ?? {};
+    log.debug(`Index [${this.type}] loaded: ${Object.keys(raw).length} items`);
+    return new Map(Object.entries(raw));
+  }
+  saveIndex() {
+    const content = Object.fromEntries(this.index);
+    Index.storage.writeJSON(this.path, content);
+  }
+  // --- basic operations ---
+  get size() {
+    return this.index.size;
+  }
+  get values() {
+    return this.index.values();
+  }
+  get keys() {
+    return this.index.keys();
+  }
+  getIndex() {
+    return this.index;
+  }
+  has(uriLike) {
+    return this.index.has(Utils.sanitize(uriLike));
+  }
+  get(uriLike) {
+    return this.index.get(Utils.sanitize(uriLike));
+  }
+  // --- manipulate index (add, update, remove items) ---
+  update(uriLike, data, allowUpdate = true, save = true) {
+    return (
+      log.catch(() => {
+        const uri = Utils.sanitize(uriLike);
+        if (!allowUpdate && this.index.has(uri)) return false;
+        log.debug(`Updating index [${this.type}] item: ${uri}`, data);
+        const item = Utils.merge('unique' /* ArrayMode.Unique */, this.index.get(uri) ?? {}, data);
+        this.index.set(uri, item);
+        if (save) this.saveIndex();
+        return item;
+      }, `Failed to update index [${this.type}] item: ${uriLike}`) ?? false
+    );
+  }
+  delta(items, allowUpdate = true) {
+    const updated = items.reduce(
+      (count, { uriLike, data }) => count + (this.update(uriLike, data, allowUpdate, false) ? 1 : 0),
+      0
+    );
+    this.saveIndex();
+    log.debug(`Index [${this.type}] delta applied: ${updated} items updated`);
+    return updated;
+  }
+  add(uriLike, data) {
+    log.debug(`Adding index [${this.type}] item: ${uriLike}`);
+    return this.update(uriLike, data, false);
+  }
+  delete(uriLike) {
+    log.debug(`Deleting index [${this.type}] item: ${uriLike}`);
+    this.index.delete(Utils.sanitize(uriLike));
+    this.saveIndex();
+  }
+  // --- search index items ---
+  search(query, looseMatch = false) {
+    const tokens = Utils.buildSearchText(query).split(' ').filter(Boolean);
+    return new Map([...this.index].filter(([_, { text }]) => Utils.tokenSearch(text, tokens, looseMatch)));
+  }
+}

package/dist/abstract/Job.d.ts

@@ -0,0 +1,19 @@
+import type { IJob } from '../interfaces/job';
+import type { TLoggingLevel } from '../types/config';
+import type { TJobClsOptions } from '../types/job';
+export declare abstract class Job<T extends TJobClsOptions = TJobClsOptions> implements IJob<T> {
+  protected static readonly config: import('../interfaces/config').IConfig;
+  protected readonly options: T;
+  protected readonly job: string;
+  protected readonly silent: boolean;
+  protected readonly safeMode: boolean;
+  constructor(options: T, job: string);
+  protected log(msg: string, meta?: any, as?: TLoggingLevel): void;
+  protected err(err: unknown, msg?: string): void;
+  protected protect<F extends (...args: any[]) => any, R = Awaited<ReturnType<F>>>(fn: F): Promise<R | undefined>;
+  getJobName(): string;
+  getOptions(): T;
+  isSilent(): boolean;
+  isSafeMode(): boolean;
+  abstract run(): void | Promise<void>;
+}

package/dist/abstract/Job.js

@@ -0,0 +1,49 @@
+import { Config } from '../core/Config.js';
+import { log } from '../core/Logger.js';
+import { Utils } from '../core/Utils.js';
+import { Parser } from '../parser/Parser.js';
+export class Job {
+  static config = Config.getInstance();
+  options;
+  job;
+  silent;
+  safeMode;
+  constructor(options, job) {
+    this.options = options;
+    this.job = job;
+    const { silent, safeMode } = Job.config.job;
+    this.silent = this.options.silent !== undefined ? Parser.boolean(this.options.silent) : silent;
+    this.safeMode = this.options.safeMode !== undefined ? Parser.boolean(this.options.safeMode) : safeMode;
+    this.log('Run job', this.options);
+  }
+  // --- helper ---
+  log(msg, meta, as = 'info') {
+    if (!this.silent) log[as](`[${this.job.toUpperCase()}] ${msg}`, meta);
+  }
+  err(err, msg) {
+    if (!this.silent) log.errMsg(err, msg ? `[${this.job.toUpperCase()}] ${msg}` : undefined);
+  }
+  async protect(fn) {
+    try {
+      const res = await Utils.measure(fn);
+      this.log(`Finished in ${(res.ms / 1000).toFixed(3)} sec`);
+      return res.result;
+    } catch (err) {
+      this.err(err);
+      if (!this.safeMode) throw err;
+    }
+  }
+  // --- getter ---
+  getJobName() {
+    return this.job;
+  }
+  getOptions() {
+    return this.options;
+  }
+  isSilent() {
+    return this.silent;
+  }
+  isSafeMode() {
+    return this.safeMode;
+  }
+}

package/dist/abstract/Snapshot.d.ts

@@ -0,0 +1,22 @@
+import type { TSnapshot } from '@rtbnext/schema/src/base/generic';
+import type { ISnapshot } from '../interfaces/snapshot';
+export declare abstract class Snapshot<T extends TSnapshot> implements ISnapshot<T> {
+  protected static readonly storage: import('../interfaces/storage').IStorage;
+  protected readonly path: string;
+  protected dates: string[];
+  protected constructor(path: string);
+  protected scanDates(): string[];
+  protected datedPath(date: string): string;
+  getDates(): string[];
+  hasDate(dateLike: string): boolean;
+  firstDate(): string | undefined;
+  latestDate(): string | undefined;
+  nearestDate(dateLike: string): string | undefined;
+  datesInRange(from: string, to: string): string[];
+  datesInYear(year: string | number): string[];
+  firstInYear(year: string | number): string | undefined;
+  latestInYear(year: string | number): string | undefined;
+  getSnapshot(dateLike: string, exactMatch?: boolean): T | false;
+  getLatest(): T | false;
+  saveSnapshot(snapshot: T, force?: boolean): boolean;
+}

package/dist/abstract/Snapshot.js

@@ -0,0 +1,78 @@
+import { join } from 'node:path';
+import { log } from '../core/Logger.js';
+import { Storage } from '../core/Storage.js';
+import { Utils } from '../core/Utils.js';
+import { Parser } from '../parser/Parser.js';
+export class Snapshot {
+  static storage = Storage.getInstance();
+  path;
+  dates;
+  constructor(path) {
+    this.path = path;
+    Snapshot.storage.ensurePath(this.path, true);
+    this.dates = this.scanDates();
+  }
+  // --- helper ---
+  scanDates() {
+    return Utils.sort(Snapshot.storage.scanDir(this.path));
+  }
+  datedPath(date) {
+    return join(this.path, `${date}`);
+  }
+  // --- basic getter ---
+  getDates() {
+    return this.dates;
+  }
+  hasDate(dateLike) {
+    return this.dates.includes(Parser.date(dateLike, 'ymd'));
+  }
+  firstDate() {
+    return this.dates[0];
+  }
+  latestDate() {
+    return this.dates.at(-1);
+  }
+  // --- special getter ---
+  nearestDate(dateLike) {
+    const target = Parser.date(dateLike);
+    return this.dates.slice().reduce((nearest, date) => (date > target ? nearest : date));
+  }
+  datesInRange(from, to) {
+    const fromDate = Parser.date(from),
+      toDate = Parser.date(to);
+    return this.dates.filter(date => date >= fromDate && date <= toDate);
+  }
+  datesInYear(year) {
+    return this.datesInRange(`${year}-01-01`, `${year}-12-31`);
+  }
+  firstInYear(year) {
+    const target = Parser.string(year);
+    return this.dates.find(date => date.substring(0, 4) === target);
+  }
+  latestInYear(year) {
+    const target = Parser.string(year);
+    return this.dates.filter(date => date.substring(0, 4) === target).at(-1);
+  }
+  // --- get snapshot data ---
+  getSnapshot(dateLike, exactMatch = true) {
+    const target = Parser.date(dateLike);
+    const date = this.hasDate(target) ? target : exactMatch ? undefined : this.nearestDate(target);
+    return date ? Snapshot.storage.readJSON(this.datedPath(date)) : false;
+  }
+  getLatest() {
+    return this.dates.length ? this.getSnapshot(this.latestDate()) : false;
+  }
+  // --- save snapshot ---
+  saveSnapshot(snapshot, force = false) {
+    log.debug(`Saving snapshot for date ${snapshot.date}`);
+    return (
+      log.catch(() => {
+        if (!force && this.hasDate(snapshot.date)) throw new Error(`Snapshot for date ${snapshot.date} already exists`);
+        const path = this.datedPath(snapshot.date);
+        if (!Snapshot.storage.writeJSON(path, snapshot)) throw new Error(`Failed to write snapshot to ${path}`);
+        this.dates = this.scanDates();
+        return true;
+      }, `Failed to save snapshot for date ${snapshot.date}`) ?? false
+    );
+  }
+}

package/dist/bin/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin/cli.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { JOBS } from '../job/index.js';
+const addGlobalOptions = command =>
+  void command
+    .optionsGroup('Global Options:')
+    .option('--silent', 'Disable logging')
+    .option('--safe-mode', 'Enable safe mode');
+const program = new Command();
+program
+  .name('ntbnext-cli')
+  .description('CLI to maintain the @rtbnext billionaires database')
+  .version('2.0.0', '-v, --version');
+for (const JobClass of JOBS) {
+  const { id, desc, options } = JobClass.command;
+  const command = program.command(id).description(desc);
+  for (const { name, desc, parser, required } of options ?? []) {
+    const fn = required ? command.requiredOption.bind(command) : command.option.bind(command);
+    if (parser) fn(name, desc, parser);
+    else fn(name, desc);
+  }
+  addGlobalOptions(command);
+  command.action(async options => await new JobClass(options).run());
+}
+await program.parseAsync();

package/dist/bin/cron.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin/cron.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { Cron } from '../core/Cron.js';
+const cron = Cron.getInstance();
+await cron.run();

package/dist/core/Config.d.ts

@@ -0,0 +1,30 @@
+import type { IConfig } from '../interfaces/config';
+import type {
+  TConfigObject,
+  TCronConfig,
+  TFetchConfig,
+  TJobConfig,
+  TLoggingConfig,
+  TQueueConfig,
+  TStorageConfig
+} from '../types/config';
+export declare class Config implements IConfig {
+  private static instance;
+  private readonly cwd;
+  private readonly path;
+  private readonly env;
+  private readonly cfg;
+  private constructor();
+  private loadConfigFile;
+  private loadConfig;
+  get root(): string;
+  get environment(): string;
+  get config(): TConfigObject;
+  get logging(): TLoggingConfig;
+  get job(): TJobConfig;
+  get cron(): TCronConfig;
+  get storage(): TStorageConfig;
+  get fetch(): TFetchConfig;
+  get queue(): TQueueConfig;
+  static getInstance(): IConfig;
+}

package/dist/core/Config.js

@@ -0,0 +1,66 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import process, { cwd } from 'node:process';
+import { parse } from 'yaml';
+import { Utils } from './Utils.js';
+export class Config {
+  static instance;
+  cwd;
+  path;
+  env;
+  cfg;
+  constructor() {
+    this.cwd = cwd();
+    this.path = join(this.cwd, 'config');
+    this.env = process.env.NODE_ENV ?? 'production';
+    this.cfg = this.loadConfig();
+  }
+  // --- config loader ---
+  loadConfigFile(path) {
+    if (!existsSync((path = join(this.path, path)))) return {};
+    try {
+      return parse(readFileSync(path, 'utf8'));
+    } catch {
+      return {};
+    }
+  }
+  loadConfig() {
+    return Utils.merge(
+      'replace' /* ArrayMode.Replace */,
+      this.loadConfigFile('default.yml'),
+      this.loadConfigFile(`${this.env}.yml`)
+    );
+  }
+  // --- public getter ---
+  get root() {
+    return this.cwd;
+  }
+  get environment() {
+    return this.env;
+  }
+  get config() {
+    return this.cfg;
+  }
+  get logging() {
+    return this.cfg.logging;
+  }
+  get job() {
+    return this.cfg.job;
+  }
+  get cron() {
+    return this.cfg.cron;
+  }
+  get storage() {
+    return this.cfg.storage;
+  }
+  get fetch() {
+    return this.cfg.fetch;
+  }
+  get queue() {
+    return this.cfg.queue;
+  }
+  // --- instantiate ---
+  static getInstance() {
+    return (Config.instance ??= new Config());
+  }
+}

package/dist/core/Cron.d.ts

@@ -0,0 +1,12 @@
+import type { ICron } from '../interfaces/cron';
+export declare class Cron implements ICron {
+  private static readonly storage;
+  private static instance;
+  private readonly config;
+  private constructor();
+  private getLastRun;
+  private setLastRun;
+  private ensureLastRun;
+  run(): Promise<void>;
+  static getInstance(): ICron;
+}

package/dist/core/Cron.js

@@ -0,0 +1,52 @@
+import { prev } from 'nxtcron';
+import { Config } from './Config.js';
+import { log } from './Logger.js';
+import { Storage } from './Storage.js';
+import { JOBS } from '../job/index.js';
+export class Cron {
+  static storage = Storage.getInstance();
+  static instance;
+  config;
+  constructor() {
+    this.config = Config.getInstance().cron;
+  }
+  // --- helper ---
+  getLastRun() {
+    const lastRun = Cron.storage.readJSON('cron.json');
+    return lastRun && lastRun.lastRun ? new Date(lastRun.lastRun) : undefined;
+  }
+  setLastRun(date) {
+    return Cron.storage.writeJSON('cron.json', { lastRun: date.toISOString() });
+  }
+  ensureLastRun() {
+    const lastRun = this.getLastRun();
+    if (lastRun instanceof Date && !Number.isNaN(lastRun.getTime())) return lastRun;
+    this.setLastRun(new Date());
+    throw new Error('Initial Cron job run - will not execute any scheduled events');
+  }
+  // --- cron runner ---
+  async run() {
+    await log.catchAsync(async () => {
+      log.debug('Run scheduled Cron jobs ...');
+      const after = this.ensureLastRun(),
+        before = new Date();
+      const cronOptions = { timezone: this.config.timezone, count: 1, after, before };
+      for (const JobClass of JOBS) {
+        if (!('cron' in JobClass)) continue;
+        for (const { cronexpr, options } of JobClass.cron) {
+          const [date] = prev(cronexpr, cronOptions);
+          if (date === undefined || !(date instanceof Date)) continue;
+          log.info(`[CRON] Run Cron job ${JobClass.command.id} scheduled @ ${date.toISOString()}`);
+          await new JobClass(options?.(date) ?? {}).run();
+          break;
+        }
+      }
+      log.debug(`Shut down Cron job runner, set last run time to ${before.toISOString()}`);
+      this.setLastRun(before);
+    }, 'Failed to run cron jobs');
+  }
+  // --- instantiate ---
+  static getInstance() {
+    return (Cron.instance ??= new Cron());
+  }
+}

package/dist/core/Fetch.d.ts

@@ -0,0 +1,28 @@
+import type { IFetch } from '../interfaces/fetch';
+import type { TFetchMethod, THeader } from '../types/fetch';
+import type { TListResponse, TProfileResponse, TResponse } from '../types/response';
+export declare class Fetch implements IFetch {
+  private static instance;
+  private readonly config;
+  private readonly wikiQuery;
+  private lastRequest;
+  private httpClient;
+  private constructor();
+  private setupHttpClient;
+  private getRandomDelay;
+  private applyRateLimit;
+  private useApiUserAgent;
+  private useRandomUserAgent;
+  private prepQuery;
+  private retErr;
+  private fetch;
+  single<T>(url: string, method?: TFetchMethod, header?: THeader): Promise<TResponse<T>>;
+  batch<T>(urls: string[], method?: TFetchMethod, header?: THeader): Promise<TResponse<T>[]>;
+  wayback<T>(url: string, ts: unknown): Promise<TResponse<T>>;
+  list<T extends object>(uriLike: string, year: string): Promise<TResponse<TListResponse<T>>>;
+  profile(...uriLike: string[]): Promise<TResponse<TProfileResponse>[]>;
+  wikidata<T>(sparql: string): Promise<TResponse<T>>;
+  wikipedia<T>(query: Record<string, unknown>, lang?: string): Promise<TResponse<T>>;
+  commons<T>(query: Record<string, unknown>): Promise<TResponse<T>>;
+  static getInstance(): IFetch;
+}