@rtbnext/core 2.0.0-alpha.1

package/dist/utils/Wiki.js

@@ -0,0 +1,168 @@
+import { Fetch } from '../core/Fetch.js';
+import { log } from '../core/Logger.js';
+import { Parser } from '../parser/Parser.js';
+export class Wiki {
+  static fetch = Fetch.getInstance();
+  static scoreWDItem(item, data) {
+    const { name: { shortName } = {}, gender, birthDate, citizenship } = data.info ?? {};
+    let score = 0;
+    // --- name matching ---
+    if (item.itemLabel.value.trim() === shortName) score += 0.2;
+    else if (item.itemLabel.xmlLang === 'en') score += 0.1;
+    else score += 0.1;
+    // --- birthdate matching ---
+    if (birthDate && item.birthdate?.value.startsWith(birthDate)) score += 0.2;
+    else if (birthDate && item.birthdate?.value.startsWith(birthDate.substring(0, 4))) score += 0.1;
+    else if (birthDate && item.birthdate?.value) score -= 0.1;
+    // --- gender matching ---
+    if (gender && item.gender?.value.endsWith(gender === 'm' ? 'Q6581097' : gender === 'f' ? 'Q6581072' : '-'))
+      score += 0.1;
+    else if (score && item.gender?.value) score -= 0.2;
+    // --- citizenship matching ---
+    if (citizenship && item.iso2?.value === citizenship.toUpperCase()) score += 0.2;
+    // --- media matching ---
+    if (item.article) score += 0.1;
+    if (item.image) score += 0.05;
+    // --- occupation matching ---
+    if (['Q131524', 'Q557880', 'Q911554', 'Q2462658'].some(e => item.occupation?.value.endsWith(e))) score += 0.2;
+    else if (item.occupation) score += 0.05;
+    // --- economic matching ---
+    if (item.employer ?? item.ownerOf) score += 0.1;
+    if (item.netWorth) score += 0.2;
+    return Math.min(1, Math.max(0, score));
+  }
+  static async queryWikidata(data) {
+    log.debug(`Querying Wikidata for: ${data.info?.name?.shortName}`);
+    return await log.catchAsync(
+      async () => {
+        const shortName = data.info?.name?.shortName;
+        if (!shortName) throw new Error('No short name provided');
+        const [first, ...rest] = shortName.split(' '),
+          last = rest.pop();
+        const nameVariants = [shortName, `${first[0]}. ${last}`, `${first} ${last}`]
+          .filter(Boolean)
+          .map(n => `"${n}"@en "${n}"@de`)
+          .join(' ');
+        const sparql = `
+        SELECT DISTINCT
+          ?item ?itemLabel ?gender ?birthdate ?article ?image ?iso2 ?occupation ?employer ?ownerOf ?netWorth
+        WHERE {
+          VALUES ?name { ${nameVariants} }
+          ?item wdt:P31 wd:Q5 .
+          { { ?item rdfs:label ?name . } UNION { ?item skos:altLabel ?name . } }
+          OPTIONAL { ?item wdt:P21 ?gender . }
+          OPTIONAL { ?item wdt:P569 ?birthdate . }
+          OPTIONAL { ?article schema:about ?item ; schema:isPartOf <https://en.wikipedia.org/> . }
+          OPTIONAL { ?item wdt:P18 ?image . }
+          OPTIONAL { ?item wdt:P27 ?country . ?country wdt:P297 ?iso2 . }
+          OPTIONAL { ?item wdt:P106 ?occupation . }
+          OPTIONAL { ?item wdt:P108 ?employer . }
+          OPTIONAL { ?item wdt:P169 ?employer . }
+          OPTIONAL { ?item wdt:P127 ?ownerOf . }
+          OPTIONAL { ?item wdt:P1830 ?ownerOf . }
+          OPTIONAL { ?item wdt:P2218 ?netWorth . }
+          SERVICE wikibase:label { bd:serviceParam wikibase:language "en,de" . }
+        }
+        LIMIT 20
+      `;
+        const res = await Wiki.fetch.wikidata(sparql);
+        let best;
+        for (const item of res.data?.results.bindings ?? []) {
+          const score = Wiki.scoreWDItem(item, data);
+          if (!best || score > best.score) best = { score, item };
+          if (best.score === 1) break;
+        }
+        if (!best || best.score < 0.65) throw new Error('No suitable Wikidata item found');
+        log.debug(`Best Wikidata item for ${shortName} has score: ${best.score}`);
+        return Parser.container({
+          qid: { value: best.item.item.value.split('/').pop(), type: 'string' },
+          confidence: { value: best.score, type: 'number', args: [3] },
+          article: { value: best.item.article?.value.split('/').pop(), type: 'decodeURI' },
+          image: { value: best.item.image?.value.split('/').pop(), type: 'decodeURI' }
+        });
+      },
+      `Failed to query Wikidata for: ${data.info?.name?.shortName ?? 'unknown'}`
+    );
+  }
+  static async queryCommonsImage(title) {
+    log.debug(`Querying Wikimedia Commons image: ${title}`);
+    return await log.catchAsync(async () => {
+      const res = await Wiki.fetch.commons({
+        action: 'query',
+        titles: `File:${title}`,
+        prop: 'imageinfo',
+        redirects: 1,
+        iiprop: 'url|extmetadata',
+        iiurlwidth: 400
+      });
+      const info = res.data?.query.pages?.[0]?.imageinfo?.[0];
+      if (!info) throw new Error(`No image info found for: ${title}`);
+      log.debug(`Wikimedia Commons image info received for: ${title}`);
+      const meta = info.extmetadata ?? {};
+      const thumbUrl = info.thumburl ?? Object.values(info.responsiveUrls ?? {}).at(0);
+      const dateTime = meta.DateTimeOriginal?.value ?? meta.DateTime?.value;
+      const credits = Parser.list([
+        meta.Attribution?.value ?? meta.Artist?.value ?? meta.Credit?.value,
+        meta.LicenseShortName?.value ?? meta.UsageTerms?.value,
+        'via Wikimedia Commons'
+      ]).join(', ');
+      return Parser.container({
+        url: { value: info.descriptionurl, type: 'string' },
+        file: { value: info.url, type: 'string' },
+        thumb: { value: thumbUrl, type: 'string' },
+        caption: { value: meta.ImageDescription?.value, type: 'safeStr' },
+        date: { value: dateTime, type: 'date', args: ['iso'] },
+        credits: { value: credits, type: 'safeStr' }
+      });
+    }, `Failed to query Wikimedia Commons image: ${title}`);
+  }
+  static async queryWikiPage(title, qid, image, confidence = 1) {
+    log.debug(`Querying Wikipedia page: ${title}`);
+    return await log.catchAsync(async () => {
+      const res = await Wiki.fetch.wikipedia({
+        action: 'query',
+        prop: 'extracts|info|pageprops|pageimages',
+        titles: title,
+        redirects: 1,
+        exintro: 1,
+        explaintext: 1,
+        exsectionformat: 'plain',
+        piprop: 'name',
+        pilimit: 1
+      });
+      if (!res?.success || !res.data || !res.data.query.pages.length)
+        throw new Error(`No Wikipedia page found for: ${title}`);
+      log.debug(`Wikipedia page info received for: ${title}`);
+      const raw = res.data.query.pages[0];
+      if (!image && raw.pageimage) {
+        log.debug(`Querying page image from Wikimedia Commons: ${raw.pageimage}`);
+        image = await this.queryCommonsImage(raw.pageimage);
+      }
+      return {
+        image,
+        ...Parser.container({
+          uri: { value: title, type: 'string' },
+          pageId: { value: raw.pageid, type: 'number' },
+          refId: { value: raw.lastrevid, type: 'number' },
+          confidence: { value: confidence, type: 'number', args: [3] },
+          name: { value: raw.title, type: 'string' },
+          lastModified: { value: raw.touched, type: 'date', args: ['iso'] },
+          summary: { value: raw.extract ?? '', type: 'list', args: ['safeStr', '\n'], strict: false },
+          sortKey: { value: raw.pageprops?.['defaultsort'], type: 'string' },
+          wikidata: { value: qid ?? raw.pageprops?.['wikibase_item'], type: 'string' },
+          desc: { value: raw.pageprops?.['wikibase-shortdesc'], type: 'safeStr' }
+        })
+      };
+    }, `Failed to query Wikipedia page: ${title}`);
+  }
+  static async fromProfileData(data) {
+    const { qid, confidence, article, image } = (await Wiki.queryWikidata(data)) ?? {};
+    log.debug(
+      `Query Wikidata for ${data.info?.name?.shortName ?? 'unknown'}: ${qid || 'no match'} ` +
+        `(score: ${confidence || 0})`
+    );
+    return article
+      ? await Wiki.queryWikiPage(article, qid, image ? await this.queryCommonsImage(image) : undefined)
+      : undefined;
+  }
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@rtbnext/core",
+  "description": "Internal core package for the rtbnext project",
+  "license": "MIT",
+  "version": "2.0.0-alpha.1",
+  "author": {
+    "name": "komed3 (Paul Köhler)",
+    "email": "webmaster@komed3.de",
+    "url": "https://komed3.de"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "bin": {
+    "rtbnext-cli": "./dist/bin/cli.js",
+    "rtbnext-cron": "./dist/bin/cron.js"
+  },
+  "type": "module",
+  "scripts": {
+    "format": "prettier --ignore-path \"\" --write \"./dist/**/*.@(js|d.ts)\" --log-level warn",
+    "build": "tsc && node scripts/copy-dts.mjs && tsc-alias && npm run format"
+  },
+  "dependencies": {
+    "@isodb/us-states": "^0.1.0",
+    "@komed3/deepmerge": "^1.0.0",
+    "@rtbnext/schema": "^2.0.0-alpha.20",
+    "axios": "^1.18.1",
+    "cmpstr": "^3.3.0",
+    "commander": "^15.0.0",
+    "csv-string": "^4.1.1",
+    "devtypes": "^2.2.0",
+    "i18n-iso-countries": "^7.14.0",
+    "js-sha256": "^0.11.1",
+    "nxtcron": "^1.0.1",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.1.0",
+    "prettier": "^3.9.4",
+    "tsc-alias": "^1.8.17",
+    "typescript": "^6.0.3"
+  }
+}