euparliamentmonitor 0.9.21 → 0.9.23

package/scripts/aggregator/metadata/resolve-helpers.d.ts

@@ -49,6 +49,39 @@ export declare function composeContextualDescription(lang: LanguageCode, baseDes
     readonly headline: string;
     readonly summary: string;
 }, date: string, _runId: string): string;
+/**
+ * Build a per-article `extendedDescription` (used for
+ * `og:description`, Twitter cards, and AI-overview surfaces) that is
+ * always ≥ {@link DESCRIPTION_MAX_LENGTH} characters whenever the
+ * editorial source paragraph is too short to satisfy
+ * {@link truncateExtendedDescription} on its own.
+ *
+ * This is the *only* code path that surfaces the localized
+ * `labels.reader` framing — the short `<meta description>` no longer
+ * carries it (see comment in {@link composeContextualDescription}).
+ * The structure is: `<base> <Date: YYYY-MM-DD.> <Context: …> <reader>`,
+ * passed through {@link truncateExtendedDescription} (300-char max with
+ * a 200-char min) so it occupies the Open Graph / Discover budget
+ * without exceeding it.
+ *
+ * @param lang - Target language code
+ * @param baseDescription - Best description from manifest/editorial/template
+ * @param editorial - Artifact-derived headline and summary
+ * @param editorial.headline - Artifact-derived headline
+ * @param editorial.summary - Artifact-derived summary
+ * @param date - ISO article date
+ * @returns Extended description ≥180 chars when feasible, otherwise `''`
+ */
+export declare function composeContextualExtendedDescription(lang: LanguageCode, baseDescription: string, editorial: {
+    readonly headline: string;
+    readonly summary: string;
+}, date: string): string;
+export declare function hasLeakySeoToken(value: string): boolean;
+declare function sanitizeDescriptionCandidate(value: string): string;
+declare function isUsableResolvedTitle(value: string, options?: {
+    readonly allowFullSentence?: boolean;
+}): boolean;
+declare function deriveHeadlineFromSummary(summary: string): string;
 /**
  * Append a short run qualifier to otherwise duplicate-prone fallback
  * titles. Sanitizes the raw `runId` so user-facing `<title>` strings
@@ -88,4 +121,5 @@ export declare function buildSeoKeywords(lang: LanguageCode, articleType: string
  * @returns First non-empty entry
  */
 export declare function pickFirstNonEmpty(candidates: readonly string[]): string;
+export { deriveHeadlineFromSummary, isUsableResolvedTitle, sanitizeDescriptionCandidate };
 //# sourceMappingURL=resolve-helpers.d.ts.map

package/scripts/aggregator/metadata/resolve-helpers.js

@@ -20,7 +20,14 @@ import { extractExtendedLedeAfterHeading, extractStrongProseLine } from './lede-
 import { isGenericHeading } from './heading-rules.js';
 import { humanizeSlug } from './slug.js';
 import { SEO_CONTEXT_LABELS } from './template-fallback.js';
-import { extractFirstSentence, truncateDescription, truncateTitle } from './text-utils.js';
+import { EXTENDED_DESCRIPTION_MAX_LENGTH } from './text-utils-constants.js';
+import { extractFirstSentence, shouldSkipDescriptionLine, truncateDescription, truncateExtendedDescription, truncateTitle, } from './text-utils.js';
+import { readEnglishBriefBody } from './brief-body.js';
+import { extractBriefingHighlight } from './briefing-highlight.js';
+import { CROSS_SITE_KEYWORDS, isNoiseKeywordToken } from './keyword-filters.js';
+import { findTitleRejectionReason } from './title-rejection.js';
+const LEAKY_RUNID_RE = /\b[a-z][a-z-]*-run-?\d+-\d{8,}\b/iu;
+const SEO_TITLE_FLOOR = 20;
 /**
  * Extract a manifest override value for a single language. Accepts either
  * a plain string (applied to every language) or a `LanguageMap` object.
@@ -53,31 +60,80 @@ export function manifestOverrideFor(value, lang) {
  */
 export function resolveEditorialContent(opts) {
     const { articleType, date, markdown, runDir } = opts;
+    // Tier 1 (NEW, May-2026): structural extraction of `## Strategic
+    // Intelligence Summary` and `## Reader Briefing` from the English
+    // brief. These two sections are the editorial heart of every
+    // current-style executive brief — they are journalistically richer
+    // than the first non-generic H1 the legacy walker picks up, so we
+    // try them first. Returns `null` for the ~200 historical briefs
+    // that pre-date the style guide, in which case we fall through.
+    const briefBody = readEnglishBriefBody(runDir ?? '');
+    const briefing = briefBody ? extractBriefingHighlight(briefBody) : null;
+    // Bridge the briefing's `string | undefined` fields into plain
+    // strings so the downstream `||` fallback chains satisfy the
+    // `prefer-nullish-coalescing` lint rule (no nullable LHS).
+    const briefingHeadline = briefing?.headline ?? '';
+    const briefingSummary = briefing?.summary ?? '';
+    const briefingExtended = briefing?.extendedSummary ?? '';
+    if (briefingHeadline) {
+        return {
+            headline: briefingHeadline,
+            summary: briefingSummary,
+            extendedSummary: briefingExtended || extractExtendedLedeAfterHeading(markdown),
+        };
+    }
     let artefactSummary = '';
     if (runDir) {
         const highlight = extractArtifactHighlight(runDir, articleType, date);
-        if (highlight?.headline) {
+        const highlightHeadline = highlight?.headline ?? '';
+        const highlightSummary = highlight?.summary ?? '';
+        if (highlightHeadline) {
             return {
-                headline: highlight.headline,
-                summary: highlight.summary,
-                extendedSummary: extractExtendedLedeAfterHeading(markdown),
+                headline: highlightHeadline,
+                summary: briefingSummary || highlightSummary,
+                extendedSummary: briefingExtended || extractExtendedLedeAfterHeading(markdown),
             };
         }
-        if (highlight?.summary) {
-            artefactSummary = highlight.summary;
+        if (highlightSummary) {
+            artefactSummary = highlightSummary;
         }
     }
+    // Per the brief-only SEO contract (2026-05-24): when an executive
+    // brief is present, we **never** fall through to the aggregated
+    // `markdown` content (which is the assembled `article.md` body
+    // including all artefact prose). The brief is the only sanctioned
+    // source for `<title>` / `<meta description>` / keywords; if it
+    // failed to yield a usable headline above, the resolver returns
+    // empty so the localized template fallback (Breaking | YYYY-MM-DD,
+    // etc.) wins. Only legacy runs that ship without a brief at all are
+    // allowed to reach the aggregated-markdown fallback.
+    const briefPresent = briefBody.trim().length > 0;
+    if (briefPresent) {
+        if (artefactSummary) {
+            const firstSentence = extractFirstSentence(artefactSummary);
+            return {
+                headline: truncateTitle(firstSentence || artefactSummary),
+                summary: briefingSummary || artefactSummary,
+                extendedSummary: briefingExtended || extractExtendedLedeAfterHeading(markdown),
+            };
+        }
+        return {
+            headline: '',
+            summary: briefingSummary,
+            extendedSummary: briefingExtended,
+        };
+    }
     const aggregatedH1 = extractFirstH1(markdown);
     const aggregatedSummary = extractStrongProseLine(markdown);
     const aggregatedExtended = extractExtendedLedeAfterHeading(markdown);
     if (aggregatedH1 && !isGenericHeading(aggregatedH1, articleType, date)) {
         return {
             headline: truncateTitle(aggregatedH1),
-            summary: artefactSummary || aggregatedSummary,
-            extendedSummary: aggregatedExtended,
+            summary: briefingSummary || artefactSummary || aggregatedSummary,
+            extendedSummary: briefingExtended || aggregatedExtended,
         };
     }
-    const summary = artefactSummary || aggregatedSummary;
+    const summary = briefingSummary || artefactSummary || aggregatedSummary;
     if (summary) {
         // The H1 is generic (category-noun, bare-institutional, or
         // template-style) so we have to derive `<title>` from the BLUF/
@@ -85,11 +141,15 @@ export function resolveEditorialContent(opts) {
         // resulting title is grammatically self-contained — falling back
         // to clause-boundary truncation downstream when the sentence
         // itself overruns TITLE_MAX_LENGTH.
+        // Fall back to the raw summary when the first-sentence extractor
+        // returns '' — happens when the source is a single sentence with no
+        // `. ` terminator inside the soft-min window. `truncateTitle` will
+        // still apply clause-boundary truncation downstream.
         const firstSentence = extractFirstSentence(summary);
         return {
-            headline: truncateTitle(firstSentence),
+            headline: truncateTitle(firstSentence || summary),
             summary,
-            extendedSummary: aggregatedExtended,
+            extendedSummary: briefingExtended || aggregatedExtended,
         };
     }
     return { headline: '', summary: '', extendedSummary: '' };
@@ -133,10 +193,123 @@ export function composeContextualDescription(lang, baseDescription, editorial, d
     if (context && !containsNormalized(parts[0] ?? '', context)) {
         parts.push(`${labels.context}: ${context}`);
     }
+    // NOTE: the localized `labels.reader` "for democratic-accountability
+    // readers …" hint is intentionally **not** appended here. That
+    // boilerplate inflates `<meta description>` past the 160-char SERP
+    // cutoff without surfacing any article-specific signal, so it is
+    // restricted to the longer {@link composeContextualExtendedDescription}
+    // path (used by `og:description` / AI-overview surfaces, which have
+    // a 250–300 char budget where the framing carries real value).
+    return truncateDescription(parts.join(' '));
+}
+/**
+ * Build a per-article `extendedDescription` (used for
+ * `og:description`, Twitter cards, and AI-overview surfaces) that is
+ * always ≥ {@link DESCRIPTION_MAX_LENGTH} characters whenever the
+ * editorial source paragraph is too short to satisfy
+ * {@link truncateExtendedDescription} on its own.
+ *
+ * This is the *only* code path that surfaces the localized
+ * `labels.reader` framing — the short `<meta description>` no longer
+ * carries it (see comment in {@link composeContextualDescription}).
+ * The structure is: `<base> <Date: YYYY-MM-DD.> <Context: …> <reader>`,
+ * passed through {@link truncateExtendedDescription} (300-char max with
+ * a 200-char min) so it occupies the Open Graph / Discover budget
+ * without exceeding it.
+ *
+ * @param lang - Target language code
+ * @param baseDescription - Best description from manifest/editorial/template
+ * @param editorial - Artifact-derived headline and summary
+ * @param editorial.headline - Artifact-derived headline
+ * @param editorial.summary - Artifact-derived summary
+ * @param date - ISO article date
+ * @returns Extended description ≥180 chars when feasible, otherwise `''`
+ */
+export function composeContextualExtendedDescription(lang, baseDescription, editorial, date) {
+    const labels = getLocalizedString(SEO_CONTEXT_LABELS, lang);
+    const base = baseDescription.trim();
+    const parts = base ? [base] : [];
+    const datePart = `${labels.date} ${date}.`;
+    if (!containsNormalized(base, `${labels.date} ${date}`)) {
+        parts.push(datePart);
+    }
+    const context = pickFirstNonEmpty([editorial.summary, editorial.headline]);
+    if (context && !containsNormalized(parts.join(' '), context)) {
+        parts.push(`${labels.context}: ${context}`);
+    }
     if (!containsNormalized(parts.join(' '), labels.reader)) {
         parts.push(labels.reader);
     }
-    return truncateDescription(parts.join(' '));
+    // Synthesizer path: clamp to the 300-char og:description budget
+    // *without* enforcing the 181-char sentence-boundary floor that
+    // {@link truncateExtendedDescription} applies. The whole point of
+    // this helper is to produce a non-empty extended description when
+    // the editorial source paragraph was too short — accepting a
+    // 130-char synthesized string is strictly better than the empty
+    // fallback that was previously emitted on 56 breaking briefs.
+    // We delegate the actual clamp to {@link truncateDescription} on
+    // the joined buffer first (which won't trip because the buffer is
+    // already under 300), then truncate again only if it overruns
+    // the larger 300-char budget.
+    const joined = parts.join(' ').trim();
+    if (!joined)
+        return '';
+    if (joined.length <= EXTENDED_DESCRIPTION_MAX_LENGTH)
+        return joined;
+    // Overran the 300-char budget — apply the same sentence-boundary
+    // preserving truncation as truncateExtendedDescription.
+    return truncateExtendedDescription(joined);
+}
+export function hasLeakySeoToken(value) {
+    if (!value)
+        return false;
+    return value.toLowerCase().includes('analysis run') || LEAKY_RUNID_RE.test(value);
+}
+function stripLeadingFragmentSeparator(value) {
+    return value.replace(/^[:;—–-]\s+/u, '').trim();
+}
+function stripLeakySentences(value) {
+    if (!value)
+        return '';
+    const parts = value
+        .split(/(?<=[.!?])\s+/u)
+        .map((part) => part.trim())
+        .filter(Boolean);
+    const clean = parts.filter((part) => !hasLeakySeoToken(part));
+    return (clean.length > 0 ? clean : parts).join(' ').trim();
+}
+function sanitizeDescriptionCandidate(value) {
+    const cleaned = stripLeadingFragmentSeparator(stripLeakySentences(value));
+    return cleaned && !shouldSkipDescriptionLine(cleaned) ? cleaned : '';
+}
+function isUsableResolvedTitle(value, options) {
+    const cleaned = stripLeadingFragmentSeparator(value);
+    if (cleaned.length < SEO_TITLE_FLOOR)
+        return false;
+    if (hasLeakySeoToken(cleaned))
+        return false;
+    // Reject section-header leaks, ellipsis-truncated strings, doc-IDs,
+    // and full-sentence fragments. See `title-rejection.ts` for the
+    // canonical denylist + structural rules. Without these guards, the
+    // 216-article audit (2026-05-24) showed `Strategic significance`,
+    // `Threat Level`, `Convergence themes`, `TA-10-2026-0160`, and
+    // ellipsis-cut paragraphs reaching the `<title>` surface.
+    //
+    // When `allowFullSentence` is true, the `sentence-fragment` reason is
+    // tolerated. This is used for summary-derived titles where the first
+    // sentence of the summary is the intended payload (e.g. recess days
+    // whose summary leads with `No new breaking developments on …`).
+    const reason = findTitleRejectionReason(cleaned);
+    if (reason && !(options?.allowFullSentence && reason === 'sentence-fragment')) {
+        return false;
+    }
+    return true;
+}
+function deriveHeadlineFromSummary(summary) {
+    const cleaned = sanitizeDescriptionCandidate(summary);
+    if (!cleaned)
+        return '';
+    return truncateTitle(extractFirstSentence(cleaned) || cleaned);
 }
 /**
  * Append a short run qualifier to otherwise duplicate-prone fallback
@@ -190,14 +363,22 @@ export function containsNormalized(haystack, needle) {
  * @returns De-duplicated keywords for `<meta name="keywords">`
  */
 export function buildSeoKeywords(lang, articleType, date, runId, title, description) {
+    // `runId` is intentionally unused: the previous implementation
+    // emitted `run <runId>` as a synthetic keyword, which surfaced
+    // opaque tokens like `run propositions-run261-1779431162` in
+    // `<meta name="keywords">`. The argument is preserved for callsite
+    // backward compatibility.
+    void runId;
     const localized = getLocalizedString(LOCALIZED_KEYWORDS, lang);
     const base = Object.getOwnPropertyDescriptor(localized, articleType)?.value;
     const fallback = ['EU Parliament', 'European Parliament', 'political intelligence'];
     const candidates = [
+        // Always-on cross-site portfolio keywords lead the list so they
+        // are guaranteed to survive the 16-entry budget cap.
+        ...CROSS_SITE_KEYWORDS,
         ...(base ?? fallback),
         humanizeSlug(articleType),
         date,
-        ...(runId ? [`run ${runId}`] : []),
         ...extractKeywordTerms(`${title} ${description}`),
     ];
     return dedupeKeywords(candidates).slice(0, 16);
@@ -205,6 +386,11 @@ export function buildSeoKeywords(lang, articleType, date, runId, title, descript
 /**
  * Extract short keyword terms from resolved SEO copy.
  *
+ * Filters out tokens that look like UUID hex fragments, run-id slugs,
+ * or digit-dominated noise (see {@link isNoiseKeywordToken}) so the
+ * keyword list never leaks internal aggregator identifiers into
+ * `<meta name="keywords">`.
+ *
  * @param text - Title and description text
  * @returns Candidate terms
  */
@@ -212,7 +398,7 @@ function extractKeywordTerms(text) {
     return text
         .split(/[^\p{L}\p{N}]+/u)
         .map((token) => token.trim())
-        .filter((token) => token.length >= 4 && !/^\d+$/.test(token))
+        .filter((token) => token.length >= 4 && !isNoiseKeywordToken(token))
         .slice(0, 18);
 }
 /**
@@ -250,4 +436,5 @@ export function pickFirstNonEmpty(candidates) {
     }
     return '';
 }
+export { deriveHeadlineFromSummary, isUsableResolvedTitle, sanitizeDescriptionCandidate };
 //# sourceMappingURL=resolve-helpers.js.map

package/scripts/aggregator/metadata/seo-budgets.d.ts

@@ -0,0 +1,140 @@
+/**
+ * @module Aggregator/Metadata/SeoBudgets
+ * @description Per-script SEO byte budgets and a script-aware clamp.
+ *
+ * Background. Google Search Central and Bing Webmaster Guidelines both
+ * document SERP snippet limits in **pixels**, not characters. Latin
+ * glyphs render at roughly half the pixel width of CJK glyphs, while
+ * Arabic/Hebrew letterforms sit between the two. A single `length`
+ * budget for `<title>` / `<meta description>` will always be wrong for
+ * at least one of the 14 publishing languages — typically over-truncating
+ * Latin copy and over-running CJK by a factor of two.
+ *
+ * This module provides:
+ *
+ *  - {@link classifyScript} — three-way `latin | cjk | rtl` family
+ *    classifier driven by the locale code (no glyph inspection — the
+ *    BCP-47 language tag is authoritative because every publishing
+ *    pipeline emits one full output per language).
+ *  - {@link SEO_BUDGETS} — per-surface × per-script byte caps derived
+ *    from the documented platform envelopes (Google ≤580 px title /
+ *    ≤155 char description; Bing slightly more generous; Facebook ≤95
+ *    chars on `og:title`; Twitter ≤70 / ≤200; LinkedIn shares OG).
+ *  - {@link budgetFor} — typed accessor returning the byte cap for a
+ *    `(lang, surface)` pair, with a uniform fallback to the strictest
+ *    Latin budget when the locale is unknown.
+ *  - {@link clampForBudget} — script-aware truncator that prefers
+ *    natural clause boundaries (CJK full-width punctuation, RTL
+ *    sentence punctuation, Latin clause separators) before falling
+ *    back to whitespace breaks. Returns the input verbatim when it
+ *    already fits.
+ *
+ * Pure, leaf module. No I/O, no dependencies on other aggregator
+ * modules beyond the existing `text-utils.ts` clause-boundary
+ * vocabulary.
+ */
+import type { LanguageCode } from '../../types/index.js';
+/**
+ * Three-way script family used as the column key in {@link SEO_BUDGETS}.
+ * `cjk` covers Chinese / Japanese / Korean (~2× Latin pixel width per
+ * glyph); `rtl` covers Arabic / Hebrew (bidi + ligature handling).
+ */
+export type ScriptFamily = 'latin' | 'cjk' | 'rtl';
+/**
+ * Iteration helper — all three script families in a deterministic
+ * order (latin → cjk → rtl). Exported so test matrices and downstream
+ * tooling can walk every column of {@link SEO_BUDGETS} without
+ * duplicating the literal list.
+ */
+export declare const ALL_SCRIPT_FAMILIES: readonly ScriptFamily[];
+/**
+ * Classify a locale code into a script family. Used to look up the
+ * correct byte cap in {@link SEO_BUDGETS}.
+ *
+ * @param lang - BCP-47 language tag (one of the 14 publishing locales)
+ * @returns Script family for SEO budget lookup
+ */
+export declare function classifyScript(lang: string): ScriptFamily;
+/**
+ * Public SEO surfaces this module budgets for. Each one has documented
+ * truncation behaviour by at least one major search engine or social
+ * platform.
+ *
+ * - `title` — HTML `<title>` (Google ≤580 px ≈ 60 Latin / 30 CJK / 55 RTL)
+ * - `metaDescription` — `<meta name="description">` (Google ≤~155 char)
+ * - `ogTitle` — Facebook / LinkedIn `og:title` (~95 Latin)
+ * - `ogDescription` — Facebook / LinkedIn `og:description` (~200 Latin)
+ * - `twitterTitle` — Twitter card title (≤70 Latin)
+ * - `twitterDescription` — Twitter card description (≤200 Latin)
+ * - `imageAlt` — `og:image:alt` / social card alt text (≤125 Latin)
+ * - `jsonLdHeadline` — Schema.org `NewsArticle.headline` (Google ≤110)
+ */
+export type SeoSurface = 'title' | 'metaDescription' | 'ogTitle' | 'ogDescription' | 'twitterTitle' | 'twitterDescription' | 'imageAlt' | 'jsonLdHeadline';
+/**
+ * Per-surface × per-script byte cap table. Numbers reflect the
+ * narrower of Google / Bing / Facebook / Twitter documented envelopes,
+ * with a ~5 % safety margin so a snippet on the edge of the budget
+ * isn't truncated mid-glyph by the rendering platform.
+ *
+ * For `jsonLdHeadline` the Schema.org `NewsArticle.headline` cap is
+ * script-independent (Google validates the literal character count at
+ * 110) — same value across the row.
+ */
+export declare const SEO_BUDGETS: Readonly<Record<SeoSurface, Readonly<Record<ScriptFamily, number>>>>;
+/**
+ * Resolve the byte cap for one `(lang, surface)` pair.
+ *
+ * @param lang - Publishing locale
+ * @param surface - SEO surface (see {@link SeoSurface})
+ * @returns Byte cap (positive integer)
+ */
+export declare function budgetFor(lang: LanguageCode | string, surface: SeoSurface): number;
+/**
+ * Truncate `text` to fit `(lang, surface)` SEO byte budget. Prefers a
+ * natural clause boundary inside the script's punctuation vocabulary
+ * (CJK / RTL / Latin) before falling back to a whitespace break.
+ *
+ * Always returns `text` verbatim when it already fits (no ellipsis
+ * appended). When truncation happens an ellipsis (`…`) is appended for
+ * Latin / RTL; for CJK the full-width ellipsis (`…`) reads as a
+ * partial-thought marker and is also appended — Schema.org and Google
+ * accept either glyph in `headline` / `description`.
+ *
+ * @param text - Source text (already plain-text — no Markdown / HTML)
+ * @param lang - Publishing locale
+ * @param surface - Target SEO surface
+ * @returns Clamped text ≤ `budgetFor(lang, surface)` characters
+ */
+export declare function clampForBudget(text: string, lang: LanguageCode | string, surface: SeoSurface): string;
+/**
+ * Optional inputs to {@link clampTitleForSurface}.
+ *
+ * `siteTitle` is the brand suffix (e.g. "EU Parliament Monitor") and
+ * `separator` is the localized glue (e.g. `" | "` / `" ・ "` / `" ׀ "`).
+ * When both are provided the function tries to keep the brand suffix
+ * inside the budget; when the article title alone already fills the
+ * budget the suffix is *dropped* (better SERP outcome than a truncated
+ * headline followed by a clipped brand).
+ *
+ * `shortSiteTitle` is the optional fallback used when the full brand
+ * suffix can't fit but a shorter variant would (e.g. `"EPM"` for CJK).
+ */
+export interface TitleSurfaceOptions {
+    readonly siteTitle?: string;
+    readonly shortSiteTitle?: string;
+    readonly separator?: string;
+}
+/**
+ * Compose `{title}{separator}{siteTitle}` while honouring the
+ * `(lang, surface)` budget. Drops the brand suffix entirely when the
+ * article title alone is already at or past the budget. Prefers the
+ * short site title when supplied and the full suffix doesn't fit.
+ *
+ * @param title - Article title (plain text)
+ * @param lang - Publishing locale
+ * @param surface - Target SEO surface (`title` / `ogTitle` / `twitterTitle`)
+ * @param opts - Optional brand suffix wiring
+ * @returns Composed title ≤ budget
+ */
+export declare function clampTitleForSurface(title: string, lang: LanguageCode | string, surface: SeoSurface, opts?: TitleSurfaceOptions): string;
+//# sourceMappingURL=seo-budgets.d.ts.map