euparliamentmonitor 0.9.21 → 0.9.23

package/scripts/aggregator/metadata/text-utils-constants.js

@@ -0,0 +1,209 @@
+// SPDX-FileCopyrightText: 2024-2026 Hack23 AB
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * @module Aggregator/Metadata/TextUtilsConstants
+ * @description Shared byte-budget constants and vocabularies used by
+ * the metadata text helpers. Extracted from `text-utils.ts` so the
+ * truncation/extraction helpers can live in `text-truncate.ts`
+ * without creating a circular import — both modules import from
+ * here, and `text-utils.ts` re-exports the truncators for back-compat
+ * with existing call-sites.
+ *
+ * **No imports.** This is a pure leaf module: only constants and
+ * vocabularies, no functions, no I/O.
+ */
+// ────────────────────────────────────────────────────────────────────────
+// Length budgets — meta description / title size envelopes
+// ────────────────────────────────────────────────────────────────────────
+/** Maximum `<meta description>` length we will emit. */
+export const DESCRIPTION_MAX_LENGTH = 180;
+/**
+ * Maximum `og:description` / `twitter:description` length we will
+ * emit. Facebook truncates at ~300 characters in the preview card;
+ * Twitter at ~200. We aim for the longer cap so LinkedIn / Slack
+ * (which use the full OG payload) get the full BLUF context, then
+ * let Twitter clip naturally. Below this length the extended
+ * description is emitted verbatim; above it we sentence-boundary
+ * truncate.
+ */
+export const EXTENDED_DESCRIPTION_MAX_LENGTH = 300;
+/** Target minimum extended-description length before we even emit it. */
+export const EXTENDED_DESCRIPTION_MIN_LENGTH = 200;
+/** Target minimum `<meta description>` length before we append context. */
+export const DESCRIPTION_MIN_LENGTH = 140;
+/**
+ * Length below which a raw description is considered too short to stand
+ * on its own and gets enriched with date/context. Independent from
+ * {@link DESCRIPTION_MIN_LENGTH} (which controls sentence-boundary
+ * truncation behaviour). Set lower than DESCRIPTION_MIN_LENGTH so a
+ * clean 100-140 char prose lede is preserved verbatim instead of being
+ * padded with date/context boilerplate.
+ */
+export const ENRICHMENT_TRIGGER_LENGTH = 100;
+/** Maximum `<title>` length — anything longer is truncated with an ellipsis. */
+export const TITLE_MAX_LENGTH = 140;
+/**
+ * Soft target for headline-style titles produced as a fallback from
+ * BLUF/lede prose. When the candidate exceeds `TITLE_MAX_LENGTH`, the
+ * truncator first looks for a natural clause boundary
+ * (`.`, `:`, `—`, `;`) inside the `[HEADLINE_SOFT_MIN, TITLE_MAX_LENGTH]`
+ * window and breaks there instead of mid-clause-with-ellipsis. This
+ * turns a 137-character truncated prose paragraph into a complete
+ * journalistic clause, which scans much better in news cards and SERP
+ * snippets without sacrificing the keyword-rich opening.
+ */
+export const HEADLINE_SOFT_MIN = 60;
+/**
+ * Lower floor for clause-boundary acceptance when the soft-min window
+ * returns nothing. Used by {@link truncateTitle} as a second-tier
+ * fallback: when a long prose paragraph has its only natural clause
+ * boundaries (`: `, ` — `) clustered in the opening 30-60 characters
+ * (typical of Reader-Briefing-style ledes like `Immediate priority:
+ * DMA enforcement — …`), accept the strongest such boundary rather
+ * than fall through to template-fallback composition. This keeps
+ * scan-friendly editorial fragments intact while still rejecting
+ * fragments shorter than a typical news-card title.
+ */
+export const HEADLINE_HARD_MIN = 30;
+/**
+ * Punctuation marks that signal a natural clause boundary inside a
+ * BLUF / lede paragraph. Listed in preferred-break order: a colon or
+ * em-dash that introduces a list of consequences is the best break,
+ * full stops are next, and semicolons last. Single ASCII space is
+ * always a fallback boundary handled separately.
+ */
+export const HEADLINE_CLAUSE_BOUNDARIES = [': ', ' — ', ' – ', '. ', '; ', ', '];
+// ────────────────────────────────────────────────────────────────────────
+// Banner / metadata-row vocabularies
+// ────────────────────────────────────────────────────────────────────────
+/**
+ * Emoji-banner prefixes that Stage-B agents use to decorate metadata rows
+ * (e.g. `📋 Analysis Owner:`). Any line starting with one of these is
+ * metadata, never prose.
+ */
+export const EMOJI_BANNER_CHARS = [
+    '📋',
+    '📅',
+    '🔍',
+    '🏛',
+    '📰',
+    '📊',
+    '🏷',
+    '📈',
+    '📉',
+    '⚠',
+    '🔔',
+    '🎯',
+    '🗳',
+    '🏢',
+    '📄',
+];
+/**
+ * Label prefixes that a prose description must never start with. Every
+ * entry matches case-insensitively at the start of a trimmed line, followed
+ * by optional space and a colon.
+ */
+export const METADATA_LINE_PREFIXES = [
+    'Admiralty Grade',
+    'Analysis Date',
+    'Analysis Owner',
+    'Article Type',
+    'Article Window',
+    'Assessment Date',
+    'Briefing',
+    'Briefing Date',
+    'Classification',
+    'Classification Date',
+    'Confidence',
+    'Confidence in Evidence',
+    'Data Sources',
+    'Date',
+    'Document Type',
+    'Filing Date',
+    'Generated',
+    'Horizon',
+    'IMF Status',
+    'Last Updated',
+    'Parliamentary Status',
+    'Parliamentary Term',
+    'Period',
+    'Prepared',
+    'Purpose',
+    'Region',
+    'Reporting',
+    'Reporting Period',
+    'Reporting Window',
+    'Run',
+    'Run ID',
+    'Series',
+    'Series Run',
+    'Source',
+    'Sources',
+    'SPDX-FileCopyrightText',
+    'SPDX-License-Identifier',
+    'Topic',
+    'Type',
+    // Bare `WEP:` (Words of Estimative Probability) lines appear in
+    // `intelligence/synthesis-summary.md` between a KJ-N heading and its
+    // prose body (e.g. `**WEP: ALMOST CERTAINLY (>95%)** | Admiralty: A1`).
+    // The line is grade/confidence metadata, not editorial prose — without
+    // this prefix it leaked into `<meta description>` as an all-caps shout
+    // (run #26223932441, propositions 2026-05-21).
+    'WEP',
+    'WEP Band',
+    'WEP Grade',
+    'Window',
+];
+// ────────────────────────────────────────────────────────────────────────
+// Trailing-cleanup vocabularies (used by truncation helpers)
+// ────────────────────────────────────────────────────────────────────────
+/** Connector / determiner words that read as broken copy when they are
+ *  the final token before a truncation ellipsis. */
+export const TRAILING_STOP_WORDS = new Set([
+    'the',
+    'a',
+    'an',
+    'of',
+    'to',
+    'for',
+    'in',
+    'on',
+    'at',
+    'by',
+    'and',
+    'or',
+    'with',
+    'from',
+]);
+/** Trailing characters we always strip before appending our own ellipsis,
+ *  so we never emit double-ellipsis or stray punctuation. */
+export const TRAILING_PUNCT = /[.,;:—\-…\s]/u;
+/**
+ * Abbreviation tokens (lowercase, including the trailing period) that
+ * should NOT count as sentence terminators when `extractFirstSentence`
+ * scans for a `.` boundary. Single-letter all-caps initials
+ * (`U.S.`, `E.U.`) are handled by the all-caps-initial check.
+ */
+export const ABBREVIATION_PREFIXES = [
+    'mr.',
+    'mrs.',
+    'ms.',
+    'dr.',
+    'st.',
+    'no.',
+    'vs.',
+    'e.g.',
+    'i.e.',
+    'etc.',
+    'cf.',
+    'al.',
+    // EP fiscal-year and quarter shorthand: Q1., Q2., Q3., Q4., H1., H2., FY.
+    'q1.',
+    'q2.',
+    'q3.',
+    'q4.',
+    'h1.',
+    'h2.',
+    'fy.',
+];
+//# sourceMappingURL=text-utils-constants.js.map

package/scripts/aggregator/metadata/text-utils.d.ts

@@ -1,97 +1,21 @@
 /**
  * @module Aggregator/Metadata/TextUtils
- * @description Pure text / Markdown utility helpers extracted from
- * `article-metadata.ts` as a leaf module in the `metadata/` bounded
- * context. Every helper here is concerned with **how to massage a
- * string** into a meta-tag-safe shape — strip Markdown decorations,
- * recognise banner / metadata rows that must never reach the
- * description, clamp text to byte budgets without producing broken
- * copy, and identify the first complete sentence in a prose paragraph.
+ * @description Pure text / Markdown classification + label-stripping
+ * helpers used by the metadata resolver chain. Constants live in
+ * `text-utils-constants.ts`; byte-budget truncators and sentence-
+ * extraction live in `text-truncate.ts`. This file re-exports the
+ * full public surface so existing call-sites keep working.
  *
- * Bounded-context rules for this file:
- * - **No upward imports** — pure helpers, no dependencies on other
- *   `src/aggregator/` modules, no I/O, no globals.
- * - **Deterministic** — same input always produces same output; safe to
- *   property-test.
+ * Bounded-context rules:
+ * - **No upward imports** — pure helpers, no I/O, no globals.
+ * - **Deterministic** — same input always produces same output.
  * - **Locale-agnostic** — every helper works on raw Markdown / prose
  *   in any of the 14 publishing languages. Banner-row detection is
  *   driven by structural shape (double-bold + pipe-separator), not by
  *   a hard-coded English vocabulary.
- *
- * The companion file `article-metadata.ts` re-exports the public surface
- * for back-compat. New code should import directly from this module.
- */
-/** Maximum `<meta description>` length we will emit. */
-export declare const DESCRIPTION_MAX_LENGTH = 180;
-/**
- * Maximum `og:description` / `twitter:description` length we will
- * emit. Facebook truncates at ~300 characters in the preview card;
- * Twitter at ~200. We aim for the longer cap so LinkedIn / Slack
- * (which use the full OG payload) get the full BLUF context, then
- * let Twitter clip naturally. Below this length the extended
- * description is emitted verbatim; above it we sentence-boundary
- * truncate the same way as {@link truncateDescription}.
- */
-export declare const EXTENDED_DESCRIPTION_MAX_LENGTH = 300;
-/** Target minimum extended-description length before we even emit it. */
-export declare const EXTENDED_DESCRIPTION_MIN_LENGTH = 200;
-/** Target minimum `<meta description>` length before we append context. */
-export declare const DESCRIPTION_MIN_LENGTH = 140;
-/**
- * Length below which a raw description is considered too short to stand
- * on its own and gets enriched with date/context. Independent from
- * {@link DESCRIPTION_MIN_LENGTH} (which controls sentence-boundary
- * truncation behaviour). Set lower than DESCRIPTION_MIN_LENGTH so a
- * clean 100-140 char prose lede is preserved verbatim instead of being
- * padded with date/context boilerplate.
- */
-export declare const ENRICHMENT_TRIGGER_LENGTH = 100;
-/** Maximum `<title>` length — anything longer is truncated with an ellipsis. */
-export declare const TITLE_MAX_LENGTH = 140;
-/**
- * Soft target for headline-style titles produced as a fallback from
- * BLUF/lede prose. When the candidate exceeds `TITLE_MAX_LENGTH`, the
- * truncator first looks for a natural clause boundary
- * (`.`, `:`, `—`, `;`) inside the `[HEADLINE_SOFT_MIN, TITLE_MAX_LENGTH]`
- * window and breaks there instead of mid-clause-with-ellipsis. This
- * turns a 137-character truncated prose paragraph into a complete
- * journalistic clause, which scans much better in news cards and SERP
- * snippets without sacrificing the keyword-rich opening.
- */
-export declare const HEADLINE_SOFT_MIN = 60;
-/**
- * Punctuation marks that signal a natural clause boundary inside a
- * BLUF / lede paragraph. Listed in preferred-break order: a colon or
- * em-dash that introduces a list of consequences is the best break,
- * full stops are next, and semicolons last. Single ASCII space is
- * always a fallback boundary handled separately.
  */
-export declare const HEADLINE_CLAUSE_BOUNDARIES: readonly string[];
-/**
- * Emoji-banner prefixes that Stage-B agents use to decorate metadata rows
- * (e.g. `📋 Analysis Owner:`). Any line starting with one of these is
- * metadata, never prose.
- */
-export declare const EMOJI_BANNER_CHARS: string[];
-/**
- * Label prefixes that a prose description must never start with. Every
- * entry matches case-insensitively at the start of a trimmed line, followed
- * by optional space and a colon.
- */
-export declare const METADATA_LINE_PREFIXES: readonly string[];
-/** Connector / determiner words that read as broken copy when they are
- *  the final token before a truncation ellipsis. */
-export declare const TRAILING_STOP_WORDS: Set<string>;
-/** Trailing characters we always strip before appending our own ellipsis,
- *  so we never emit double-ellipsis or stray punctuation. */
-export declare const TRAILING_PUNCT: RegExp;
-/**
- * Abbreviation tokens (lowercase, including the trailing period) that
- * should NOT count as sentence terminators when {@link extractFirstSentence}
- * scans for a `.` boundary. Single-letter all-caps initials
- * (`U.S.`, `E.U.`) are handled by the all-caps-initial check below.
- */
-export declare const ABBREVIATION_PREFIXES: readonly string[];
+export { ABBREVIATION_PREFIXES, DESCRIPTION_MAX_LENGTH, DESCRIPTION_MIN_LENGTH, EMOJI_BANNER_CHARS, ENRICHMENT_TRIGGER_LENGTH, EXTENDED_DESCRIPTION_MAX_LENGTH, EXTENDED_DESCRIPTION_MIN_LENGTH, HEADLINE_CLAUSE_BOUNDARIES, HEADLINE_SOFT_MIN, METADATA_LINE_PREFIXES, TITLE_MAX_LENGTH, TRAILING_PUNCT, TRAILING_STOP_WORDS, } from './text-utils-constants.js';
+export { extractFirstSentence, stripTrailingStopWordsAndPunctuation, truncateDescription, truncateExtendedDescription, truncateTitle, } from './text-truncate.js';
 /**
  * Return `true` when a line cannot serve as a prose description. Rejects
  * Markdown structural lines (headings, blockquotes, tables, HTML),
@@ -118,6 +42,28 @@ export declare function shouldSkipDescriptionLine(line: string): boolean;
  * @returns Line with the all-caps opener removed
  */
 export declare function stripLeadingProseLabel(line: string): string;
+/**
+ * Strip a leading `**Label:**` / `**Label：**` prefix from a Markdown
+ * BLUF line, in any of the 14 publishing languages. Translated
+ * executive briefs open the `## FOR IMMEDIATE ACTION` section with
+ * patterns such as `**Issue:** …`, `**Fråga:** …`, `**Asunto:** …`,
+ * `**主題:** …`, `**الموضوع:** …`, `**Thema:** …`, `**Sujet :** …` —
+ * without this stripper the localized label leaked into
+ * `<meta description>` for every non-English locale (the English
+ * `**Issue:**` line is already filtered by `METADATA_LINE_PREFIXES`).
+ *
+ * The matcher is *structural*, not vocabulary-driven: it accepts up to
+ * 5 word/glyph tokens (letters, marks, digits, spaces, hyphens),
+ * followed by either an ASCII colon `:` or full-width colon `：`,
+ * followed by `**`, followed by whitespace. Returns the line verbatim
+ * when no qualifying opener is present so it is safe to apply
+ * unconditionally.
+ *
+ * @param raw - Raw Markdown line (still carrying `**…**` decorations)
+ * @returns Line with the leading `**Label:**` prefix removed, or the
+ * original input when no such prefix exists
+ */
+export declare function stripLeadingBoldLabel(raw: string): string;
 /**
  * Strip inline Markdown decorations so we can use the remaining text as
  * plain-text meta-tag content. Removes link syntax, emphasis, inline code
@@ -128,61 +74,4 @@ export declare function stripLeadingProseLabel(line: string): string;
  * @returns Plain-text variant
  */
 export declare function stripInlineMarkdown(raw: string): string;
-/**
- * Clamp a string to `DESCRIPTION_MAX_LENGTH` characters, appending
- * an ellipsis when truncation actually happens. Does not break words if
- * avoidable — a trailing partial word is trimmed back to the previous
- * space first.
- *
- * @param text - Raw description text
- * @returns Truncated description with trailing ellipsis when clipped
- */
-export declare function truncateDescription(text: string): string;
-/**
- * Clamp an extended description to {@link EXTENDED_DESCRIPTION_MAX_LENGTH}
- * characters using the same sentence-boundary-preserving logic as
- * {@link truncateDescription}. Returns `''` when the input is empty
- * or shorter than the meta-description maximum (no point in emitting
- * an "extended" description that's actually shorter than the regular
- * one).
- *
- * @param text - Raw extended-description text (e.g. full BLUF paragraph)
- * @returns Truncated extended description, or `''` when not worth emitting
- */
-export declare function truncateExtendedDescription(text: string): string;
-/**
- * Clamp a title to `TITLE_MAX_LENGTH` characters in the same
- * word-boundary-preserving fashion as {@link truncateDescription}.
- *
- * @param text - Raw title text
- * @returns Truncated title with trailing ellipsis when clipped
- */
-export declare function truncateTitle(text: string): string;
-/**
- * Return the first complete sentence from a prose paragraph, suitable
- * for use as a fallback editorial title when the artefact H1 is
- * categorical (e.g. `# EU Parliament Committee Reports`) and the
- * resolver must derive `<title>` from the BLUF / lede summary instead.
- *
- * A "sentence" is the prefix up to the first sentence-terminator
- * (`. `, `! `, `? `, `; `) inside the `[HEADLINE_SOFT_MIN,
- * TITLE_MAX_LENGTH]` window. Common abbreviations (`Q1.`, `Q2.`,
- * `H1.`, `H2.`, `Mr.`, `Mrs.`, `e.g.`, `i.e.`, `vs.`) are skipped
- * so they don't terminate the sentence prematurely. When no
- * acceptable terminator exists in the window, returns the entire
- * input unchanged so {@link truncateTitle} can handle clause-boundary
- * truncation downstream.
- *
- * This produces journalistically clean titles even for the
- * propositions / committee-reports cases where the BLUF paragraph
- * opens with a single long sentence that exceeds 140 chars —
- * `truncateTitle` then breaks on a clause boundary, and the result is
- * still grammatical because the input was a sentence prefix rather
- * than an arbitrary paragraph slice.
- *
- * @param paragraph - Prose paragraph (post-{@link stripInlineMarkdown})
- * @returns First sentence, or the original paragraph when none can be
- *   identified within the soft-min window
- */
-export declare function extractFirstSentence(paragraph: string): string;
 //# sourceMappingURL=text-utils.d.ts.map