euparliamentmonitor 0.8.41 → 0.8.43

package/README.md

@@ -131,15 +131,24 @@ import {
 
 **MCP Server Integration**: The project uses the
 [European-Parliament-MCP-Server](https://github.com/Hack23/European-Parliament-MCP-Server)
-v1.2.11 for accessing real EU Parliament data via the Model Context Protocol.
+v1.2.13 for accessing real EU Parliament data via the Model Context Protocol.
 
 - **MCP Server Status**: ✅ Fully operational — 60+ EP data tools available
   (feeds, direct lookups, analytical tools, intelligence correlation)
-- **Agentic Workflows**: 10 gh-aw markdown workflows (compiled with
-  `gh-aw v0.69.3`) for automated news generation with AI-driven political
+- **Agentic Workflows**: 9 unified gh-aw markdown workflows — 8 article types (`news-<type>.md`, Stages A → B → C → D → E in one session) + `news-translate.md` (14-language flush translation) — compiled with
+  `gh-aw v0.69.3` to `.lock.yml` for automated news generation with AI-driven political
   intelligence analysis. See [`.github/workflows/README.md`](.github/workflows/README.md).
-- **Analysis Chain**: 5-stage pipeline (Data → Analysis → Completeness Gate →
-  Article → Single PR) producing 39 structured analysis templates per run.
+- **Analysis-Artifact-Driven Article Pipeline**: Agents author the full
+  Stage-B analysis-artifact set (`analysis/daily/<date>/<slug>-run<NN>/`, 39
+  structured templates per run) and commit it; the deterministic aggregator
+  (`src/aggregator/**`, invoked via
+  `npm run generate-article -- --run <analysis-run-dir>` or
+  `npm run generate-article:all` for batch regen) then walks `manifest.json`,
+  cleans each artifact, and emits the final Markdown source + HTML with the
+  shared site chrome (stacked header + embedded 14-language switcher + TOC
+  sidebar + footer stats) and 14-language `<link rel="alternate" hreflang>`
+  entries. There is no AI-authored HTML step; article quality is guaranteed
+  editorially at the Stage-C completeness review over the artifact markdown.
   See [`analysis/README.md`](analysis/README.md) and
   [`analysis/methodologies/ai-driven-analysis-guide.md`](analysis/methodologies/ai-driven-analysis-guide.md).
 - **Fallback Mode**: News generation can work with reduced data when EP API
@@ -168,6 +177,13 @@ covering:
 - 🤖 **GitHub Actions Integration**: Automated daily news generation
 - 📊 **SEO Optimized**: Proper metadata, structured data, and sitemap generation
 - ✅ **Code Quality**: ESLint, Prettier, and automated quality gates
+- 🌐 **IMF-grade Economic Context**: IMF-primary citations are the
+  editorial standard for every policy article — see
+  [`analysis/imf/README.md`](analysis/imf/README.md) and
+  [`analysis/methodologies/imf-indicator-mapping.md`](analysis/methodologies/imf-indicator-mapping.md).
+  World Bank is retained only for non-economic indicators (health,
+  education, social, environment, demographics, defence, agriculture,
+  innovation, governance).
 
 ## 🔒 Security Architecture
 
@@ -398,19 +414,17 @@ import {
   // MCP Client for EU Parliament data
   EuropeanParliamentMCPClient,
   getEPMCPClient,
-  // Intelligence analysis
-  scoreVotingAnomaly,
-  analyzeCoalitionCohesion,
-  assessPoliticalThreats,
-  // Article generation
-  generateArticleHTML,
-  scoreArticleQuality,
+  // Intelligence index (cross-article relationships)
+  buildIndexFromEntries,
+  findRelatedArticles,
+  generateCrossReferences,
+  // Article generation (aggregator pipeline)
+  generateArticle,
+  aggregateAnalysisRun,
+  renderMarkdown,
   // Multi-language support (14 languages)
   ALL_LANGUAGES,
   LANGUAGE_NAMES,
-  // Content validation
-  validateArticleContent,
-  validateTranslationCompleteness,
 } from 'euparliamentmonitor';
 
 // Or import specific modules for tree-shaking
@@ -459,27 +473,119 @@ without it using placeholder content.
 
 ### Generate News Articles
 
+Agentic workflows generate articles automatically. For manual article generation from an analysis run:
+
 ```bash
-# Generate week ahead article in English
-npm run generate-news -- --types=week-ahead --languages=en
+# Render one article from a specific analysis run directory
+npm run generate-article -- --run analysis/daily/2025-01-01/breaking
 
-# Generate multiple article types in multiple languages
-npm run generate-news -- --types=week-ahead,committee-reports --languages=en,de,fr
+# Regenerate EVERY committed analysis run in one pass (skips runs whose
+# manifest.json has no valid articleType; collision-suffixes same-date /
+# same-type runs with a sanitised runId)
+npm run generate-article:all
 
-# Generate in all eu-core preset languages
-npm run generate-news -- --types=week-ahead --languages=eu-core
+# Only regenerate runs from a given date onward
+npm run generate-article -- --all --since 2026-04-01
+```
+
+Each invocation writes:
+
+- `news/<slug>.en.md` — aggregated Markdown (provenance block + 19-section
+  ordered artifact set + Tradecraft References appendix + Analysis Index appendix)
+- `news/<slug>-<lang>.html` — 14 language variants, each wrapped in the
+  shared site chrome (stacked header with embedded language switcher,
+  article-level Table of Contents sidebar, shared footer with live
+  article-count stats)
+
+The aggregator lives under `src/aggregator/` — see the
+[aggregator pipeline](ARCHITECTURE.md#aggregator-pipeline) section of the
+architecture doc for the full data flow.
+
+#### Editorial-highlight metadata resolver
+
+The `<title>` / `<meta description>` / Open Graph / Twitter / JSON-LD
+`NewsArticle` fields for every article are resolved by
+`src/aggregator/article-metadata.ts` through a 5-tier priority ladder:
+
+1. **Manifest override** — the Stage-B agent writes one of these keys in
+   `manifest.json` when it has an editorial headline:
+
+   ```jsonc
+   {
+     "articleType": "breaking",
+     // String form — applied to all 14 language variants (recommended
+     // when only an English headline is available):
+     "title": "Banking Union Breakthrough and Anti-Corruption Landmark",
+     "description": "The plenary closes a six-year debate and triggers immediate criticism from two national delegations about implementation timelines.",
+     // OR per-language form when translations already exist:
+     "title": { "en": "…", "sv": "…", "de": "…" },
+     "description": { "en": "…", "sv": "…" }
+   }
+   ```
+
+2. **First artefact H1** — the aggregator walks the manifest's ordered
+   artefact list (`intelligence/synthesis-summary.md`,
+   `executive-summary.md`, `breaking-news-analysis.md`, …) and promotes
+   the first non-generic `# …` heading.
+3. **Aggregated-markdown H1** — any non-generic top-level heading in the
+   rendered Markdown.
+4. **First strong prose paragraph** — with a tightened leak filter that
+   blocks mermaid `%%{init}` blocks, `title …` directives, emoji-banner
+   metadata, and `Analysis Date:` / `Classification:` / `Run:` / `Window:`
+   / `Purpose:` / `BLUF (ICD-203):` style rows.
+5. **Localized template** — `*_TITLES(date)` from
+   `src/constants/language-articles.ts` — last resort when no editorial
+   content exists at all.
+
+Downstream generators (`news-indexes.ts`, `sitemap.ts`, political-intelligence
+cards, RSS) call `extractArticleMeta()` in `src/utils/file-utils.ts`, which
+reads the `<head><title>` value (with the ` — EU Parliament Monitor`
+suffix stripped) as the primary title, so the resolver's output
+propagates everywhere without separate code changes in each generator.
+
+#### Backport SEO metadata into existing articles
+
+`scripts/backport-article-seo.js` rewrites `<title>`, the meta
+description, Open Graph, Twitter, and JSON-LD `headline`/`description`
+for every already-rendered `news/*-<lang>.html` file by extracting the
+first editorial H1 and first strong prose paragraph from the article
+body itself. The rewrite is idempotent — re-running over a backported
+file is a byte-identical no-op — and the article body is never
+modified.
+
+```bash
+# Preview the change-set (dry-run is the default; prints a per-article-
+# type summary and up to 12 sample before/after diffs):
+node scripts/backport-article-seo.js
 
-# Generate in all supported languages
-npm run generate-news -- --types=week-ahead --languages=all
+# Write changes:
+node scripts/backport-article-seo.js --apply
+
+# Scope to a subset of article types:
+node scripts/backport-article-seo.js --apply --only breaking,motions
+
+# Use a non-default news directory:
+node scripts/backport-article-seo.js --apply --dir news
+```
+
+After backporting, regenerate the downstream surfaces that depend on
+article metadata:
+
+```bash
+node scripts/generators/news-indexes.js   # 14 language index pages
+node scripts/generators/sitemap.js        # 14 sitemap HTMLs + sitemap.xml + rss.xml
 ```
 
 ### Generate Indexes and Sitemap
 
 ```bash
-# Generate language-specific index pages
+# Generate per-run news index pages (consumed by the aggregator's
+# transparency footer and political-intelligence.html)
 npm run generate-news-indexes
 
-# Generate sitemap.xml
+# Generate sitemap.xml, sitemap_<lang>.html, and political-intelligence_<lang>.html
+# (14 language-specific sitemap pages + 14 language-specific Political Intelligence
+#  pages that index every methodology, template, and daily analysis run)
 npm run generate-sitemap
 ```
 
@@ -523,7 +629,11 @@ euparliamentmonitor/
 ├── index-{lang}.html                # Language-specific index pages
 ├── typedoc.json                     # TypeDoc configuration
 ├── tsconfig.json                    # TypeScript configuration
-├── sitemap.xml                      # SEO sitemap
+├── sitemap.xml                      # SEO sitemap with hreflang alternates
+├── sitemap.html                     # Human-readable sitemap (English)
+├── sitemap_{lang}.html              # Per-language human-readable sitemaps
+├── political-intelligence.html      # Index of every methodology + template + daily analysis run
+├── political-intelligence_{lang}.html # Localized political-intelligence pages
 └── package.json                     # Project dependencies
 ```
 

package/SECURITY.md

@@ -88,15 +88,20 @@ See [SECURITY_ARCHITECTURE.md - Security Testing](SECURITY_ARCHITECTURE.md#-secu
 ### Scope
 
 **In Scope**:
-- News generation scripts (scripts/)
+- News generation scripts (`scripts/`)
+- Analysis-artifact aggregator (`src/aggregator/**` — `artifact-order.ts`, `clean-artifact.ts`, `analysis-aggregator.ts`, `markdown-renderer.ts`, `article-html.ts`, `article-generator.ts` CLI)
+- HTML sanitiser (`src/utils/html-sanitize.ts`) and the `markdown-it` render pipeline with plugins (`markdown-it-anchor`, `markdown-it-footnote`, `markdown-it-attrs`, `markdown-it-deflist`)
+- MCP clients (`src/mcp/**` — European Parliament, IMF, World Bank)
+- Committed analysis artifacts under `analysis/daily/**` (as attack surface for aggregator rendering)
+- Vendored client-side diagram renderer (`js/mermaid.esm.min.mjs`) and its CSP `script-src 'self'` constraint
 - HTML templates and output
-- MCP client integration
-- GitHub Actions workflows
-- Dependencies and supply chain
+- GitHub Actions workflows and gh-aw agentic workflows (`.github/workflows/news-*.md`)
+- AWS S3 + CloudFront deployment pipeline (`deploy-s3.yml`)
+- Dependencies and supply chain (OpenSSF Scorecard + SLSA L3 provenance)
 
 **Out of Scope**:
-- Third-party services (GitHub, European Parliament APIs)
-- Infrastructure (GitHub Pages hosting)
+- Third-party services (GitHub, European Parliament APIs, IMF SDMX REST, World Bank MCP)
+- Infrastructure (AWS account-level, GitHub Pages hosting as fallback)
 - Client-side browser vulnerabilities (not controlled by this project)
 
 ### Recognition and Anonymity

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "euparliamentmonitor",
-  "version": "0.8.41",
+  "version": "0.8.43",
   "type": "module",
   "description": "European Parliament Intelligence Platform - Monitor political activity with systematic transparency",
   "main": "scripts/index.js",
@@ -60,15 +60,11 @@
     "build": "tsc",
     "build:check": "tsc --noEmit",
     "build:check-tests": "tsc --project tsconfig.test.json --noEmit",
-    "copy-vendor": "mkdir -p js/vendor && cp node_modules/chart.js/dist/chart.umd.min.js js/vendor/ && cp node_modules/chartjs-plugin-annotation/dist/chartjs-plugin-annotation.min.js js/vendor/ && cp node_modules/d3/dist/d3.min.js js/vendor/",
-    "generate-news": "node scripts/generators/news-enhanced.js",
+    "copy-vendor": "mkdir -p js/vendor && cp node_modules/chart.js/dist/chart.umd.min.js js/vendor/ && cp node_modules/chartjs-plugin-annotation/dist/chartjs-plugin-annotation.min.js js/vendor/ && cp node_modules/d3/dist/d3.min.js js/vendor/ && (test -f node_modules/mermaid/dist/mermaid.esm.min.mjs && cp node_modules/mermaid/dist/mermaid.esm.min.mjs js/vendor/ || echo 'mermaid vendor asset not installed; skipping')",
+    "generate-article": "node scripts/aggregator/article-generator.js",
+    "generate-article:all": "node scripts/aggregator/article-generator.js --all",
     "generate-news-indexes": "node scripts/generators/news-indexes.js",
     "generate-sitemap": "node scripts/generators/sitemap.js",
-    "fix-articles": "npx tsx src/utils/fix-articles.ts",
-    "retrofit-analysis": "npx tsx src/utils/retrofit-analysis-links.ts",
-    "validate-articles": "npx tsx src/utils/validate-articles.ts",
-    "validate-articles:strict": "npx tsx src/utils/validate-articles.ts --strict",
-    "validate-analysis": "npx tsx src/utils/validate-analysis-completeness.ts",
     "validate-ep-api": "npx tsx src/utils/validate-ep-api.ts",
     "lint:prompts": "node scripts/lint-prompts.js",
     "htmlhint": "sh -c 'htmlhint *.html; set -- news/*.html; if [ -e \"$1\" ]; then htmlhint \"$@\"; else echo \"No news/*.html files to lint\"; fi'",
@@ -141,6 +137,7 @@
     "@eslint/js": "10.0.1",
     "@playwright/test": "1.59.1",
     "@types/d3": "7.4.3",
+    "@types/markdown-it": "^14.1.2",
     "@types/node": "25.6.0",
     "@types/papaparse": "5.5.2",
     "@typescript-eslint/eslint-plugin": "8.59.0",
@@ -172,7 +169,12 @@
     "node": ">=25"
   },
   "dependencies": {
-    "european-parliament-mcp-server": "1.2.11"
+    "european-parliament-mcp-server": "1.2.13",
+    "markdown-it": "^14.1.1",
+    "markdown-it-anchor": "^9.2.0",
+    "markdown-it-attrs": "^4.3.1",
+    "markdown-it-deflist": "^3.0.0",
+    "markdown-it-footnote": "^4.0.0"
   },
   "optionalDependencies": {
     "worldbank-mcp": "1.0.1"

package/scripts/aggregator/analysis-aggregator.d.ts

@@ -0,0 +1,168 @@
+import { type ArtifactSection } from './artifact-order.js';
+/** Raw manifest shape as committed by the analysis pipeline. */
+export interface AnalysisManifest {
+    readonly articleType: string;
+    readonly runId?: string;
+    readonly date?: string;
+    readonly analysisDir?: string;
+    readonly files?: ManifestFiles;
+    readonly history?: readonly ManifestHistoryEntry[];
+}
+/** `manifest.files` can be nested category → paths or flat path → description. */
+export type ManifestFiles = Record<string, readonly string[] | Record<string, string>>;
+/** One entry in `manifest.history[]`; only fields we read are typed. */
+export interface ManifestHistoryEntry {
+    readonly stage?: string;
+    readonly completedAt?: string;
+    readonly startedAt?: string;
+    readonly finishedAt?: string;
+    readonly runId?: string;
+    readonly gateResult?: string;
+    readonly summary?: string;
+    readonly filesWritten?: readonly string[];
+}
+/** Result of {@link aggregateAnalysisRun}. */
+export interface AggregatedRun {
+    /** Final Markdown document (provenance + sections + appendices). */
+    readonly markdown: string;
+    /** Repo-relative path of the run dir (e.g. `analysis/daily/2026-01-15/breaking-run1`). */
+    readonly runDirRelPath: string;
+    /** Article type slug from the manifest. */
+    readonly articleType: string;
+    /** ISO date string of the run (YYYY-MM-DD). */
+    readonly date: string;
+    /** List of every artifact included, in the order they appear. */
+    readonly includedArtifacts: readonly IncludedArtifact[];
+    /** Latest resolved gate result read from `manifest.history[]`. */
+    readonly gateResult: string;
+    /**
+     * Ordered list of top-level (H2) sections that were actually emitted into
+     * the aggregate — used by the HTML renderer to build the article
+     * table-of-contents sidebar. Includes canonical sections, the
+     * supplementary-intelligence bucket, the tradecraft-references appendix,
+     * and the analysis-index appendix, in document order.
+     */
+    readonly sectionToc: readonly TocSection[];
+}
+/** One entry in the article-level table of contents (H2 level). */
+export interface TocSection {
+    /** Fragment identifier — matches the `id="…"` on the rendered H2. */
+    readonly id: string;
+    /** Display title shown in the sidebar nav. */
+    readonly title: string;
+}
+/** Metadata for one artifact included in the aggregate. */
+export interface IncludedArtifact {
+    /** Path relative to the run dir. */
+    readonly runRelPath: string;
+    /** Path relative to the repo root. */
+    readonly repoRelPath: string;
+    /** Id of the section this artifact belongs to. */
+    readonly sectionId: string;
+}
+/** Options for {@link aggregateAnalysisRun}. */
+export interface AggregateOptions {
+    /** Absolute path to the analysis run directory. */
+    readonly runDir: string;
+    /** Absolute path to the repo root (used to build repo-relative paths). */
+    readonly repoRoot: string;
+    /**
+     * Optional: all methodology files and template files that should be
+     * listed in the tradecraft appendix. If omitted, the aggregator
+     * discovers them under `analysis/methodologies/*.md` +
+     * `analysis/templates/*.md`.
+     */
+    readonly tradecraftFiles?: readonly string[];
+}
+/**
+ * Normalise `manifest.files` into a flat list of `runRelPath` strings.
+ *
+ * @param files - Manifest `files` section (nested or flat)
+ * @returns De-duplicated list of run-relative artifact paths
+ */
+export declare function flattenManifestFiles(files: ManifestFiles | undefined): string[];
+/**
+ * Pick the latest non-PENDING gateResult from `manifest.history[]`, falling
+ * back to `PENDING` if none is recorded. Mirrors the behaviour of
+ * {@link readLatestResolvedGateResult} in `src/utils/file-utils.ts` but
+ * operates on an in-memory manifest.
+ *
+ * @param manifest - Parsed manifest object
+ * @returns The latest non-PENDING gate result, or `"PENDING"` when none found
+ */
+export declare function latestGateResult(manifest: AnalysisManifest): string;
+/**
+ * Expand an `artifacts` entry from {@link ArtifactSection} into a list of
+ * concrete artifact paths. Exact paths are kept as-is; directory prefixes
+ * ending in `/` expand to every remaining `.md` under that directory
+ * (lexical order), excluding files already claimed by higher-priority
+ * sections.
+ *
+ * @param section - Canonical section descriptor from {@link ARTIFACT_SECTIONS}
+ * @param available - Set of every known artifact path (run-relative)
+ * @param consumed - Mutable set of paths already claimed by earlier sections
+ * @returns Ordered list of artifact paths that belong to this section
+ */
+export declare function expandSectionArtifacts(section: ArtifactSection, available: Set<string>, consumed: Set<string>): string[];
+/**
+ * Discover tradecraft files (methodologies + templates) under a repo root.
+ * Returned paths are repo-relative with POSIX separators and sorted
+ * lexically.
+ *
+ * @param repoRoot - Absolute path of the repo root
+ * @returns Sorted list of `analysis/methodologies/*.md` + `analysis/templates/*.md`
+ */
+export declare function discoverTradecraftFiles(repoRoot: string): string[];
+/**
+ * Render the provenance block shown at the very top of the aggregated
+ * document. Shows run metadata, gate result, and a direct link to the
+ * manifest on GitHub so the reader can audit the full artifact set.
+ *
+ * @param params - Provenance metadata for the aggregated run
+ * @param params.articleType - Article type slug (e.g. `breaking`)
+ * @param params.date - ISO date of the run (`YYYY-MM-DD`)
+ * @param params.runId - Stable identifier for the run
+ * @param params.gateResult - Latest non-PENDING gate result
+ * @param params.runDirRelPath - Repo-relative path of the run directory
+ * @param params.manifestRelPath - Repo-relative path of `manifest.json`
+ * @returns Markdown blockquote ready to be concatenated into the aggregate
+ */
+export declare function renderProvenanceBlock(params: {
+    articleType: string;
+    date: string;
+    runId: string;
+    gateResult: string;
+    runDirRelPath: string;
+    manifestRelPath: string;
+}): string;
+/**
+ * Render the tradecraft-references appendix — one bullet per
+ * methodology/template file with a GitHub blob link.
+ *
+ * @param files - Repo-relative paths under `analysis/methodologies/` and
+ *                `analysis/templates/`
+ * @returns Markdown block with two subsections (methodologies, templates)
+ */
+export declare function renderTradecraftAppendix(files: readonly string[]): string;
+/**
+ * Render the analysis-index appendix — a compact table of every included
+ * artifact and its section, plus a direct link to the manifest.
+ *
+ * @param included - Artifacts that contributed to the aggregated document
+ * @param manifestRelPath - Repo-relative path of `manifest.json`
+ * @returns Markdown block with the index table
+ */
+export declare function renderAnalysisIndex(included: readonly IncludedArtifact[], manifestRelPath: string): string;
+/**
+ * Read, clean, and concatenate every artifact declared by the run's manifest
+ * (with discovery fallback when manifest.files is missing), returning a
+ * single aggregated Markdown document.
+ *
+ * The function is deterministic: given the same inputs it produces the
+ * same output byte-for-byte.
+ *
+ * @param options - Aggregation options (run dir, repo root, tradecraft files)
+ * @returns {@link AggregatedRun} describing the rendered document
+ */
+export declare function aggregateAnalysisRun(options: AggregateOptions): AggregatedRun;
+//# sourceMappingURL=analysis-aggregator.d.ts.map