@relvehq/agent-ready 0.4.2 → 0.4.3

package/dist/wp/relve-agent-ready/includes/class-relve-ar-converter.php

@@ -7,8 +7,11 @@
  * headings, paragraphs, br, strong/em/del, links, images (alt + src), nested
  * ul/ol, blockquote, pre/code (language from the class), inline code, GFM
  * tables, hr. Scripts, styles, forms, iframes, nav, svg and hidden elements
- * are dropped, as are shortcodes a page builder left behind. Whitespace is
- * collapsed and URLs are made absolute against the page URL.
+ * are dropped, as are shortcodes a page builder left behind and loading
+ * placeholders. Accordions are opened first (the question kept as text, the
+ * panel shown), and a <summary> keeps its heading or turns bold. Whitespace
+ * is collapsed and URLs are made absolute against the page URL. The
+ * accordion and placeholder rules are the package's (core/convert.ts).
  *
  * Pure PHP (ext-dom): no WordPress calls.
  *
@@ -30,6 +33,27 @@ class Relve_AR_Converter {
 	/** Elements that start a new block. Everything else is inline. */
 	const BLOCKS = array( 'address', 'article', 'aside', 'blockquote', 'center', 'dd', 'details', 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'ol', 'p', 'pre', 'section', 'summary', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th', 'ul', 'body', 'html', 'caption' );
 
+	/** Class words that mark an accordion panel or FAQ answer. */
+	const DISCLOSURE_CLASS_RE = '/(^|[\s_-])(accordion|faqs?)([\s_-]|$)/i';
+
+	/** Trigger roles that are not accordions. */
+	const NOT_A_DISCLOSURE_ROLE = '/^(tab|menuitem|combobox|option)$/i';
+
+	/** Panel roles that are not accordion panels. */
+	const NOT_A_PANEL_ROLE = '/^(menu|menubar|listbox|dialog|alertdialog|navigation|tree|grid)$/i';
+
+	/** Expander labels: controls, not content. Their panel opens, the label goes. */
+	const CONTROL_LABEL_RE = '/^(?:(?:show|read|view|see|load)\s+(?:more|less|all)|expand|collapse|toggle|open|close)\b/i';
+
+	/** Class or id words of loading placeholders. */
+	const PLACEHOLDER_RE = '/(^|[\s_-])(skeleton|shimmer|spinner|loader|loading)([\s_-]|$)|(^|\s)animate-(pulse|spin)(\s|$)/i';
+
+	/** A block whose whole text is a loading message. */
+	const LOADING_TEXT_RE = '/^(?:loading|fetching|please wait)\b[^.!?]{0,40}(?:…|\.{3})$|^(?:loading|fetching|please wait)\.?$/iu';
+
+	/** Placeholders hold less text than this. */
+	const PLACEHOLDER_MAX_TEXT = 200;
+
 	/** Absolute URL of the page: base for relative links and images. @var string */
 	private $base;
 
@@ -81,6 +105,8 @@ class Relve_AR_Converter {
 		if ( ! $body ) {
 			return '';
 		}
+		$this->open_disclosures( $doc );
+		$this->drop_placeholders( $doc );
 		$this->clean( $body );
 		$blocks = $this->blocks( $body );
 		return trim( implode( "\n\n", $blocks ) );
@@ -121,6 +147,158 @@ class Relve_AR_Converter {
 		}
 	}
 
+	/**
+	 * Accordions and FAQ toggles: the question is a <button> wired to its
+	 * panel by aria-controls, and a closed panel is hidden. Show the panel and
+	 * keep the question as text, before buttons and hidden elements are
+	 * dropped. Tabs, popups and menus are left alone, and so is a panel that
+	 * is a menu, listbox or dialog or holds navigation, and a trigger whose
+	 * panel isn't there. Hidden elements whose class names an accordion or
+	 * FAQ are shown as well.
+	 *
+	 * @param DOMDocument $doc Document.
+	 * @return void
+	 */
+	private function open_disclosures( DOMDocument $doc ) {
+		$xpath = new DOMXPath( $doc );
+		foreach ( iterator_to_array( $xpath->query( '//*[@aria-controls and @aria-expanded]' ) ) as $trigger ) {
+			if ( $trigger->hasAttribute( 'aria-haspopup' ) || preg_match( self::NOT_A_DISCLOSURE_ROLE, $trigger->getAttribute( 'role' ) ) ) {
+				continue;
+			}
+			$panels = array();
+			foreach ( preg_split( '/\s+/', trim( $trigger->getAttribute( 'aria-controls' ) ) ) as $id ) {
+				$found = '' === $id ? null : $xpath->query( '//*[@id=' . self::xpath_literal( $id ) . ']' );
+				if ( $found && $found->length ) {
+					$panels[] = $found->item( 0 );
+				}
+			}
+			if ( ! $panels ) {
+				continue;
+			}
+			foreach ( $panels as $panel ) {
+				if ( preg_match( self::NOT_A_PANEL_ROLE, $panel->getAttribute( 'role' ) ) || 'nav' === strtolower( $panel->nodeName )
+					|| $xpath->query( './/nav | .//*[@role="navigation"] | .//*[@role="menu"]', $panel )->length ) {
+					continue 2;
+				}
+			}
+			foreach ( $panels as $panel ) {
+				$this->show( $panel );
+			}
+			$this->keep_trigger_text( $doc, $trigger );
+		}
+		foreach ( iterator_to_array( $xpath->query( '//*[@class]' ) ) as $el ) {
+			if ( preg_match( self::DISCLOSURE_CLASS_RE, $el->getAttribute( 'class' ) ) ) {
+				$this->show( $el );
+			}
+		}
+	}
+
+	/**
+	 * Undo hidden, aria-hidden and an inline display:none / visibility:hidden.
+	 *
+	 * @param DOMElement $el Element.
+	 * @return void
+	 */
+	private function show( DOMElement $el ) {
+		$el->removeAttribute( 'hidden' );
+		if ( 'true' === strtolower( $el->getAttribute( 'aria-hidden' ) ) ) {
+			$el->removeAttribute( 'aria-hidden' );
+		}
+		if ( $el->hasAttribute( 'style' ) ) {
+			$el->setAttribute( 'style', (string) preg_replace( '/(?:display\s*:\s*none|visibility\s*:\s*hidden)\s*;?/i', '', $el->getAttribute( 'style' ) ) );
+		}
+	}
+
+	/**
+	 * A question <button> becomes its text: plain inside a heading, summary
+	 * or dt, else a bold paragraph. Expander labels are left to be dropped.
+	 *
+	 * @param DOMDocument $doc     Document.
+	 * @param DOMElement  $trigger Trigger.
+	 * @return void
+	 */
+	private function keep_trigger_text( DOMDocument $doc, DOMElement $trigger ) {
+		if ( 'button' !== strtolower( $trigger->nodeName ) || ! $trigger->parentNode ) {
+			return;
+		}
+		$xpath = new DOMXPath( $doc );
+		foreach ( iterator_to_array( $xpath->query( './/svg | .//*[@aria-hidden="true"]', $trigger ) ) as $icon ) {
+			$icon->parentNode->removeChild( $icon );
+		}
+		$text = $this->one_line( $trigger->textContent );
+		if ( '' === $text || preg_match( self::CONTROL_LABEL_RE, $text ) ) {
+			return;
+		}
+		for ( $up = $trigger->parentNode; $up && XML_ELEMENT_NODE === $up->nodeType; $up = $up->parentNode ) {
+			if ( preg_match( '/^(h[1-6]|summary|dt)$/', strtolower( $up->nodeName ) ) ) {
+				$trigger->parentNode->replaceChild( $doc->createTextNode( $text ), $trigger );
+				return;
+			}
+		}
+		$p      = $doc->createElement( 'p' );
+		$strong = $doc->createElement( 'strong' );
+		$strong->appendChild( $doc->createTextNode( $text ) );
+		$p->appendChild( $strong );
+		$trigger->parentNode->replaceChild( $p, $trigger );
+	}
+
+	/**
+	 * Loading placeholders: busy regions, progress bars, skeleton / shimmer /
+	 * spinner / loader elements (small, no media), and blocks whose whole text
+	 * is a loading message ("Loading reviews…"). Text that only starts with
+	 * "Loading" stays: the message needs an ellipsis, or to be the bare word.
+	 *
+	 * @param DOMDocument $doc Document.
+	 * @return void
+	 */
+	private function drop_placeholders( DOMDocument $doc ) {
+		$xpath  = new DOMXPath( $doc );
+		$small  = function ( DOMElement $el ) use ( $xpath ) {
+			$text = $this->one_line( $el->textContent );
+			$len  = function_exists( 'mb_strlen' ) ? mb_strlen( $text, 'UTF-8' ) : strlen( $text );
+			return $len < self::PLACEHOLDER_MAX_TEXT && 0 === $xpath->query( './/img | .//picture | .//video | .//table', $el )->length;
+		};
+		$skip   = function ( DOMElement $el ) {
+			return in_array( strtolower( $el->nodeName ), array( 'html', 'body', 'main', 'img', 'picture', 'video', 'source', 'td', 'th' ), true );
+		};
+		$remove = function ( DOMNode $el ) {
+			if ( $el->parentNode ) {
+				$el->parentNode->removeChild( $el );
+			}
+		};
+		foreach ( iterator_to_array( $xpath->query( '//*[@aria-busy="true"] | //*[@role="progressbar"]' ) ) as $el ) {
+			if ( ! $skip( $el ) && $small( $el ) ) {
+				$remove( $el );
+			}
+		}
+		foreach ( iterator_to_array( $xpath->query( '//*[@id or @class]' ) ) as $el ) {
+			if ( ! $skip( $el ) && preg_match( self::PLACEHOLDER_RE, $el->getAttribute( 'id' ) . ' ' . $el->getAttribute( 'class' ) ) && $small( $el ) ) {
+				$remove( $el );
+			}
+		}
+		foreach ( iterator_to_array( $xpath->query( '//p | //div | //span | //li | //section | //small | //em | //strong' ) ) as $el ) {
+			if ( $el->parentNode && preg_match( self::LOADING_TEXT_RE, $this->one_line( $el->textContent ) ) ) {
+				$remove( $el );
+			}
+		}
+	}
+
+	/**
+	 * A string as an XPath literal (XPath 1.0 has no escapes).
+	 *
+	 * @param string $s Value.
+	 * @return string
+	 */
+	private static function xpath_literal( $s ) {
+		if ( false === strpos( $s, "'" ) ) {
+			return "'" . $s . "'";
+		}
+		if ( false === strpos( $s, '"' ) ) {
+			return '"' . $s . '"';
+		}
+		return "concat('" . str_replace( "'", "', \"'\", '", $s ) . "')";
+	}
+
 	/**
 	 * Hidden by attribute or inline style.
 	 *
@@ -248,9 +426,19 @@ class Relve_AR_Converter {
 				}
 				$text = $this->one_line( $this->inline_children( $el ) );
 				return '' === $text ? array() : array( str_repeat( '#', $level ) . ' ' . $text );
+			case 'summary':
+				// A heading inside keeps its level; a text-only summary is a
+				// question or a label: bold.
+				$inner = $this->blocks( $el );
+				foreach ( $inner as $b ) {
+					if ( preg_match( '/^#{1,6} /', $b ) ) {
+						return $inner;
+					}
+				}
+				$text = $this->one_line( implode( ' ', $inner ) );
+				return '' === $text ? array() : array( '**' . $text . '**' );
 			case 'p':
 			case 'dt':
-			case 'summary':
 			case 'caption':
 				$p = $this->paragraph( $this->inline_children( $el ) );
 				if ( 'dt' === $tag && '' !== $p ) {

package/dist/wp/relve-agent-ready/includes/class-relve-ar-markdown-builder.php

@@ -26,7 +26,8 @@
  * The front-matter rules are the package's (core/enrich.ts): empty values
  * dropped, URLs / ISO dates / language tags plain, everything else
  * JSON-quoted. A document that already has front matter (a refined file
- * usually does not, but a hand-made one may) is left exactly as it is.
+ * usually does, a hand-made one may) keeps its block; only a title or
+ * description it lacks is added.
  *
  * @package Relve_Agent_Ready
  */
@@ -107,7 +108,8 @@ class Relve_AR_Markdown_Builder {
 
 	/**
 	 * Final document: front matter, then the body. A body that already has
-	 * front matter is returned as it is (plus a final newline).
+	 * front matter keeps it, with a missing title or description added from
+	 * $fields (plus a final newline).
 	 *
 	 * @param array  $fields Front-matter fields.
 	 * @param string $body   Markdown body (starting with its H1).
@@ -116,11 +118,59 @@ class Relve_AR_Markdown_Builder {
 	public static function assemble( $fields, $body ) {
 		$body = trim( (string) $body );
 		if ( self::has_front_matter( $body . "\n" ) ) {
-			return $body . "\n";
+			return self::merge_front_matter( $body, $fields ) . "\n";
 		}
 		return self::front_matter( $fields ) . "\n\n" . $body . "\n";
 	}
 
+	/**
+	 * Add a title or description the document's front matter lacks (title
+	 * first, description after the title). Keys it has, and every other
+	 * line, are kept as they are. No front matter: returned unchanged.
+	 *
+	 * @param string $markdown Markdown starting with front matter.
+	 * @param array  $fields   Field values.
+	 * @return string
+	 */
+	public static function merge_front_matter( $markdown, $fields ) {
+		if ( ! preg_match( '/^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/', (string) $markdown, $m ) ) {
+			return (string) $markdown;
+		}
+		$lines = preg_split( '/\r?\n/', $m[1] );
+		$keys  = array();
+		foreach ( $lines as $line ) {
+			if ( preg_match( '/^([A-Za-z_][\w-]*):/', $line, $k ) ) {
+				$keys[ $k[1] ] = true;
+			}
+		}
+		$changed = false;
+		foreach ( array( 'title', 'description' ) as $key ) {
+			$value = isset( $fields[ $key ] ) && is_scalar( $fields[ $key ] ) ? self::one_line( (string) $fields[ $key ] ) : '';
+			if ( isset( $keys[ $key ] ) || '' === $value ) {
+				continue;
+			}
+			$at = 0;
+			if ( 'description' === $key ) {
+				foreach ( $lines as $i => $line ) {
+					if ( 0 === strpos( $line, 'title:' ) ) {
+						$at = $i + 1;
+						while ( $at < count( $lines ) && preg_match( '/^[ \t]/', $lines[ $at ] ) ) {
+							++$at;
+						}
+						break;
+					}
+				}
+			}
+			array_splice( $lines, $at, 0, array( $key . ': ' . self::yaml_scalar( $value ) ) );
+			$keys[ $key ] = true;
+			$changed      = true;
+		}
+		if ( ! $changed ) {
+			return (string) $markdown;
+		}
+		return "---\n" . implode( "\n", $lines ) . "\n---" . ( '' !== $m[2] ? "\n" : '' ) . substr( (string) $markdown, strlen( $m[0] ) );
+	}
+
 	/**
 	 * Body of a single post or page: one H1, optional image, content.
 	 *
@@ -341,7 +391,10 @@ class Relve_AR_Markdown_Builder {
 	}
 
 	/**
-	 * Front matter for a stored (refined) file that has none of its own.
+	 * Front matter for a stored (refined) file: a whole block when it has
+	 * none, else its own with a missing title or description added. Files
+	 * written before descriptions were added get one this way, without a
+	 * rewrite.
 	 *
 	 * @param string       $markdown  Stored Markdown.
 	 * @param string       $canonical Canonical URL.
@@ -355,28 +408,26 @@ class Relve_AR_Markdown_Builder {
 		}
 		return self::assemble(
 			array(
-				'title'     => $title,
-				'canonical' => $canonical,
-				'source'    => $canonical,
-				'generator' => 'agent-ready/stored',
-				'lang'      => get_bloginfo( 'language' ),
+				'title'       => $title,
+				'description' => $post ? self::description_for( $post ) : '',
+				'canonical'   => $canonical,
+				'source'      => $canonical,
+				'generator'   => 'agent-ready/stored',
+				'lang'        => get_bloginfo( 'language' ),
 			),
 			$markdown
 		);
 	}
 
 	/**
-	 * Description: Yoast meta description, else the excerpt, else the first
-	 * words of the content.
+	 * Description: the SEO plugin's meta description, else the excerpt, else
+	 * the first words of the content.
 	 *
 	 * @param WP_Post $post Post.
 	 * @return string
 	 */
 	public static function description_for( $post ) {
-		$desc = (string) get_post_meta( $post->ID, '_yoast_wpseo_metadesc', true );
-		if ( '' !== $desc && false !== strpos( $desc, '%%' ) ) {
-			$desc = function_exists( 'wpseo_replace_vars' ) ? (string) wpseo_replace_vars( $desc, $post ) : '';
-		}
+		$desc = self::seo_description( $post );
 		if ( '' === trim( $desc ) && '' !== trim( (string) $post->post_excerpt ) ) {
 			$desc = $post->post_excerpt;
 		}
@@ -386,6 +437,60 @@ class Relve_AR_Markdown_Builder {
 		return self::shorten( html_entity_decode( wp_strip_all_tags( $desc ), ENT_QUOTES, 'UTF-8' ), 300 );
 	}
 
+	/**
+	 * The meta description an SEO plugin stores for the post: Yoast, then
+	 * Rank Math, SEOPress and All in One SEO. Template variables are resolved
+	 * with the plugin's own function when it is loaded; a description still
+	 * holding them is skipped. "" when none has one.
+	 *
+	 * @param WP_Post $post Post.
+	 * @return string
+	 */
+	public static function seo_description( $post ) {
+		// Yoast: %%vars%%.
+		$desc = (string) get_post_meta( $post->ID, '_yoast_wpseo_metadesc', true );
+		if ( '' !== $desc && false !== strpos( $desc, '%%' ) ) {
+			$desc = function_exists( 'wpseo_replace_vars' ) ? (string) wpseo_replace_vars( $desc, $post ) : '';
+		}
+		if ( '' !== trim( $desc ) ) {
+			return $desc;
+		}
+
+		// Rank Math: %vars%.
+		$desc = (string) get_post_meta( $post->ID, 'rank_math_description', true );
+		if ( '' !== $desc && preg_match( '/%[a-z_]+%/i', $desc ) ) {
+			$replace = array( 'RankMath\Helper', 'replace_vars' );
+			$desc    = is_callable( $replace ) ? (string) call_user_func( $replace, $desc, $post ) : '';
+		}
+		if ( '' !== trim( $desc ) && ! preg_match( '/%[a-z_]+%/i', $desc ) ) {
+			return $desc;
+		}
+
+		// SEOPress: %%vars%%, resolved only at render time; skipped when present.
+		$desc = (string) get_post_meta( $post->ID, '_seopress_titles_desc', true );
+		if ( '' !== trim( $desc ) && false === strpos( $desc, '%%' ) ) {
+			return $desc;
+		}
+
+		// All in One SEO: #tags, resolved by its own API.
+		$desc = '';
+		if ( function_exists( 'aioseo' ) ) {
+			try {
+				$aioseo = aioseo();
+				if ( isset( $aioseo->meta->description ) && is_callable( array( $aioseo->meta->description, 'getDescription' ) ) ) {
+					$desc = (string) $aioseo->meta->description->getDescription( $post );
+				}
+			} catch ( Throwable $e ) {
+				$desc = '';
+			}
+		}
+		if ( '' !== trim( $desc ) ) {
+			return $desc;
+		}
+		$desc = (string) get_post_meta( $post->ID, '_aioseo_description', true );
+		return preg_match( '/#[a-z_]+/i', $desc ) ? '' : $desc;
+	}
+
 	/**
 	 * Markdown for the posts front page / posts page or a category, tag or
 	 * author archive: an H1 and its first 50 posts.

package/dist/wp/relve-agent-ready/readme.txt

@@ -22,7 +22,7 @@ An agent can ask in three ways:
 
 What it serves:
 
-* Posts and pages (published, public, not password-protected, of the post types you enable): YAML front matter (title, canonical, author, published, modified, categories, tags, description, source), one `# Title` heading, the featured image and the content converted to Markdown (headings, lists, links, images, tables, code blocks).
+* Posts and pages (published, public, not password-protected, of the post types you enable): YAML front matter (title, canonical, author, published, modified, categories, tags, description, source), one `# Title` heading, the featured image and the content converted to Markdown (headings, lists, links, images, tables, code blocks). Collapsed accordion and FAQ answers are included, with their questions; loading placeholders such as "Loading…" are left out.
 * The posts front page and category, tag and author archives: a heading and a list of their latest 50 posts.
 * `llms.txt` under your home path: your site name, tagline and links to the `.md` version of up to 200 recent posts and pages.
 * Rewritten files: with your own OpenAI key, each post is rewritten into clean Markdown with Relve's prompt (see below) and served first.
@@ -37,7 +37,7 @@ The plugin is installed switched off. Nothing changes until you tick "Enabled" i
 The Markdown built from WordPress is faithful but plain. For agent-ready pages that read well (a clear summary, clean headings, tables kept, theme clutter gone), save your OpenAI API key in Settings -> Agent Ready:
 
 * When a post of an enabled type is published or updated, WP-Cron sends it about 30 seconds later (never while you wait on the editor) to Relve's refine service at `https://relvehq.com/api/agent-ready/v1/refine`: the permalink, and a small HTML page with the title, description, canonical link, structured data and the rendered content.
-* Relve rewrites it with its own prompt, using your key for that single OpenAI call, checks the result and sends back the Markdown. The plugin stores it as that post's file in `wp-content/uploads/relve-agent-ready/md/`; it is served first (`x-agent-ready-source: stored`).
+* Relve rewrites it with its own prompt, using your key for that single OpenAI call, checks the result and sends back the Markdown. The post's own description and its FAQ answers (from its structured data) are kept word for word. The plugin stores it as that post's file in `wp-content/uploads/relve-agent-ready/md/`; it is served first (`x-agent-ready-source: stored`), with the post's description in its front matter.
 * A post that has not changed since its last rewrite is answered "unchanged" without a model call, so it costs nothing.
 * "Rewrite all now" queues every published post of the enabled types; WP-Cron works through them 5 at a time. The settings page shows progress, posts rewritten, the last run and the last error code.
 * If OpenAI rejects the key or the account is out of quota, rewriting pauses and an admin notice says so. Saving a new key resumes it.
@@ -69,9 +69,9 @@ The `Accept` header, paths ending in `.md`, the `format` query parameter and `/l
 
 The plugin sets `DONOTCACHEPAGE` on every Markdown response and tells WP Rocket never to cache `.md` URLs or llms.txt. A page cache that already holds the HTML of a page may still answer an `Accept: text/markdown` request with that HTML. That is safe (the agent gets the normal page); the `.md` URL always returns Markdown.
 
-= I use Yoast SEO. =
+= I use Yoast SEO, Rank Math, SEOPress or All in One SEO. =
 
-The description in the front matter uses your Yoast meta description when there is one. If Yoast's own llms.txt feature is on, the "auto" mode leaves llms.txt to Yoast; choose "On" to serve this plugin's version instead.
+The description in the front matter uses your SEO plugin's meta description when there is one (Yoast first, then Rank Math, SEOPress and All in One SEO), else the excerpt, else the first words of the post. If Yoast's own llms.txt feature is on, the "auto" mode leaves llms.txt to Yoast; choose "On" to serve this plugin's version instead.
 
 = Does Relve see my OpenAI key? =
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@relvehq/agent-ready",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "license": "MIT",
   "description": "Make your site agent-ready: serve every page as clean Markdown to AI agents (Accept: text/markdown, .md URLs, llms.txt), refine it with AI ahead of time, and prove it with a free checker and a visits dashboard. Next.js adapter included.",
   "keywords": [