crawlforge-extractors 1.2.2 → 1.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, and structural fingerprinting. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/templates.js

@@ -220,6 +220,18 @@ function githubLicense(license) {
   return (spdx && spdx !== 'NOASSERTION' ? spdx : license.name) || null;
 }
 
+/**
+ * og:description is never just the About text. GitHub appends "Contribute to
+ * owner/repo development by creating an account on GitHub." to it, and that
+ * sentence is the *whole* value on a repo with no description at all. Strip
+ * it; whatever remains is the description, and nothing remaining means the
+ * repo has none.
+ */
+function githubOgDescription($) {
+  const og = attr($, 'meta[property="og:description"]', 'content') || '';
+  return tidy(og.replace(/\s*Contribute to \S+ development by creating an account on GitHub\.?/, ''));
+}
+
 /**
  * Repo tab counters stamp the exact number in title= ("5,102") and an
  * abbreviated text ("5.1k"); a counter with nothing to show stamps
@@ -391,7 +403,12 @@ export const TEMPLATES = [
       const about = githubSidebarAbout($);
       return {
         name: text($, 'strong[itemprop="name"] a') || text($, '.repository-content h1'),
-        description: attr($, 'meta[property="og:description"]', 'content') || text($, 'p.f4.my-3'),
+        // Like homepage: when the sidebar payload is present it is
+        // authoritative, so a repo that set no description reports null
+        // rather than GitHub's "Contribute to ..." boilerplate.
+        description: about
+          ? tidy(about.description)
+          : tidy(text($, 'p.f4.my-3')) || githubOgDescription($),
         stars: text($, '#repo-stars-counter-star') || text($, '[aria-label*="stargazers"]'),
         forks: text($, '#repo-network-counter') || text($, '[aria-label*="forks"]'),
         // React (logged-out) layout has no watchers aria-label; the count is