@mintlify/scraping 4.0.1117 → 4.0.1118

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1117",
+  "version": "4.0.1118",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1245",
+    "@mintlify/common": "1.0.1246",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "2e4488638a4ebfbec9edce575ab84a0df7d8d329"
+  "gitHead": "00242cc7f875194e9a9140c29c0fe8a00e5cd318"
 }

package/src/sdk/converters/sphinx.ts

@@ -1,3 +1,4 @@
+import { hastToText } from '@mintlify/common';
 import fse from 'fs-extra';
 import type { Element, ElementContent, Root as HastRoot } from 'hast';
 import path from 'node:path';
@@ -196,7 +197,7 @@ function explodePyDl(dl: Element, depth: number): ElementContent[] {
     if (child.tagName === 'dt' && classList(child).includes('sig')) {
       const name = findDescName(child);
       if (name) out.push(heading(Math.min(3 + depth, 6), callable ? `${name}()` : name));
-      out.push(pythonCodeBlock(textOf(child).replace(/\s+/g, ' ').trim()));
+      out.push(pythonCodeBlock(hastToText(child).replace(/\s+/g, ' ').trim()));
     } else if (child.tagName === 'dd') {
       transformPyDls(child, depth + 1);
       out.push(...child.children);
@@ -209,7 +210,7 @@ function findDescName(node: Element): string | undefined {
   let name: string | undefined;
   visit(node, 'element', (inner: Element) => {
     if (!classList(inner).includes('descname')) return;
-    name = textOf(inner).trim();
+    name = hastToText(inner).trim();
     return SKIP;
   });
   return name;
@@ -238,7 +239,7 @@ function extractDescription(hast: HastRoot): string | undefined {
   visit(hast, 'element', (node: Element, _index, parent) => {
     if (description || node.tagName !== 'p') return;
     if (parent && parent.type === 'element' && parent.tagName === 'li') return SKIP;
-    const text = textOf(node).replace(/\s+/g, ' ').trim();
+    const text = hastToText(node).replace(/\s+/g, ' ').trim();
     if (!text || text.startsWith('Bases:')) return;
     const sentence = text.split(/(?<=\.)\s/)[0] ?? text;
     description = sentence.length > 160 ? `${sentence.slice(0, 159)}…` : sentence;
@@ -246,12 +247,6 @@ function extractDescription(hast: HastRoot): string | undefined {
   return description;
 }
 
-function textOf(node: ElementContent): string {
-  if (node.type === 'text') return node.value;
-  if ('children' in node) return node.children.map(textOf).join('');
-  return '';
-}
-
 function classList(node: Element): string[] {
   const value = node.properties.className;
   return Array.isArray(value)