@mintlify/scraping 3.0.187 → 3.0.188

package/src/cli.ts

@@ -2,106 +2,69 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-import { checkUrl, checkVersion } from './checks.js';
-import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
-import { FrameworkHint, frameworks } from './scraping/detectFramework.js';
-import { scrapePageAutomatically } from './scraping/scrapePageCommands.js';
-import { scrapeSectionAutomatically } from './scraping/scrapeSectionCommands.js';
+import { FINAL_SUCCESS_MESSAGE } from './constants.js';
+import { scrapePageGroup } from './scrapingPipeline/group.js';
+import { htmlToHast } from './scrapingPipeline/root.js';
+import { scrapeAllSiteTabs } from './scrapingPipeline/tabs.js';
+import { detectFramework, framework } from './utils/detectFramework.js';
+import { write } from './utils/file.js';
+import { log } from './utils/log.js';
+import { fetchPageHtml } from './utils/network.js';
+import { checkUrl } from './utils/url.js';
 
 await yargs(hideBin(process.argv))
   .command(
     'page <url>',
-    'Scrapes a page',
-    (yargs) =>
-      yargs
-        .positional('url', { type: 'string', demandOption: true })
-        .check(checkUrl)
-        .option('overwrite', { alias: 'O', type: 'boolean', default: false })
-        .option('tool', { alias: 't', choices: frameworks })
-        .option('docusaurusVersion', {
-          alias: 'd',
-          type: 'string',
-          choices: ['1', '2', '3'] as const,
-        })
-        .check(checkVersion),
-    async ({ url, overwrite, tool, docusaurusVersion }) => {
-      const frameworkHint: FrameworkHint = {
-        framework: tool,
-        version: docusaurusVersion ?? '3',
-      };
-
-      await scrapePageAutomatically(url, overwrite, frameworkHint);
-    }
+    'Scrapes the docs page for the URL provided',
+    (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl),
+    async ({ url }) => await page(url)
   )
-  .command(
-    'section <url>',
-    'Scrapes the docs in the section',
-    (yargs) =>
-      yargs
-        .positional('url', { type: 'string', demandOption: true })
-        .check(checkUrl)
-        .option('overwrite', { alias: 'O', type: 'boolean', default: false })
-        .option('tool', { alias: 't', choices: frameworks })
-        .option('docusaurusVersion', {
-          alias: 'd',
-          type: 'string',
-          choices: ['1', '2', '3'] as const,
-        })
-        .check(checkVersion),
-    async ({ url, overwrite, tool, docusaurusVersion }) => {
-      const frameworkHint: FrameworkHint = {
-        framework: tool,
-        version: docusaurusVersion ?? '3',
-      };
 
-      await scrapeSectionAutomatically(url, overwrite, frameworkHint);
-    }
-  )
   .command(
-    'openapi-file <openapiFilename>',
-    'Creates MDX files from an OpenAPI spec',
-    (yargs) =>
-      yargs
-        .positional('openapiFilename', {
-          describe: 'The filename of the OpenAPI spec',
-          type: 'string',
-          demandOption: true,
-        })
-        .option('writeFiles', {
-          describe: 'Whether or not to write the frontmatter files',
-          default: true,
-          type: 'boolean',
-          alias: 'w',
-        })
-        .option('outDir', {
-          describe: 'The folder in which to write any created frontmatter files',
-          type: 'string',
-          alias: 'o',
-        }),
-    async (argv) => {
-      try {
-        const { nav } = await generateOpenApiPages(
-          argv.openapiFilename,
-          argv.writeFiles,
-          argv.outDir
-        );
-        console.log('navigation object suggestion:');
-        console.log(JSON.stringify(nav, undefined, 2));
-      } catch (error) {
-        if (error instanceof Error) {
-          console.error(error.message);
-        } else {
-          console.error(error);
-        }
-      }
-    }
+    'section <url>',
+    'Scrapes the entire docs site based on the URL provided',
+    (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl),
+    async ({ url }) => await site(url)
   )
-  // Print the help menu when the user enters an invalid command.
+
   .strictCommands()
   .demandCommand(1, 'Unknown command. See above for the list of supported commands.')
-
-  // Alias option flags --help = -h, --version = -v
   .alias('h', 'help')
   .alias('v', 'version')
-
   .parse();
+
+async function page(url: string) {
+  const urlObj = new URL(url);
+  const html = await fetchPageHtml(urlObj);
+  log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
+
+  const hast = htmlToHast(html);
+  detectFramework(hast);
+
+  const needsBrowser = framework.vendor === 'gitbook';
+  const results = await scrapePageGroup([urlObj], needsBrowser);
+  const result = results[0] || {
+    success: false,
+    message: `An unknown error occurred when scraping ${url}`,
+  };
+
+  if (result.success) {
+    log(`Successfully scraped ${url} ${result.data ? `into ${result.data[1]}` : ''}`);
+  } else {
+    log(result.message);
+  }
+}
+
+async function site(url: string) {
+  const urlObj = new URL(url);
+  const html = await fetchPageHtml(urlObj);
+  log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
+
+  const result = await scrapeAllSiteTabs(html, urlObj);
+  if (result.success) {
+    write('mint.json', JSON.stringify(result.data, undefined, 2));
+    log(FINAL_SUCCESS_MESSAGE);
+  } else {
+    log(result.message);
+  }
+}

package/src/components/Accordion.ts

@@ -0,0 +1,84 @@
+import type { Element, ElementContent } from 'hast';
+
+import { assertIsDefined, assertIsNumber } from '../assert.js';
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+import { findTitle } from '../utils/title.js';
+
+export function gitBookScrapeAccordion(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'details') {
+    return undefined;
+  }
+
+  const title = findTitle(node, { delete: true, nodeType: 'element', tagName: 'summary' });
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Accordion',
+    properties: {
+      title: title,
+    },
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function readmeScrapeAccordion(
+  node: HastNode,
+  index: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (
+    node.tagName !== 'button' ||
+    !node.properties.className ||
+    !(node.properties.className as Array<string>).includes('accordion')
+  ) {
+    return undefined;
+  }
+
+  assertIsNumber(index);
+  assertIsDefined(parent);
+
+  const title = findTitle(node);
+
+  parent.children.shift();
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Accordion',
+    properties: {
+      title: title,
+    },
+    children: turnChildrenIntoMdx(parent.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function docusaurusScrapeAccordion(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'details') {
+    return undefined;
+  }
+
+  const title = findTitle(node, { delete: true, nodeType: 'element', tagName: 'summary' });
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Accordion',
+    properties: {
+      title: title,
+    },
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}

package/src/components/AccordionGroup.ts

@@ -0,0 +1,69 @@
+import type { Element } from 'hast';
+
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+
+export function gitBookScrapeAccordionGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Accordion') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let accordionCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  }
+
+  if (accordionCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'AccordionGroup';
+  }
+
+  return undefined;
+}
+
+export function readmeScrapeAccordionGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Accordion') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let accordionCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  }
+
+  if (accordionCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'AccordionGroup';
+  }
+
+  return undefined;
+}
+
+export function docusaurusScrapeAccordionGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Accordion') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let accordionCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  }
+
+  if (accordionCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'AccordionGroup';
+  }
+
+  return undefined;
+}

package/src/components/Callout.ts

@@ -0,0 +1,159 @@
+import type { Element, ElementContent } from 'hast';
+
+import { assertIsStringArray } from '../assert.js';
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+
+export function gitBookScrapeCallout(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    node.tagName !== 'div' ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    !node.properties.className
+      .join(' ')
+      .startsWith(
+        'px-4 py-4 transition-colors rounded-md straight-corners:rounded-none bg-gradient-to-b'
+      )
+  ) {
+    return undefined;
+  }
+
+  assertIsStringArray(node.properties.className);
+
+  const className =
+    node.properties.className
+      .find((className) => /^from-[a-z]+\/[0-9]$/.test(className))
+      ?.replace(/^from-|\/[0-9]$/g, '') || null;
+
+  let tagName = 'Note';
+  switch (className) {
+    case 'periwinkle':
+      tagName = 'Info';
+      break;
+    case 'teal':
+      tagName = 'Check';
+      break;
+    case 'yellow':
+    case 'pomegranate':
+      tagName = 'Warning';
+      break;
+    default:
+      tagName = 'Info';
+      break;
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: tagName,
+    properties: {},
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function readmeScrapeCallout(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    node.tagName !== 'blockquote' ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    !node.properties.className.includes('callout')
+  ) {
+    return undefined;
+  }
+
+  node.children.shift();
+
+  assertIsStringArray(node.properties.className);
+  const calloutClassNames = node.properties.className.filter((className) =>
+    className.includes('callout_')
+  );
+  const calloutClassName: string = calloutClassNames[0] ? calloutClassNames[0] : 'callout_info';
+
+  let tagName = 'Note';
+  switch (calloutClassName) {
+    case 'callout_default':
+    case 'callout_info':
+      tagName = 'Info';
+      break;
+    case 'callout_warn':
+    case 'callout_error':
+      tagName = 'Warning';
+      break;
+    case 'callout_okay':
+      tagName = 'Check';
+      break;
+    default:
+      tagName = 'Info';
+      break;
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: tagName,
+    properties: {},
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function docusaurusScrapeCallout(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    node.tagName !== 'div' ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    (!node.properties.className.includes('admonition') &&
+      !node.properties.className.includes('theme-admonition'))
+  ) {
+    return undefined;
+  }
+
+  node.children.shift();
+
+  const calloutClassNames = node.properties.className.filter(
+    (className) => typeof className === 'string' && className.includes('alert--')
+  );
+  const calloutClassName = calloutClassNames.length ? calloutClassNames[0] : 'alert--info';
+
+  let tagName = 'Note';
+  switch (calloutClassName) {
+    case 'alert--info':
+      tagName = 'Info';
+      break;
+    case 'alert--secondary':
+      tagName = 'Note';
+      break;
+    case 'alert--danger':
+    case 'alert--warning':
+      tagName = 'Warning';
+      break;
+    case 'alert--success':
+      tagName = 'Check';
+      break;
+    default:
+      tagName = 'Info';
+      break;
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: tagName,
+    properties: {},
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  return newNode;
+}

package/src/components/Card.ts

@@ -0,0 +1,168 @@
+import type { Element, ElementContent } from 'hast';
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
+
+import { assertIsDefined } from '../assert.js';
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+import { findImg } from '../utils/img.js';
+import { findTitle } from '../utils/title.js';
+
+export function gitBookScrapeCard(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName !== 'a' && node.tagName !== 'div') ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    !node.properties.className
+      .join(' ')
+      .startsWith(
+        'group grid shadow-1xs shadow-dark/[0.02] rounded-md straight-corners:rounded-none dark:shadow-transparent'
+      )
+  ) {
+    return undefined;
+  }
+
+  let firstTextElement: Element | undefined = undefined;
+  visit(node, 'element', function (subNode, index, parent) {
+    if (
+      !subNode.properties.className ||
+      !Array.isArray(subNode.properties.className) ||
+      subNode.properties.className.join(' ') !== 'w-full space-y-2 lg:space-y-3 leading-normal'
+    )
+      return CONTINUE;
+
+    firstTextElement = subNode;
+    if (parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+    return EXIT;
+  });
+
+  const title = findTitle(firstTextElement);
+  const imgSrc = findImg(node);
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Card',
+    properties: {
+      title: title,
+    },
+    children: turnChildrenIntoMdx(node.children) as Array<ElementContent>,
+  };
+
+  if (node.properties.href) newNode.properties.href = node.properties.href;
+  if (imgSrc) newNode.properties.img = imgSrc;
+
+  return newNode;
+}
+
+export function readmeScrapeCard(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName !== 'div' && node.tagName !== 'a') ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    (!node.properties.className.includes('Tile') &&
+      !node.properties.className.includes('card') &&
+      !node.properties.className.includes('Card') &&
+      !node.properties.className.includes('docs-card') &&
+      !node.properties.className.join(' ').includes('_card') &&
+      !node.properties.className.join(' ').includes('-card'))
+  ) {
+    return undefined;
+  }
+
+  const title = findTitle(node);
+
+  let href: string | undefined = undefined;
+  if (node.properties.href) {
+    href = node.properties.href as string;
+  } else if (node.properties.onclick && typeof node.properties.onclick === 'string') {
+    const str = node.properties.onclick.split("'")[1];
+    href = str ? `./${str}` : undefined;
+  } else {
+    visit(node, 'element', function (subNode) {
+      if (subNode.properties.href) {
+        href = subNode.properties.href as string;
+        return EXIT;
+      } else if (subNode.properties.onclick && typeof node.properties.onclick === 'string') {
+        const str = node.properties.onclick.split("'")[1];
+        href = str ? `./${str}` : undefined;
+        return EXIT;
+      }
+    });
+  }
+
+  assertIsDefined(parent);
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Card',
+    properties: {
+      title: title,
+      href: href,
+    },
+    children: turnChildrenIntoMdx(parent.children as Array<Element>) as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function docusaurusScrapeCard(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName !== 'div' && node.tagName !== 'a') ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    (!node.properties.className.includes('Tile') &&
+      !node.properties.className.includes('card') &&
+      !node.properties.className.includes('Card') &&
+      !node.properties.className.includes('docs-card') &&
+      !node.properties.className.join(' ').includes('_card') &&
+      !node.properties.className.join(' ').includes('-card'))
+  ) {
+    return undefined;
+  }
+
+  const title = findTitle(node);
+
+  let href: string | undefined = undefined;
+  if (node.properties.href) {
+    href = node.properties.href as string;
+  } else if (node.properties.onclick && typeof node.properties.onclick === 'string') {
+    const str = node.properties.onclick.split("'")[1];
+    href = str ? `./${str}` : undefined;
+  } else {
+    visit(node, 'element', function (subNode) {
+      if (subNode.properties.href) {
+        href = subNode.properties.href as string;
+        return EXIT;
+      } else if (subNode.properties.onclick && typeof node.properties.onclick === 'string') {
+        const str = node.properties.onclick.split("'")[1];
+        href = str ? `./${str}` : undefined;
+        return EXIT;
+      }
+    });
+  }
+
+  assertIsDefined(parent);
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Card',
+    properties: {
+      title: title,
+      href: href,
+    },
+    children: turnChildrenIntoMdx(parent.children as Array<Element>) as Array<ElementContent>,
+  };
+
+  return newNode;
+}