@mintlify/scraping 3.0.13 → 3.0.15

package/bin/util.d.ts

@@ -1,4 +1,4 @@
-import { Ora as OraType } from "ora";
+import { Ora as OraType } from 'ora';
 export declare const MintConfig: (name: string, color: string, ctaName: string, ctaUrl: string, filename: string) => {
     name: string;
     logo: string;

package/bin/util.js

@@ -1,13 +1,13 @@
-import { mkdirSync, writeFileSync } from "fs";
-import Ora from "ora";
-import path from "path";
-import shell from "shelljs";
-import stopIfInvalidLink from "./validation/stopIfInvalidLink.js";
+import { mkdirSync, writeFileSync } from 'fs';
+import Ora from 'ora';
+import path from 'path';
+import shell from 'shelljs';
+import stopIfInvalidLink from './validation/stopIfInvalidLink.js';
 export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
     return {
         name,
-        logo: "",
-        favicon: "",
+        logo: '',
+        favicon: '',
         colors: {
             primary: color,
         },
@@ -19,7 +19,7 @@ export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
         anchors: [],
         navigation: [
             {
-                group: "Home",
+                group: 'Home',
                 pages: [filename],
             },
         ],
@@ -38,9 +38,7 @@ export const Page = (title, description, markdown) => {
     if (!endsWithQuote) {
         title = title + '"';
     }
-    const optionalDescription = description
-        ? `\ndescription: "${description}"`
-        : "";
+    const optionalDescription = description ? `\ndescription: "${description}"` : '';
     return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
 };
 export function getOrigin(url) {
@@ -50,43 +48,43 @@ export function getOrigin(url) {
 }
 export function objToReadableString(objs) {
     // Two spaces as indentation
-    return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
+    return objs.map((obj) => JSON.stringify(obj, null, 2)).join(',\n');
 }
 export const toFilename = (title) => {
     // Gets rid of special characters at the start and end
     // of the name by converting to spaces then using trim.
     return title
-        .replace(/[^a-z0-9]/gi, " ")
+        .replace(/[^a-z0-9]/gi, ' ')
         .trim()
-        .replace(/ /g, "-")
+        .replace(/ /g, '-')
         .toLowerCase();
 };
 export const addMdx = (fileName) => {
-    if (fileName.endsWith(".mdx")) {
+    if (fileName.endsWith('.mdx')) {
         return fileName;
     }
-    return fileName + ".mdx";
+    return fileName + '.mdx';
 };
-export const createPage = (title, description, markdown, overwrite = false, rootDir = "", fileName) => {
+export const createPage = (title, description, markdown, overwrite = false, rootDir = '', fileName) => {
     const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
     // Create the folders needed if they're missing
     mkdirSync(rootDir, { recursive: true });
     // Write the page to memory
     if (overwrite) {
         writeFileSync(writePath, Page(title, description, markdown));
-        console.log("✏️ - " + writePath);
+        console.log('✏️ - ' + writePath);
     }
     else {
         try {
             writeFileSync(writePath, Page(title, description, markdown), {
-                flag: "wx",
+                flag: 'wx',
             });
-            console.log("✏️ - " + writePath);
+            console.log('✏️ - ' + writePath);
         }
         catch (e) {
             // We do a try-catch instead of an if-statement to avoid a race condition
             // of the file being created after we started writing.
-            if (e?.code === "EEXIST") {
+            if (e?.code === 'EEXIST') {
                 console.log(`❌ Skipping existing file ${writePath}`);
             }
             else {
@@ -100,23 +98,22 @@ export function getHrefFromArgs(argv) {
     stopIfInvalidLink(href);
     return href;
 }
-export const buildLogger = (startText = "") => {
+export const buildLogger = (startText = '') => {
     const logger = Ora().start(startText);
     return logger;
 };
 export const getFileExtension = (filename) => {
-    const ext = filename.substring(filename.lastIndexOf(".") + 1, filename.length);
+    const ext = filename.substring(filename.lastIndexOf('.') + 1, filename.length);
     if (filename === ext)
         return undefined;
     return ext.toLowerCase();
 };
 export const fileBelongsInPagesFolder = (filename) => {
     const extension = getFileExtension(filename);
-    return (extension &&
-        (extension === "mdx" || extension === "md" || extension === "tsx"));
+    return extension && (extension === 'mdx' || extension === 'md' || extension === 'tsx');
 };
 export const ensureYarn = (logger) => {
-    const yarnInstalled = shell.which("yarn");
+    const yarnInstalled = shell.which('yarn');
     if (!yarnInstalled) {
         logger.fail(`yarn must be installed, run
 

package/bin/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,EAAE;IACF,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW;QACrC,CAAC,CAAC,mBAAmB,WAAW,GAAG;QACnC,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAA2B;IAC7D,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAK,CAAuB,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC/C,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAC5B,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAC7B,QAAQ,CAAC,MAAM,CAChB,CAAC;IACF,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,CACL,SAAS;QACT,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CACnE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAe,EAAE,EAAE;IAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAA2B;IAC7D,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAK,CAAsB,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC9C,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAe,EAAE,EAAE;IAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC,CAAC"}

package/bin/validation/stopIfInvalidLink.js

@@ -1,8 +1,8 @@
-import isValidLink from "./isValidLink.js";
+import isValidLink from './isValidLink.js';
 export default function stopIfInvalidLink(href) {
     if (!isValidLink(href)) {
-        console.log("Invalid link: " + href);
-        console.log("Make sure the link starts with http:// or https://");
+        console.log('Invalid link: ' + href);
+        console.log('Make sure the link starts with http:// or https://');
         process.exit(1);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "3.0.13",
+  "version": "3.0.15",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -34,13 +34,16 @@
     "prepare": "npm run build",
     "build": "tsc",
     "watch": "tsc --watch",
-    "lint": "eslint . --cache"
+    "lint": "eslint . --cache",
+    "format": "prettier \"./src/**/*.ts\" --write",
+    "format:check": "prettier \"./src/**/*.ts\" --check"
   },
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
     "axios": "^1.2.2",
     "cheerio": "^0.22.0",
     "favicons": "^7.0.1",
+    "fs-extra": "^11.1.1",
     "node-html-markdown": "^1.3.0",
     "ora": "^6.1.2",
     "puppeteer": "^19.4.0",
@@ -50,7 +53,9 @@
   "devDependencies": {
     "@mintlify/eslint-config": "1.0.3",
     "@mintlify/eslint-config-typescript": "1.0.7",
+    "@mintlify/prettier-config": "1.0.1",
     "@mintlify/ts-config": "1.0.7",
+    "@trivago/prettier-plugin-sort-imports": "3.x",
     "@tsconfig/recommended": "1.x",
     "@types/cheerio": "^0.22.31",
     "@types/node": "^18.7.13",
@@ -61,8 +66,10 @@
     "eslint": "8.x",
     "eslint-config-prettier": "8.x",
     "eslint-plugin-unused-imports": "2.x",
+    "jest": "^29.3.1",
+    "openapi-types": "^12.1.3",
     "prettier": "2.x",
     "typescript": "^4.8.2"
   },
-  "gitHead": "87b89d302e0b8e464d9682316d3f9b838aeab6e6"
+  "gitHead": "cf3d6884a0cc69eece8129eacc2cc74b751a38e3"
 }

package/src/browser.ts

@@ -1,4 +1,4 @@
-import { launch } from "puppeteer";
+import { launch } from 'puppeteer';
 
 export async function startBrowser() {
   try {
@@ -7,7 +7,7 @@ export async function startBrowser() {
       ignoreHTTPSErrors: true,
     });
   } catch (err) {
-    console.log("Could not create a browser instance: ", err);
+    console.log('Could not create a browser instance: ', err);
     process.exit(1);
   }
 }
@@ -16,7 +16,7 @@ export async function getHtmlWithPuppeteer(href: string) {
   const browser = await startBrowser();
   const page = await browser.newPage();
   await page.goto(href, {
-    waitUntil: "networkidle2",
+    waitUntil: 'networkidle2',
   });
   const html = await page.content();
   browser.close();

package/src/constants.ts

@@ -1,37 +1,37 @@
-import path from "path";
-import * as url from "url";
-import os from "os";
+import os from 'os';
+import path from 'path';
+import * as url from 'url';
 
 // package installation location
-export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
+export const INSTALL_PATH = url.fileURLToPath(new URL('.', import.meta.url));
 
 export const HOME_DIR = os.homedir();
 
-export const DOT_MINTLIFY = path.join(HOME_DIR, ".mintlify");
+export const DOT_MINTLIFY = path.join(HOME_DIR, '.mintlify');
 
-export const VERSION_PATH = path.join(DOT_MINTLIFY, "mint", "mint-version.txt");
+export const VERSION_PATH = path.join(DOT_MINTLIFY, 'mint', 'mint-version.txt');
 
-export const CLIENT_PATH = path.join(DOT_MINTLIFY, "mint", "client");
+export const CLIENT_PATH = path.join(DOT_MINTLIFY, 'mint', 'client');
 
-export const MINT_PATH = path.join(DOT_MINTLIFY, "mint");
+export const MINT_PATH = path.join(DOT_MINTLIFY, 'mint');
 
 // command execution location
 export const CMD_EXEC_PATH = process.cwd();
 
 export const SUPPORTED_MEDIA_EXTENSIONS = [
-  "jpeg",
-  "jpg",
-  "jfif",
-  "pjpeg",
-  "pjp",
-  "png",
-  "svg",
-  "svgz",
-  "ico",
-  "webp",
-  "gif",
-  "apng",
-  "avif",
-  "bmp",
-  "mp4",
+  'jpeg',
+  'jpg',
+  'jfif',
+  'pjpeg',
+  'pjp',
+  'png',
+  'svg',
+  'svgz',
+  'ico',
+  'webp',
+  'gif',
+  'apng',
+  'avif',
+  'bmp',
+  'mp4',
 ];

package/src/downloadImage.ts

@@ -1,18 +1,15 @@
-import { existsSync, mkdirSync, createWriteStream } from "fs";
-import path from "path";
-import axios from "axios";
-import { getFileExtension } from "./util.js";
-import { SUPPORTED_MEDIA_EXTENSIONS } from "./constants.js";
+import axios from 'axios';
+import { existsSync, mkdirSync, createWriteStream } from 'fs';
+import path from 'path';
 
-async function writeImageToFile(
-  imageSrc: string,
-  writePath: string,
-  overwrite: boolean
-) {
+import { SUPPORTED_MEDIA_EXTENSIONS } from './constants.js';
+import { getFileExtension } from './util.js';
+
+async function writeImageToFile(imageSrc: string, writePath: string, overwrite: boolean) {
   // Avoid unnecessary downloads
   if (existsSync(writePath) && !overwrite) {
     return Promise.reject({
-      code: "EEXIST",
+      code: 'EEXIST',
     });
   }
 
@@ -23,21 +20,21 @@ async function writeImageToFile(
 
   try {
     const response = await axios.get(imageSrc, {
-      responseType: "stream",
+      responseType: 'stream',
     });
     // wx prevents overwriting an image with the exact same name
     // being created in the time we were downloading
     response.data.pipe(writer, {
-      flag: "wx",
+      flag: 'wx',
     });
 
     return new Promise((resolve, reject) => {
-      writer.on("finish", resolve);
-      writer.on("error", reject);
+      writer.on('finish', resolve);
+      writer.on('error', reject);
     });
   } catch (e) {
     return Promise.reject({
-      code: "ENOTFOUND",
+      code: 'ENOTFOUND',
     });
   }
 }
@@ -47,7 +44,7 @@ export function isValidImageSrc(src: string) {
     return false;
   }
   // We do not support downloading base64 in-line images.
-  if (src.startsWith("data:")) {
+  if (src.startsWith('data:')) {
     return false;
   }
 
@@ -55,7 +52,7 @@ export function isValidImageSrc(src: string) {
   const ext = getFileExtension(imageHref);
 
   if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
-    console.error("🚨 We do not support the file extension: " + ext);
+    console.error('🚨 We do not support the file extension: ' + ext);
     return false;
   }
 
@@ -64,7 +61,7 @@ export function isValidImageSrc(src: string) {
 
 export function removeMetadataFromImageSrc(src: string) {
   // Part of the URL standard
-  const metadataSymbols = ["?", "#"];
+  const metadataSymbols = ['?', '#'];
 
   metadataSymbols.forEach((dividerSymbol) => {
     // Some frameworks add metadata after the file extension, we need to remove that.
@@ -76,7 +73,7 @@ export function removeMetadataFromImageSrc(src: string) {
 
 export function cleanImageSrc(src: string, origin: string) {
   // Add origin if the image tags are using relative sources
-  return src.startsWith("http") ? src : new URL(src, origin).href;
+  return src.startsWith('http') ? src : new URL(src, origin).href;
 }
 
 export default async function downloadImage(
@@ -86,15 +83,13 @@ export default async function downloadImage(
 ) {
   await writeImageToFile(imageSrc, writePath, overwrite)
     .then(() => {
-      console.log("🖼️ - " + writePath);
+      console.log('🖼️ - ' + writePath);
     })
     .catch((e) => {
-      if (e.code === "EEXIST") {
+      if (e.code === 'EEXIST') {
         console.log(`❌ Skipping existing image ${writePath}`);
-      } else if (e.code === "ENOTFOUND") {
-        console.error(
-          `🚨 Cannot download the image, address not found ${imageSrc}`
-        );
+      } else if (e.code === 'ENOTFOUND') {
+        console.error(`🚨 Cannot download the image, address not found ${imageSrc}`);
       } else {
         console.error(e);
       }

package/src/scraping/detectFramework.ts

@@ -1,10 +1,10 @@
-import cheerio from "cheerio";
+import cheerio from 'cheerio';
 
 export enum Frameworks {
-  DOCUSAURUS = "DOCUSAURUS",
-  GITBOOK = "GITBOOK",
-  README = "README",
-  INTERCOM = "INTERCOM",
+  DOCUSAURUS = 'DOCUSAURUS',
+  GITBOOK = 'GITBOOK',
+  README = 'README',
+  INTERCOM = 'INTERCOM',
 }
 
 export function detectFramework(html) {
@@ -13,25 +13,25 @@ export function detectFramework(html) {
 
   if (
     docusaurusMeta.length > 0 &&
-    docusaurusMeta.attr("content") &&
-    typeof docusaurusMeta.attr("content") === "string" &&
-    (docusaurusMeta.attr("content") as string).includes("Docusaurus")
+    docusaurusMeta.attr('content') &&
+    typeof docusaurusMeta.attr('content') === 'string' &&
+    (docusaurusMeta.attr('content') as string).includes('Docusaurus')
   ) {
-    const metaAttrString = docusaurusMeta.attr("content") as string;
-    if (metaAttrString.includes("v3")) {
-      return { framework: Frameworks.DOCUSAURUS, version: "3" };
+    const metaAttrString = docusaurusMeta.attr('content') as string;
+    if (metaAttrString.includes('v3')) {
+      return { framework: Frameworks.DOCUSAURUS, version: '3' };
     }
-    if (metaAttrString.includes("v2")) {
-      return { framework: Frameworks.DOCUSAURUS, version: "2" };
-    } else if (metaAttrString.includes("v1")) {
+    if (metaAttrString.includes('v2')) {
+      return { framework: Frameworks.DOCUSAURUS, version: '2' };
+    } else if (metaAttrString.includes('v1')) {
       console.warn(
-        "WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3."
+        'WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.'
       );
-      return { framework: Frameworks.DOCUSAURUS, version: "1" };
+      return { framework: Frameworks.DOCUSAURUS, version: '1' };
     }
   }
 
-  const isGitBook = $(".gitbook-root").length > 0;
+  const isGitBook = $('.gitbook-root').length > 0;
   if (isGitBook) {
     return { framework: Frameworks.GITBOOK };
   }
@@ -46,5 +46,5 @@ export function detectFramework(html) {
     return { framework: Frameworks.INTERCOM };
   }
 
-  return { framework: undefined};
+  return { framework: undefined };
 }

package/src/scraping/downloadAllImages.ts

@@ -1,9 +1,10 @@
-import path from "path";
+import path from 'path';
+
 import downloadImage, {
   cleanImageSrc,
   isValidImageSrc,
   removeMetadataFromImageSrc,
-} from "../downloadImage.js";
+} from '../downloadImage.js';
 
 // To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadAllImages(
@@ -15,7 +16,7 @@ export default async function downloadAllImages(
   modifyFileName?: any
 ) {
   if (!baseDir) {
-    console.debug("Skipping image downloading");
+    console.debug('Skipping image downloading');
     return;
   }
 
@@ -24,8 +25,8 @@ export default async function downloadAllImages(
   const imageSrcs = [
     ...new Set<string>(
       content
-        .find("img[src]")
-        .map((i, image) => $(image).attr("src"))
+        .find('img[src]')
+        .map((i, image) => $(image).attr('src'))
         .toArray()
     ),
   ];
@@ -52,8 +53,5 @@ export default async function downloadAllImages(
     })
   );
 
-  return origToNewArray.reduce(
-    (result, current) => Object.assign(result, current),
-    {}
-  );
+  return origToNewArray.reduce((result, current) => Object.assign(result, current), {});
 }

package/src/scraping/downloadLogoImage.ts

@@ -1,10 +1,11 @@
-import path from "path";
+import path from 'path';
+
 import downloadImage, {
   cleanImageSrc,
   isValidImageSrc,
   removeMetadataFromImageSrc,
-} from "../downloadImage.js";
-import { getFileExtension } from "../util.js";
+} from '../downloadImage.js';
+import { getFileExtension } from '../util.js';
 
 // To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadLogoImage(
@@ -18,7 +19,7 @@ export default async function downloadLogoImage(
   const imageHref = cleanImageSrc(imageSrc, origin);
 
   const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
-  const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
+  const imagePath = path.join(imageBaseDir, 'logo', 'logo-light-mode.' + ext);
 
   await downloadImage(imageHref, imagePath, overwrite);
 }

package/src/scraping/getSitemapLinks.ts

@@ -1,10 +1,10 @@
-import axios from "axios";
+import axios from 'axios';
 
 // Not in use.
 // Gets all links in a sitemap.
 export const getSitemapLinks = async (url: URL) => {
-  const hostname = url.hostname.replace(".", "\\.");
-  const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
+  const hostname = url.hostname.replace('.', '\\.');
+  const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, 'gmi');
 
   try {
     const indexData = (await axios.get(url.href)).data as string;
@@ -12,7 +12,7 @@ export const getSitemapLinks = async (url: URL) => {
     return array || [];
   } catch (err) {
     console.error(err);
-    console.log("Skipping sitemap links because we encountered an error.");
+    console.log('Skipping sitemap links because we encountered an error.');
     return [];
   }
 };

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -1,7 +1,8 @@
-import path from "path";
-import axios from "axios";
-import { getHtmlWithPuppeteer } from "../browser.js";
-import { createPage } from "../util.js";
+import axios from 'axios';
+import path from 'path';
+
+import { getHtmlWithPuppeteer } from '../browser.js';
+import { createPage } from '../util.js';
 
 export async function scrapeFileGettingFileNameFromUrl(
   pathname: string,
@@ -25,22 +26,22 @@ export async function scrapeFileGettingFileNameFromUrl(
   baseToRemove?: string
 ): Promise<MintNavigationEntry> {
   // Skip scraping external links
-  if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
+  if (pathname.startsWith('https://') || pathname.startsWith('http://')) {
     return pathname;
   }
 
   // Removes file name from the end
-  const splitSubpath = pathname.split("/");
-  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
+  const splitSubpath = pathname.split('/');
+  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join('/');
 
   // Remove base dir if passed in
   if (baseToRemove && folders.startsWith(baseToRemove)) {
-    folders = folders.replace(baseToRemove, "");
+    folders = folders.replace(baseToRemove, '');
   }
 
   // TO DO: Improve this by putting each page's images in a separate
   // folder named after the title of the page.
-  const imageBaseDir = path.join(cliDir, "images", folders);
+  const imageBaseDir = path.join(cliDir, 'images', folders);
 
   // Scrape each page separately
   const href = new URL(pathname, origin).href;
@@ -69,17 +70,10 @@ export async function scrapeFileGettingFileNameFromUrl(
   const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
 
   // Default to introduction.mdx if we encountered index.html
-  const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
+  const fileName = splitSubpath[splitSubpath.length - 1] || 'introduction';
 
   // Will create subfolders as needed
-  createPage(
-    title ?? '',
-    description,
-    markdown,
-    overwrite,
-    newFileLocation,
-    fileName
-  );
+  createPage(title ?? '', description, markdown, overwrite, newFileLocation, fileName);
 
   // Removes first slash if we are in a folder, Mintlify doesn't need it
   return folders ? path.join(folders, fileName).substring(1) : fileName;

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,4 +1,4 @@
-import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
+import { scrapeFileGettingFileNameFromUrl } from './scrapeFileGettingFileNameFromUrl.js';
 
 export async function scrapeGettingFileNameFromUrl(
   navEntry: MintNavigationEntry,
@@ -21,10 +21,11 @@ export async function scrapeGettingFileNameFromUrl(
   version: string | undefined,
   baseToRemove?: string
 ): Promise<MintNavigationEntry> {
-  if (typeof navEntry !== "string") {
-    const newPages: (MintNavigationEntry)[] = [];
+  if (typeof navEntry !== 'string') {
+    const newPages: MintNavigationEntry[] = [];
     for (const nestedNavEntry of navEntry.pages) {
-      newPages.push(await scrapeGettingFileNameFromUrl(
+      newPages.push(
+        await scrapeGettingFileNameFromUrl(
           nestedNavEntry,
           cliDir,
           origin,
@@ -33,7 +34,8 @@ export async function scrapeGettingFileNameFromUrl(
           puppeteer,
           version,
           baseToRemove
-      ))
+        )
+      );
     }
     navEntry.pages = newPages;
     return navEntry;