@aiquants/html-to-markdown 0.1.3 → 0.1.4

package/dist/{core-0q2nkcWH.js → core-DOzMLPij.js}

@@ -20384,6 +20384,123 @@ const rehypeAbsoluteLinks = (options) => {
     });
   };
 };
+const blockLevelTags = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "br",
+  "details",
+  "dialog",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  // "h1",  // # 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h2",  // ## 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h3",  // ### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h4",  // #### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h5",  // ##### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h6",  // ###### 見出し<span id="custom-id"></span> となってしまうのを回避
+  "header",
+  "hgroup",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "ul"
+]);
+function hasBlockElement(node2) {
+  let found = false;
+  visit(node2, "element", (el) => {
+    if (blockLevelTags.has(el.tagName)) {
+      found = true;
+      return EXIT;
+    }
+  });
+  return found;
+}
+const slugger = new BananaSlug();
+const rehypeNamedAnchors = () => (tree) => {
+  visit(tree, "element", (node2, index2, parent) => {
+    if (parent && typeof index2 === "number" && node2.properties && node2.properties.id) {
+      const id = String(node2.properties.id);
+      if (id && ["h1", "h2", "h3", "h4", "h5", "h6"].includes(node2.tagName)) {
+        const customId = slugger.slug(id);
+        node2.children.push({
+          type: "text",
+          value: ` {#${customId}}`
+        });
+        return;
+      }
+      if (node2.tagName === "span" && id && Object.keys(node2.properties || {}).length === 1) {
+        return;
+      }
+      const anchorNode = {
+        type: "element",
+        tagName: "span",
+        properties: { id },
+        children: []
+      };
+      if (hasBlockElement(node2)) {
+        let firstInlineInfo = null;
+        visit(node2, (child, childIndex, childParent) => {
+          if (childParent && typeof childIndex === "number") {
+            if (child.type === "element" && !blockLevelTags.has(child.tagName) || child.type === "text" && child.value.trim() !== "") {
+              firstInlineInfo = {
+                node: node2,
+                parent: childParent,
+                index: childIndex
+              };
+              return EXIT;
+            }
+          }
+        });
+        if (firstInlineInfo) {
+          const inline = firstInlineInfo;
+          inline.parent.children.splice(inline.index + 1, 0, anchorNode);
+          delete node2.properties.id;
+        }
+        return;
+      } else {
+        parent.children.splice(index2 + 1, 0, anchorNode);
+        delete node2.properties.id;
+        return index2 + 1;
+      }
+    }
+  });
+};
+const rehypeParagraphWrapper = () => (tree) => {
+  visit(tree, "element", (node2, index2, parent) => {
+    if (parent && typeof index2 === "number") {
+      const nextNode = parent.children[index2 + 1];
+      if (nextNode && nextNode.type === "element" && nextNode.tagName === "span" && nextNode.properties?.id && Object.keys(nextNode.properties).length === 1) {
+        const parentIsBlock = !parent || parent.type === "root" || ["body", "div", "section", "article", "aside", "footer", "header", "nav", "td", "th", "li"].includes(parent.tagName);
+        if (parentIsBlock) {
+          const paragraph2 = {
+            type: "element",
+            tagName: "p",
+            properties: {},
+            children: [node2, nextNode]
+            // 現在のノードと次の span ノードを子として含める
+          };
+          parent.children.splice(index2, 2, paragraph2);
+          return index2 + 1;
+        }
+      }
+    }
+  });
+};
 const rehypeSanitizeHtml = () => {
   return (tree) => {
     visit(tree, (node2, index2, parent) => {
@@ -20477,7 +20594,7 @@ const rehypeWikipediaFootnotes = () => {
   };
 };
 const handleTableCell = (_state, node2, _parent) => {
-  const allowedTags = ["br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "em", "b", "i", "u", "s", "strike", "del", "ins", "code", "ul", "ol", "li", "a", "img", "blockquote", "pre", "table", "thead", "tbody", "tfoot", "tr", "th", "td"];
+  const allowedTags = ["br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "em", "b", "i", "u", "s", "strike", "del", "ins", "code", "ul", "ol", "li", "a", "img", "blockquote", "pre", "table", "thead", "tbody", "tfoot", "tr", "th", "td", "style"];
   const allowedAttributes = ["style", "class", "id", "colspan", "rowspan", "width", "height", "align", "href", "src", "title", "alt"];
   visit(node2, "element", (elementNode, index2, parent) => {
     if (!parent || typeof index2 !== "number") {
@@ -20535,22 +20652,37 @@ const handleTableCell = (_state, node2, _parent) => {
 };
 const convertHtmlToMarkdown = async (html2, baseUrl, locale) => {
   console.log(getMessage(locale, "convert_start"));
-  const slugger = new BananaSlug();
-  const file = await unified().use(rehypeParse, { fragment: true }).use(rehypeRaw).use(rehypeSanitizeHtml).use(rehypeSlug).use(rehypeAbsoluteLinks, { baseUrl }).use(rehypeWikipediaFootnotes).use(rehypeRemark, {
+  const slugger2 = new BananaSlug();
+  const file = await unified().use(rehypeParse, { fragment: true }).use(rehypeRaw).use(rehypeSanitizeHtml).use(rehypeWikipediaFootnotes).use(rehypeNamedAnchors).use(rehypeSlug).use(rehypeParagraphWrapper).use(rehypeAbsoluteLinks, { baseUrl }).use(rehypeRemark, {
     handlers: {
+      span(state, node2) {
+        const id = node2.properties?.id;
+        if (id && Object.keys(node2.properties || {}).length === 1) {
+          const children = state.all(node2);
+          if (children.length === 0) {
+            return { type: "html", value: `<span id="${String(id)}"></span>` };
+          }
+          return [
+            { type: "html", value: `<span id="${String(id)}">` },
+            ...children,
+            { type: "html", value: "</span>" }
+          ];
+        }
+        return state.all(node2);
+      },
       a(state, node2) {
         const hasComplexChildren = node2.children.some((child) => child.type === "element" && ["dl", "ul", "ol", "table", "div"].includes(child.tagName));
         if (hasComplexChildren && node2.properties?.href) {
           const linkText2 = String(node2.properties.ariaLabel || "");
           if (linkText2) {
-            const url2 = String(node2.properties.href);
-            const linkNode = {
+            const url = String(node2.properties.href);
+            const linkNode2 = {
               type: "link",
-              url: url2,
+              url,
               children: [{ type: "text", value: linkText2 }]
             };
             const otherContent = state.all(node2);
-            return [linkNode, { type: "text", value: "\n" }, ...otherContent];
+            return [linkNode2, { type: "text", value: "\n" }, ...otherContent];
           }
         }
         const extractTextFromHast = (n) => {
@@ -20584,24 +20716,27 @@ const convertHtmlToMarkdown = async (html2, baseUrl, locale) => {
             return false;
           });
         };
+        let linkNode;
         if (containsImage(node2)) {
-          return {
+          linkNode = {
             type: "link",
             url: String(node2.properties?.href || ""),
             title: node2.properties?.title ? String(node2.properties.title) : null,
             children: state.all(node2)
           };
+        } else {
+          let url = String(node2.properties?.href || "");
+          if (url.startsWith("#")) {
+            url = `#${slugger2.slug(url.substring(1))}`;
+          }
+          linkNode = {
+            type: "link",
+            url,
+            title: node2.properties?.title ? String(node2.properties.title) : null,
+            children: [{ type: "text", value: linkText }]
+          };
         }
-        let url = String(node2.properties?.href || "");
-        if (url.startsWith("#")) {
-          url = `#${slugger.slug(url.substring(1))}`;
-        }
-        return {
-          type: "link",
-          url,
-          title: node2.properties?.title ? String(node2.properties.title) : null,
-          children: [{ type: "text", value: linkText }]
-        };
+        return linkNode;
       },
       td(state, node2, parent) {
         return handleTableCell(state, node2);
@@ -20632,13 +20767,13 @@ const htmlToMarkdown = async (url, options = {}) => {
     const htmlContent = await getHtmlWithPlaywright(url, locale);
     if (!htmlContent) {
       console.error(`No HTML content found for URL: ${url}`);
-      return "";
+      return { html: "", markdown: "" };
     }
     const markdown = await convertHtmlToMarkdown(htmlContent, url, locale);
-    return markdown;
+    return { html: htmlContent, markdown };
   } catch (error) {
     console.error(`An error occurred during the conversion process for URL: ${url}`, error);
-    return "";
+    return { html: "", markdown: "" };
   }
 };
 export {

package/dist/index.cjs

@@ -1 +1 @@
-const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e}),Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./core-DUdcm-c8.cjs");exports.getMessage=o.getMessage,exports.htmlToMarkdown=o.htmlToMarkdown;
+const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e}),Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./core-CVMYjtlI.cjs");exports.getMessage=o.getMessage,exports.htmlToMarkdown=o.htmlToMarkdown;

package/dist/index.js

@@ -5,7 +5,7 @@ Object.assign(global, {
   window,
   self: window
 });
-import { g, h } from "./core-0q2nkcWH.js";
+import { g, h } from "./core-DOzMLPij.js";
 export {
   g as getMessage,
   h as htmlToMarkdown

package/dist/main.cjs

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-Object.create,Object.defineProperty,Object.getOwnPropertyDescriptor,Object.getOwnPropertyNames,Object.getPrototypeOf,Object.prototype.hasOwnProperty;const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e});const t=require("./core-DUdcm-c8.cjs");var r="undefined"!=typeof document?document.currentScript:null;(async()=>{(("undefined"==typeof document?require("url").pathToFileURL(__filename).href:r&&"SCRIPT"===r.tagName.toUpperCase()&&r.src||new URL("main.cjs",document.baseURI).href).endsWith(process.argv[1])||process.argv[1]&&process.argv[1].includes("vite-node"))&&(async()=>{const e=(await import("yargs-parser")).default,r=await import("node:fs"),o=await import("node:path"),c=e(process.argv.slice(2),{string:["locale","output"],alias:{output:["o"]},default:{locale:"en-US"}}),s=c._[0],n=c.locale,a=c.output;s||process.exit(1);try{new URL(s)}catch(i){process.exit(1)}try{const e=await t.htmlToMarkdown(s,{locale:n});let c;if(a){c=o.resolve(process.cwd(),a);const e=o.dirname(c);r.existsSync(e)||r.mkdirSync(e,{recursive:!0})}else{const e=`${s.replace(/https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"_")}_${Date.now()}.md`,t=o.join(process.cwd(),".outputs","raw");r.existsSync(t)||r.mkdirSync(t,{recursive:!0}),c=o.join(t,e)}r.writeFileSync(c,e)}catch(i){process.exit(1)}})()})();
+Object.create,Object.defineProperty,Object.getOwnPropertyDescriptor,Object.getOwnPropertyNames,Object.getPrototypeOf,Object.prototype.hasOwnProperty;const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e});const t=require("./core-CVMYjtlI.cjs");var r="undefined"!=typeof document?document.currentScript:null;(async()=>{(("undefined"==typeof document?require("url").pathToFileURL(__filename).href:r&&"SCRIPT"===r.tagName.toUpperCase()&&r.src||new URL("main.cjs",document.baseURI).href).endsWith(process.argv[1])||process.argv[1]&&process.argv[1].includes("vite-node"))&&(async()=>{const e=(await import("yargs-parser")).default,r=await import("node:fs"),o=await import("node:path"),c=e(process.argv.slice(2),{string:["locale","output"],alias:{output:["o"]},default:{locale:"en-US"}}),s=c._[0],n=c.locale,a=c.output;s||process.exit(1);try{new URL(s)}catch(i){process.exit(1)}try{const{markdown:e,html:c}=await t.htmlToMarkdown(s,{locale:n});let i;if(a){i=o.resolve(process.cwd(),a);const e=o.dirname(i);r.existsSync(e)||r.mkdirSync(e,{recursive:!0})}else{const e=`${s.replace(/https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"_")}_${Date.now()}.md`,t=o.join(process.cwd(),".outputs","raw");r.existsSync(t)||r.mkdirSync(t,{recursive:!0}),i=o.join(t,e)}r.writeFileSync(i,e);const p=o.parse(i),l=o.join(p.dir,`${p.name}.html`);r.writeFileSync(l,c)}catch(i){process.exit(1)}})()})();

package/dist/main.js

@@ -6,7 +6,7 @@ Object.assign(global, {
   window,
   self: window
 });
-import { g as getMessage, h as htmlToMarkdown } from "./core-0q2nkcWH.js";
+import { g as getMessage, h as htmlToMarkdown } from "./core-DOzMLPij.js";
 const runCli = async () => {
   const yargsParser = (await import("yargs-parser")).default;
   const fs = await import("node:fs");
@@ -44,7 +44,7 @@ ${getMessage(locale, "cli_start")}`);
     if (outputFile) {
       console.info(getMessage(locale, "cli_output", { path: outputFile }));
     }
-    const markdown = await htmlToMarkdown(targetUrl, { locale });
+    const { markdown, html } = await htmlToMarkdown(targetUrl, { locale });
     let filePath;
     if (outputFile) {
       filePath = path.resolve(process.cwd(), outputFile);
@@ -63,9 +63,13 @@ ${getMessage(locale, "cli_start")}`);
       filePath = path.join(outputDir, fileName);
     }
     fs.writeFileSync(filePath, markdown);
+    const parsedPath = path.parse(filePath);
+    const htmlFilePath = path.join(parsedPath.dir, `${parsedPath.name}.html`);
+    fs.writeFileSync(htmlFilePath, html);
     console.info(`
 ${getMessage(locale, "success_save")}`);
     console.info(getMessage(locale, "success_path", { path: filePath }));
+    console.info(getMessage(locale, "success_path", { path: htmlFilePath }));
   } catch (error) {
     console.error(`
 ${getMessage(locale, "error_app")}`, error);

package/dist/src/core.d.ts

@@ -5,6 +5,9 @@ import { HtmlToMarkdownOptions } from './types.js';
  *
  * @param url The URL of the web page to convert. / 変換対象のWebページのURL。
  * @param options Options for the conversion process. / 変換プロセスのオプション。
- * @returns A promise that resolves to the Markdown string. / Markdown文字列を解決するPromise。
+ * @returns A promise that resolves to an object containing the HTML and Markdown strings. / HTMLとMarkdown文字列を含むオブジェクトを解決するPromise。
  */
-export declare const htmlToMarkdown: (url: string, options?: HtmlToMarkdownOptions) => Promise<string>;
+export declare const htmlToMarkdown: (url: string, options?: HtmlToMarkdownOptions) => Promise<{
+    html: string;
+    markdown: string;
+}>;

package/dist/src/plugins/index.d.ts

@@ -3,5 +3,7 @@
  * Rehype プラグインのインデックス
  */
 export { rehypeAbsoluteLinks } from './rehype-absolute-links.js';
+export { rehypeNamedAnchors } from './rehype-named-anchors.js';
+export { rehypeParagraphWrapper } from './rehype-paragraph-wrapper.js';
 export { rehypeSanitizeHtml } from './rehype-sanitize-html.js';
 export { rehypeWikipediaFootnotes } from './rehype-wikipedia-footnotes.js';

package/dist/src/plugins/rehype-named-anchors.d.ts

@@ -0,0 +1,2 @@
+import { Root } from 'hast';
+export declare const rehypeNamedAnchors: () => (tree: Root) => void;

package/dist/src/plugins/rehype-paragraph-wrapper.d.ts

@@ -0,0 +1,6 @@
+import { Root } from 'hast';
+/**
+ * A rehype plugin to wrap an element and its adjacent anchor span with a paragraph.
+ * 要素とそれに隣接するアンカースパンを段落でラップする rehype プラグイン。
+ */
+export declare const rehypeParagraphWrapper: () => (tree: Root) => void;

package/dist/tests/rehype-named-anchors.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiquants/html-to-markdown",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "HTML to Markdown converter",
   "type": "module",
   "main": "./dist/index.js",
@@ -20,12 +20,15 @@
     "LICENSE"
   ],
   "scripts": {
+    "dev": "NODE_OPTIONS='--enable-source-maps' vite-node --inspect=9229 --watch main.ts --",
+    "debug": "NODE_OPTIONS='--enable-source-maps' vite-node --inspect-brk=9229 main.ts --",
     "start": "vite-node main.ts",
     "build": "vite build",
     "lint": "biome lint app/",
     "format": "biome format --write",
     "check": "biome check --fix src/",
     "test": "vitest run",
+    "test1": "pnpm -F @aiquants/html-to-markdown test --test-name-pattern 'rehypeNamedAnchors plugin'",
     "coverage": "vitest run --coverage"
   },
   "keywords": [