@aiquants/html-to-markdown 0.1.2 → 0.1.4

package/dist/{core-D07ieSmY.js → core-DOzMLPij.js}

@@ -9,11 +9,12 @@ import { chromium } from "playwright";
 const messages = {
   "ja-JP": {
     error_url_required: "エラー: 変換対象のURLを引数で指定してください。",
-    usage: "使用法: node dist/main.js <URL> [--locale <ロケール>]",
-    example: "例: node dist/main.js https://ja.wikipedia.org/wiki/Node.js --locale ja-JP",
+    usage: "使用法: node dist/main.js <URL> [--output <ファイルパス>] [--locale <ロケール>]",
+    example: "例: node dist/main.js https://ja.wikipedia.org/wiki/Node.js -o output.md --locale ja-JP",
     cli_start: "🚀 コマンドラインから処理を開始します...",
     cli_url: "   - URL: {url}",
     cli_locale: "   - Locale: {locale}",
+    cli_output: "   - Output: {path}",
     success_save: "✅ Markdownファイルが正常に保存されました。",
     success_path: "   -> {path}",
     error_app: "❌ アプリケーションの実行中にエラーが発生しました。",
@@ -28,11 +29,12 @@ const messages = {
   },
   "en-US": {
     error_url_required: "Error: Please specify the URL to convert as an argument.",
-    usage: "Usage: node dist/main.js <URL> [--locale <locale>]",
-    example: "Example: node dist/main.js https://en.wikipedia.org/wiki/Node.js --locale en-US",
+    usage: "Usage: node dist/main.js <URL> [--output <file path>] [--locale <locale>]",
+    example: "Example: node dist/main.js https://en.wikipedia.org/wiki/Node.js -o output.md --locale en-US",
     cli_start: "🚀 Starting process from command line...",
     cli_url: "   - URL: {url}",
     cli_locale: "   - Locale: {locale}",
+    cli_output: "   - Output: {path}",
     success_save: "✅ Markdown file saved successfully.",
     success_path: "   -> {path}",
     error_app: "❌ An error occurred during application execution.",
@@ -20382,6 +20384,123 @@ const rehypeAbsoluteLinks = (options) => {
     });
   };
 };
+const blockLevelTags = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "br",
+  "details",
+  "dialog",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  // "h1",  // # 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h2",  // ## 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h3",  // ### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h4",  // #### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h5",  // ##### 見出し<span id="custom-id"></span> となってしまうのを回避
+  // "h6",  // ###### 見出し<span id="custom-id"></span> となってしまうのを回避
+  "header",
+  "hgroup",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "ul"
+]);
+function hasBlockElement(node2) {
+  let found = false;
+  visit(node2, "element", (el) => {
+    if (blockLevelTags.has(el.tagName)) {
+      found = true;
+      return EXIT;
+    }
+  });
+  return found;
+}
+const slugger = new BananaSlug();
+const rehypeNamedAnchors = () => (tree) => {
+  visit(tree, "element", (node2, index2, parent) => {
+    if (parent && typeof index2 === "number" && node2.properties && node2.properties.id) {
+      const id = String(node2.properties.id);
+      if (id && ["h1", "h2", "h3", "h4", "h5", "h6"].includes(node2.tagName)) {
+        const customId = slugger.slug(id);
+        node2.children.push({
+          type: "text",
+          value: ` {#${customId}}`
+        });
+        return;
+      }
+      if (node2.tagName === "span" && id && Object.keys(node2.properties || {}).length === 1) {
+        return;
+      }
+      const anchorNode = {
+        type: "element",
+        tagName: "span",
+        properties: { id },
+        children: []
+      };
+      if (hasBlockElement(node2)) {
+        let firstInlineInfo = null;
+        visit(node2, (child, childIndex, childParent) => {
+          if (childParent && typeof childIndex === "number") {
+            if (child.type === "element" && !blockLevelTags.has(child.tagName) || child.type === "text" && child.value.trim() !== "") {
+              firstInlineInfo = {
+                node: node2,
+                parent: childParent,
+                index: childIndex
+              };
+              return EXIT;
+            }
+          }
+        });
+        if (firstInlineInfo) {
+          const inline = firstInlineInfo;
+          inline.parent.children.splice(inline.index + 1, 0, anchorNode);
+          delete node2.properties.id;
+        }
+        return;
+      } else {
+        parent.children.splice(index2 + 1, 0, anchorNode);
+        delete node2.properties.id;
+        return index2 + 1;
+      }
+    }
+  });
+};
+const rehypeParagraphWrapper = () => (tree) => {
+  visit(tree, "element", (node2, index2, parent) => {
+    if (parent && typeof index2 === "number") {
+      const nextNode = parent.children[index2 + 1];
+      if (nextNode && nextNode.type === "element" && nextNode.tagName === "span" && nextNode.properties?.id && Object.keys(nextNode.properties).length === 1) {
+        const parentIsBlock = !parent || parent.type === "root" || ["body", "div", "section", "article", "aside", "footer", "header", "nav", "td", "th", "li"].includes(parent.tagName);
+        if (parentIsBlock) {
+          const paragraph2 = {
+            type: "element",
+            tagName: "p",
+            properties: {},
+            children: [node2, nextNode]
+            // 現在のノードと次の span ノードを子として含める
+          };
+          parent.children.splice(index2, 2, paragraph2);
+          return index2 + 1;
+        }
+      }
+    }
+  });
+};
 const rehypeSanitizeHtml = () => {
   return (tree) => {
     visit(tree, (node2, index2, parent) => {
@@ -20475,7 +20594,7 @@ const rehypeWikipediaFootnotes = () => {
   };
 };
 const handleTableCell = (_state, node2, _parent) => {
-  const allowedTags = ["br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "em", "b", "i", "u", "s", "strike", "del", "ins", "code", "ul", "ol", "li", "a", "img", "blockquote", "pre", "table", "thead", "tbody", "tfoot", "tr", "th", "td"];
+  const allowedTags = ["br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "em", "b", "i", "u", "s", "strike", "del", "ins", "code", "ul", "ol", "li", "a", "img", "blockquote", "pre", "table", "thead", "tbody", "tfoot", "tr", "th", "td", "style"];
   const allowedAttributes = ["style", "class", "id", "colspan", "rowspan", "width", "height", "align", "href", "src", "title", "alt"];
   visit(node2, "element", (elementNode, index2, parent) => {
     if (!parent || typeof index2 !== "number") {
@@ -20533,22 +20652,37 @@ const handleTableCell = (_state, node2, _parent) => {
 };
 const convertHtmlToMarkdown = async (html2, baseUrl, locale) => {
   console.log(getMessage(locale, "convert_start"));
-  const slugger = new BananaSlug();
-  const file = await unified().use(rehypeParse, { fragment: true }).use(rehypeRaw).use(rehypeSanitizeHtml).use(rehypeSlug).use(rehypeAbsoluteLinks, { baseUrl }).use(rehypeWikipediaFootnotes).use(rehypeRemark, {
+  const slugger2 = new BananaSlug();
+  const file = await unified().use(rehypeParse, { fragment: true }).use(rehypeRaw).use(rehypeSanitizeHtml).use(rehypeWikipediaFootnotes).use(rehypeNamedAnchors).use(rehypeSlug).use(rehypeParagraphWrapper).use(rehypeAbsoluteLinks, { baseUrl }).use(rehypeRemark, {
     handlers: {
+      span(state, node2) {
+        const id = node2.properties?.id;
+        if (id && Object.keys(node2.properties || {}).length === 1) {
+          const children = state.all(node2);
+          if (children.length === 0) {
+            return { type: "html", value: `<span id="${String(id)}"></span>` };
+          }
+          return [
+            { type: "html", value: `<span id="${String(id)}">` },
+            ...children,
+            { type: "html", value: "</span>" }
+          ];
+        }
+        return state.all(node2);
+      },
       a(state, node2) {
         const hasComplexChildren = node2.children.some((child) => child.type === "element" && ["dl", "ul", "ol", "table", "div"].includes(child.tagName));
         if (hasComplexChildren && node2.properties?.href) {
           const linkText2 = String(node2.properties.ariaLabel || "");
           if (linkText2) {
-            const url2 = String(node2.properties.href);
-            const linkNode = {
+            const url = String(node2.properties.href);
+            const linkNode2 = {
               type: "link",
-              url: url2,
+              url,
               children: [{ type: "text", value: linkText2 }]
             };
             const otherContent = state.all(node2);
-            return [linkNode, { type: "text", value: "\n" }, ...otherContent];
+            return [linkNode2, { type: "text", value: "\n" }, ...otherContent];
           }
         }
         const extractTextFromHast = (n) => {
@@ -20582,24 +20716,27 @@ const convertHtmlToMarkdown = async (html2, baseUrl, locale) => {
             return false;
           });
         };
+        let linkNode;
         if (containsImage(node2)) {
-          return {
+          linkNode = {
             type: "link",
             url: String(node2.properties?.href || ""),
             title: node2.properties?.title ? String(node2.properties.title) : null,
             children: state.all(node2)
           };
+        } else {
+          let url = String(node2.properties?.href || "");
+          if (url.startsWith("#")) {
+            url = `#${slugger2.slug(url.substring(1))}`;
+          }
+          linkNode = {
+            type: "link",
+            url,
+            title: node2.properties?.title ? String(node2.properties.title) : null,
+            children: [{ type: "text", value: linkText }]
+          };
         }
-        let url = String(node2.properties?.href || "");
-        if (url.startsWith("#")) {
-          url = `#${slugger.slug(url.substring(1))}`;
-        }
-        return {
-          type: "link",
-          url,
-          title: node2.properties?.title ? String(node2.properties.title) : null,
-          children: [{ type: "text", value: linkText }]
-        };
+        return linkNode;
       },
       td(state, node2, parent) {
         return handleTableCell(state, node2);
@@ -20630,13 +20767,13 @@ const htmlToMarkdown = async (url, options = {}) => {
     const htmlContent = await getHtmlWithPlaywright(url, locale);
     if (!htmlContent) {
       console.error(`No HTML content found for URL: ${url}`);
-      return "";
+      return { html: "", markdown: "" };
     }
     const markdown = await convertHtmlToMarkdown(htmlContent, url, locale);
-    return markdown;
+    return { html: htmlContent, markdown };
   } catch (error) {
     console.error(`An error occurred during the conversion process for URL: ${url}`, error);
-    return "";
+    return { html: "", markdown: "" };
   }
 };
 export {

package/dist/index.cjs

@@ -1 +1 @@
-const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e}),Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./core-C3IjbMzg.cjs");exports.getMessage=o.getMessage,exports.htmlToMarkdown=o.htmlToMarkdown;
+const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e}),Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./core-CVMYjtlI.cjs");exports.getMessage=o.getMessage,exports.htmlToMarkdown=o.htmlToMarkdown;

package/dist/index.js

@@ -5,7 +5,7 @@ Object.assign(global, {
   window,
   self: window
 });
-import { g, h } from "./core-D07ieSmY.js";
+import { g, h } from "./core-DOzMLPij.js";
 export {
   g as getMessage,
   h as htmlToMarkdown

package/dist/main.cjs

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-Object.create,Object.defineProperty,Object.getOwnPropertyDescriptor,Object.getOwnPropertyNames,Object.getPrototypeOf,Object.prototype.hasOwnProperty;const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e});const t=require("./core-C3IjbMzg.cjs");var r="undefined"!=typeof document?document.currentScript:null;(async()=>{(("undefined"==typeof document?require("url").pathToFileURL(__filename).href:r&&"SCRIPT"===r.tagName.toUpperCase()&&r.src||new URL("main.cjs",document.baseURI).href).endsWith(process.argv[1])||process.argv[1]&&process.argv[1].includes("vite-node"))&&(async()=>{const e=(await import("yargs-parser")).default,r=await import("node:fs"),c=await import("node:path"),o=e(process.argv.slice(2),{string:["locale"],default:{locale:"en-US"}}),s=o._[0],a=o.locale;s||process.exit(1);try{new URL(s)}catch(n){process.exit(1)}try{const e=await t.htmlToMarkdown(s,{locale:a}),o=`${s.replace(/https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"_")}_${Date.now()}.md`,n=c.join(process.cwd(),".outputs","raw");r.existsSync(n)||r.mkdirSync(n,{recursive:!0});const i=c.join(n,o);r.writeFileSync(i,e)}catch(n){process.exit(1)}})()})();
+Object.create,Object.defineProperty,Object.getOwnPropertyDescriptor,Object.getOwnPropertyNames,Object.getPrototypeOf,Object.prototype.hasOwnProperty;const e=new(require("happy-dom").Window);Object.assign(global,{document:e.document,window:e,self:e});const t=require("./core-CVMYjtlI.cjs");var r="undefined"!=typeof document?document.currentScript:null;(async()=>{(("undefined"==typeof document?require("url").pathToFileURL(__filename).href:r&&"SCRIPT"===r.tagName.toUpperCase()&&r.src||new URL("main.cjs",document.baseURI).href).endsWith(process.argv[1])||process.argv[1]&&process.argv[1].includes("vite-node"))&&(async()=>{const e=(await import("yargs-parser")).default,r=await import("node:fs"),o=await import("node:path"),c=e(process.argv.slice(2),{string:["locale","output"],alias:{output:["o"]},default:{locale:"en-US"}}),s=c._[0],n=c.locale,a=c.output;s||process.exit(1);try{new URL(s)}catch(i){process.exit(1)}try{const{markdown:e,html:c}=await t.htmlToMarkdown(s,{locale:n});let i;if(a){i=o.resolve(process.cwd(),a);const e=o.dirname(i);r.existsSync(e)||r.mkdirSync(e,{recursive:!0})}else{const e=`${s.replace(/https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"_")}_${Date.now()}.md`,t=o.join(process.cwd(),".outputs","raw");r.existsSync(t)||r.mkdirSync(t,{recursive:!0}),i=o.join(t,e)}r.writeFileSync(i,e);const p=o.parse(i),l=o.join(p.dir,`${p.name}.html`);r.writeFileSync(l,c)}catch(i){process.exit(1)}})()})();

package/dist/main.js

@@ -6,13 +6,16 @@ Object.assign(global, {
   window,
   self: window
 });
-import { g as getMessage, h as htmlToMarkdown } from "./core-D07ieSmY.js";
+import { g as getMessage, h as htmlToMarkdown } from "./core-DOzMLPij.js";
 const runCli = async () => {
   const yargsParser = (await import("yargs-parser")).default;
   const fs = await import("node:fs");
   const path = await import("node:path");
   const argv = yargsParser(process.argv.slice(2), {
-    string: ["locale"],
+    string: ["locale", "output"],
+    alias: {
+      output: ["o"]
+    },
     default: {
       locale: "en-US"
       // デフォルトのロケール
@@ -20,6 +23,7 @@ const runCli = async () => {
   });
   const targetUrl = argv._[0];
   const locale = argv.locale;
+  const outputFile = argv.output;
   if (!targetUrl) {
     console.error(getMessage(locale, "error_url_required"));
     console.info(getMessage(locale, "usage"));
@@ -37,19 +41,35 @@ const runCli = async () => {
 ${getMessage(locale, "cli_start")}`);
     console.info(getMessage(locale, "cli_url", { url: targetUrl }));
     console.info(getMessage(locale, "cli_locale", { locale }));
-    const markdown = await htmlToMarkdown(targetUrl, { locale });
-    const safeFileName = targetUrl.replace(/https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "_");
-    const timestamp = Date.now();
-    const fileName = `${safeFileName}_${timestamp}.md`;
-    const outputDir = path.join(process.cwd(), ".outputs", "raw");
-    if (!fs.existsSync(outputDir)) {
-      fs.mkdirSync(outputDir, { recursive: true });
+    if (outputFile) {
+      console.info(getMessage(locale, "cli_output", { path: outputFile }));
+    }
+    const { markdown, html } = await htmlToMarkdown(targetUrl, { locale });
+    let filePath;
+    if (outputFile) {
+      filePath = path.resolve(process.cwd(), outputFile);
+      const outputDir = path.dirname(filePath);
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+    } else {
+      const safeFileName = targetUrl.replace(/https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "_");
+      const timestamp = Date.now();
+      const fileName = `${safeFileName}_${timestamp}.md`;
+      const outputDir = path.join(process.cwd(), ".outputs", "raw");
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+      filePath = path.join(outputDir, fileName);
     }
-    const filePath = path.join(outputDir, fileName);
     fs.writeFileSync(filePath, markdown);
+    const parsedPath = path.parse(filePath);
+    const htmlFilePath = path.join(parsedPath.dir, `${parsedPath.name}.html`);
+    fs.writeFileSync(htmlFilePath, html);
     console.info(`
 ${getMessage(locale, "success_save")}`);
     console.info(getMessage(locale, "success_path", { path: filePath }));
+    console.info(getMessage(locale, "success_path", { path: htmlFilePath }));
   } catch (error) {
     console.error(`
 ${getMessage(locale, "error_app")}`, error);

package/dist/src/core.d.ts

@@ -5,6 +5,9 @@ import { HtmlToMarkdownOptions } from './types.js';
  *
  * @param url The URL of the web page to convert. / 変換対象のWebページのURL。
  * @param options Options for the conversion process. / 変換プロセスのオプション。
- * @returns A promise that resolves to the Markdown string. / Markdown文字列を解決するPromise。
+ * @returns A promise that resolves to an object containing the HTML and Markdown strings. / HTMLとMarkdown文字列を含むオブジェクトを解決するPromise。
  */
-export declare const htmlToMarkdown: (url: string, options?: HtmlToMarkdownOptions) => Promise<string>;
+export declare const htmlToMarkdown: (url: string, options?: HtmlToMarkdownOptions) => Promise<{
+    html: string;
+    markdown: string;
+}>;

package/dist/src/plugins/index.d.ts

@@ -3,5 +3,7 @@
  * Rehype プラグインのインデックス
  */
 export { rehypeAbsoluteLinks } from './rehype-absolute-links.js';
+export { rehypeNamedAnchors } from './rehype-named-anchors.js';
+export { rehypeParagraphWrapper } from './rehype-paragraph-wrapper.js';
 export { rehypeSanitizeHtml } from './rehype-sanitize-html.js';
 export { rehypeWikipediaFootnotes } from './rehype-wikipedia-footnotes.js';

package/dist/src/plugins/rehype-named-anchors.d.ts

@@ -0,0 +1,2 @@
+import { Root } from 'hast';
+export declare const rehypeNamedAnchors: () => (tree: Root) => void;

package/dist/src/plugins/rehype-paragraph-wrapper.d.ts

@@ -0,0 +1,6 @@
+import { Root } from 'hast';
+/**
+ * A rehype plugin to wrap an element and its adjacent anchor span with a paragraph.
+ * 要素とそれに隣接するアンカースパンを段落でラップする rehype プラグイン。
+ */
+export declare const rehypeParagraphWrapper: () => (tree: Root) => void;

package/dist/src/types.d.ts

@@ -19,7 +19,7 @@ export interface HtmlToMarkdownOptions {
  * Message keys for i18n.
  * i18n 用のメッセージキー
  */
-export type MessageKey = "error_url_required" | "usage" | "example" | "cli_start" | "cli_url" | "cli_locale" | "success_save" | "success_path" | "error_app" | "playwright_launch" | "playwright_goto" | "playwright_get_content" | "playwright_close" | "playwright_error" | "convert_start" | "convert_success" | "url_parse_error";
+export type MessageKey = "error_url_required" | "usage" | "example" | "cli_start" | "cli_url" | "cli_locale" | "cli_output" | "success_save" | "success_path" | "error_app" | "playwright_launch" | "playwright_goto" | "playwright_get_content" | "playwright_close" | "playwright_error" | "convert_start" | "convert_success" | "url_parse_error";
 /**
  * Supported locales.
  * サポートされているロケール

package/dist/tests/rehype-named-anchors.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiquants/html-to-markdown",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "HTML to Markdown converter",
   "type": "module",
   "main": "./dist/index.js",
@@ -20,12 +20,15 @@
     "LICENSE"
   ],
   "scripts": {
+    "dev": "NODE_OPTIONS='--enable-source-maps' vite-node --inspect=9229 --watch main.ts --",
+    "debug": "NODE_OPTIONS='--enable-source-maps' vite-node --inspect-brk=9229 main.ts --",
     "start": "vite-node main.ts",
     "build": "vite build",
     "lint": "biome lint app/",
     "format": "biome format --write",
     "check": "biome check --fix src/",
     "test": "vitest run",
+    "test1": "pnpm -F @aiquants/html-to-markdown test --test-name-pattern 'rehypeNamedAnchors plugin'",
     "coverage": "vitest run --coverage"
   },
   "keywords": [