@research-copilot/plugin 1.1.16 → 1.1.18

package/dist/skills/llm-wiki/deps/baoyu-url-to-markdown/scripts/main.ts

@@ -0,0 +1,314 @@
+import { createInterface } from "node:readline";
+import { writeFile, mkdir, access } from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+
+import { CdpConnection, getFreePort, findExistingChromePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js";
+import { absolutizeUrlsScript, extractContent, createMarkdownDocument, type ConversionResult } from "./html-to-markdown.js";
+import { localizeMarkdownMedia, countRemoteMedia } from "./media-localizer.js";
+import { resolveUrlToMarkdownDataDir } from "./paths.js";
+import { DEFAULT_TIMEOUT_MS, CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS, POST_LOAD_DELAY_MS, SCROLL_STEP_WAIT_MS, SCROLL_MAX_STEPS } from "./constants.js";
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function fileExists(filePath: string): Promise<boolean> {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+interface Args {
+  url: string;
+  output?: string;
+  outputDir?: string;
+  wait: boolean;
+  timeout: number;
+  downloadMedia: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { url: "", wait: false, timeout: DEFAULT_TIMEOUT_MS, downloadMedia: false };
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--wait" || arg === "-w") {
+      args.wait = true;
+    } else if (arg === "-o" || arg === "--output") {
+      args.output = argv[++i];
+    } else if (arg === "--timeout" || arg === "-t") {
+      args.timeout = parseInt(argv[++i], 10) || DEFAULT_TIMEOUT_MS;
+    } else if (arg === "--output-dir") {
+      args.outputDir = argv[++i];
+    } else if (arg === "--download-media") {
+      args.downloadMedia = true;
+    } else if (!arg.startsWith("-") && !args.url) {
+      args.url = arg;
+    }
+  }
+  return args;
+}
+
+function generateSlug(title: string, url: string): string {
+  const text = title || new URL(url).pathname.replace(/\//g, "-");
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 50) || "page";
+}
+
+function formatTimestamp(): string {
+  const now = new Date();
+  const pad = (n: number) => n.toString().padStart(2, "0");
+  return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+}
+
+function deriveHtmlSnapshotPath(markdownPath: string): string {
+  const parsed = path.parse(markdownPath);
+  const basename = parsed.ext ? parsed.name : parsed.base;
+  return path.join(parsed.dir, `${basename}-captured.html`);
+}
+
+function extractTitleFromMarkdownDocument(document: string): string {
+  const normalized = document.replace(/\r\n/g, "\n");
+  const frontmatterMatch = normalized.match(/^---\n([\s\S]*?)\n---\n?/);
+  if (frontmatterMatch) {
+    const titleLine = frontmatterMatch[1]
+      .split("\n")
+      .find((line) => /^title:\s*/i.test(line));
+
+    if (titleLine) {
+      const rawValue = titleLine.replace(/^title:\s*/i, "").trim();
+      const unquoted = rawValue
+        .replace(/^"(.*)"$/, "$1")
+        .replace(/^'(.*)'$/, "$1")
+        .replace(/\\"/g, '"');
+      if (unquoted) return unquoted;
+    }
+  }
+
+  const headingMatch = normalized.match(/^#\s+(.+)$/m);
+  return headingMatch?.[1]?.trim() ?? "";
+}
+
+function buildDefuddleApiUrl(targetUrl: string): string {
+  return `https://defuddle.md/${encodeURIComponent(targetUrl)}`;
+}
+
+async function fetchDefuddleApiMarkdown(targetUrl: string): Promise<{ markdown: string; title: string }> {
+  const apiUrl = buildDefuddleApiUrl(targetUrl);
+  const response = await fetch(apiUrl, {
+    headers: {
+      accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
+  }
+
+  const markdown = (await response.text()).replace(/\r\n/g, "\n").trim();
+  if (!markdown) {
+    throw new Error("defuddle.md returned empty markdown");
+  }
+
+  return {
+    markdown,
+    title: extractTitleFromMarkdownDocument(markdown),
+  };
+}
+
+async function generateOutputPath(url: string, title: string, outputDir?: string): Promise<string> {
+  const domain = new URL(url).hostname.replace(/^www\./, "");
+  const slug = generateSlug(title, url);
+  const dataDir = outputDir ? path.resolve(outputDir) : resolveUrlToMarkdownDataDir();
+  const basePath = path.join(dataDir, domain, `${slug}.md`);
+
+  if (!(await fileExists(basePath))) {
+    return basePath;
+  }
+
+  const timestampSlug = `${slug}-${formatTimestamp()}`;
+  return path.join(dataDir, domain, `${timestampSlug}.md`);
+}
+
+async function waitForUserSignal(): Promise<void> {
+  console.log("Page opened. Press Enter when ready to capture...");
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  await new Promise<void>((resolve) => {
+    rl.once("line", () => { rl.close(); resolve(); });
+  });
+}
+
+async function captureUrl(args: Args): Promise<ConversionResult> {
+  const existingPort = await findExistingChromePort();
+  const reusing = existingPort !== null;
+  const port = existingPort ?? await getFreePort();
+  const chrome = reusing ? null : await launchChrome(args.url, port, false);
+
+  if (reusing) console.log(`Reusing existing Chrome on port ${port}`);
+
+  let cdp: CdpConnection | null = null;
+  let targetId: string | null = null;
+  try {
+    const wsUrl = await waitForChromeDebugPort(port, 30_000);
+    cdp = await CdpConnection.connect(wsUrl, CDP_CONNECT_TIMEOUT_MS);
+
+    let sessionId: string;
+    if (reusing) {
+      const created = await cdp.send<{ targetId: string }>("Target.createTarget", { url: args.url });
+      targetId = created.targetId;
+      const attached = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
+      sessionId = attached.sessionId;
+      await cdp.send("Network.enable", {}, { sessionId });
+      await cdp.send("Page.enable", {}, { sessionId });
+    } else {
+      const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; type: string; url: string }> }>("Target.getTargets");
+      const pageTarget = targets.targetInfos.find(t => t.type === "page" && t.url.startsWith("http"));
+      if (!pageTarget) throw new Error("No page target found");
+      targetId = pageTarget.targetId;
+      const attached = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
+      sessionId = attached.sessionId;
+      await cdp.send("Network.enable", {}, { sessionId });
+      await cdp.send("Page.enable", {}, { sessionId });
+    }
+
+    if (args.wait) {
+      await waitForUserSignal();
+    } else {
+      console.log("Waiting for page to load...");
+      await Promise.race([
+        waitForPageLoad(cdp, sessionId, 15_000),
+        sleep(8_000)
+      ]);
+      await waitForNetworkIdle(cdp, sessionId, NETWORK_IDLE_TIMEOUT_MS);
+      await sleep(POST_LOAD_DELAY_MS);
+      console.log("Scrolling to trigger lazy load...");
+      await autoScroll(cdp, sessionId, SCROLL_MAX_STEPS, SCROLL_STEP_WAIT_MS);
+      await sleep(POST_LOAD_DELAY_MS);
+    }
+
+    console.log("Capturing page content...");
+    const { html } = await evaluateScript<{ html: string }>(
+      cdp, sessionId, absolutizeUrlsScript, args.timeout
+    );
+
+    return await extractContent(html, args.url);
+  } finally {
+    if (reusing) {
+      if (cdp && targetId) {
+        try { await cdp.send("Target.closeTarget", { targetId }, { timeoutMs: 5_000 }); } catch {}
+      }
+      if (cdp) cdp.close();
+    } else {
+      if (cdp) {
+        try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}
+        cdp.close();
+      }
+      if (chrome) killChrome(chrome);
+    }
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv);
+  if (!args.url) {
+    console.error("Usage: bun main.ts <url> [-o output.md] [--output-dir dir] [--wait] [--timeout ms] [--download-media]");
+    process.exit(1);
+  }
+
+  try {
+    new URL(args.url);
+  } catch {
+    console.error(`Invalid URL: ${args.url}`);
+    process.exit(1);
+  }
+
+  if (args.output) {
+    const stat = await import("node:fs").then(fs => fs.statSync(args.output!, { throwIfNoEntry: false }));
+    if (stat?.isDirectory()) {
+      console.error(`Error: -o path is a directory, not a file: ${args.output}`);
+      process.exit(1);
+    }
+  }
+
+  console.log(`Fetching: ${args.url}`);
+  console.log(`Mode: ${args.wait ? "wait" : "auto"}`);
+
+  let outputPath: string;
+  let htmlSnapshotPath: string | null = null;
+  let document: string;
+  let conversionMethod: string;
+  let fallbackReason: string | undefined;
+
+  try {
+    const result = await captureUrl(args);
+    outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
+    const outputDir = path.dirname(outputPath);
+    htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
+    await mkdir(outputDir, { recursive: true });
+    await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
+
+    document = createMarkdownDocument(result);
+    conversionMethod = result.conversionMethod;
+    fallbackReason = result.fallbackReason;
+  } catch (error) {
+    const primaryError = error instanceof Error ? error.message : String(error);
+    console.warn(`Primary capture failed: ${primaryError}`);
+    console.warn("Trying defuddle.md API fallback...");
+
+    try {
+      const remoteResult = await fetchDefuddleApiMarkdown(args.url);
+      outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir);
+      await mkdir(path.dirname(outputPath), { recursive: true });
+
+      document = remoteResult.markdown;
+      conversionMethod = "defuddle-api";
+      fallbackReason = `Local browser capture failed: ${primaryError}`;
+    } catch (remoteError) {
+      const remoteMessage = remoteError instanceof Error ? remoteError.message : String(remoteError);
+      throw new Error(`Local browser capture failed (${primaryError}); defuddle.md fallback failed (${remoteMessage})`);
+    }
+  }
+
+  if (args.downloadMedia) {
+    const mediaResult = await localizeMarkdownMedia(document, {
+      markdownPath: outputPath,
+      log: console.log,
+    });
+    document = mediaResult.markdown;
+    if (mediaResult.downloadedImages > 0 || mediaResult.downloadedVideos > 0) {
+      console.log(`Downloaded: ${mediaResult.downloadedImages} images, ${mediaResult.downloadedVideos} videos`);
+    }
+  } else {
+    const { images, videos } = countRemoteMedia(document);
+    if (images > 0 || videos > 0) {
+      console.log(`Remote media found: ${images} images, ${videos} videos`);
+    }
+  }
+
+  await writeFile(outputPath, document, "utf-8");
+
+  console.log(`Saved: ${outputPath}`);
+  if (htmlSnapshotPath) {
+    console.log(`Saved HTML: ${htmlSnapshotPath}`);
+  } else {
+    console.log("Saved HTML: unavailable (defuddle.md fallback)");
+  }
+  console.log(`Title: ${extractTitleFromMarkdownDocument(document) || "(no title)"}`);
+  console.log(`Converter: ${conversionMethod}`);
+  if (fallbackReason) {
+    console.warn(`Fallback used: ${fallbackReason}`);
+  }
+}
+
+main().catch((err) => {
+  console.error("Error:", err instanceof Error ? err.message : String(err));
+  process.exit(1);
+});

package/dist/skills/llm-wiki/deps/baoyu-url-to-markdown/scripts/markdown-conversion-shared.ts

@@ -0,0 +1,305 @@
+import { parseHTML } from "linkedom";
+
+export interface PageMetadata {
+  url: string;
+  title: string;
+  description?: string;
+  author?: string;
+  published?: string;
+  coverImage?: string;
+  language?: string;
+  captured_at: string;
+}
+
+export interface ConversionResult {
+  metadata: PageMetadata;
+  markdown: string;
+  rawHtml: string;
+  conversionMethod: string;
+  fallbackReason?: string;
+  variables?: Record<string, string>;
+}
+
+export type AnyRecord = Record<string, unknown>;
+
+export const MIN_CONTENT_LENGTH = 120;
+export const GOOD_CONTENT_LENGTH = 900;
+
+const PUBLISHED_TIME_SELECTORS = [
+  "meta[property='article:published_time']",
+  "meta[name='pubdate']",
+  "meta[name='publishdate']",
+  "meta[name='date']",
+  "time[datetime]",
+];
+
+const ARTICLE_TYPES = new Set([
+  "Article",
+  "NewsArticle",
+  "BlogPosting",
+  "WebPage",
+  "ReportageNewsArticle",
+]);
+
+export function pickString(...values: unknown[]): string | null {
+  for (const value of values) {
+    if (typeof value === "string") {
+      const trimmed = value.trim();
+      if (trimmed) return trimmed;
+    }
+  }
+  return null;
+}
+
+export function normalizeMarkdown(markdown: string): string {
+  return markdown
+    .replace(/\r\n/g, "\n")
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export function parseDocument(html: string): Document {
+  const normalized = /<\s*html[\s>]/i.test(html)
+    ? html
+    : `<!doctype html><html><body>${html}</body></html>`;
+  return parseHTML(normalized).document as unknown as Document;
+}
+
+export function sanitizeHtml(html: string): string {
+  const { document } = parseHTML(`<div id="__root">${html}</div>`);
+  const root = document.querySelector("#__root");
+  if (!root) return html;
+
+  for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
+    for (const el of root.querySelectorAll(selector)) {
+      el.remove();
+    }
+  }
+
+  return root.innerHTML;
+}
+
+export function extractTextFromHtml(html: string): string {
+  const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
+  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
+    for (const el of document.querySelectorAll(selector)) {
+      el.remove();
+    }
+  }
+  return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
+}
+
+export function getMetaContent(document: Document, names: string[]): string | null {
+  for (const name of names) {
+    const element =
+      document.querySelector(`meta[name="${name}"]`) ??
+      document.querySelector(`meta[property="${name}"]`);
+    const content = element?.getAttribute("content");
+    if (content && content.trim()) return content.trim();
+  }
+  return null;
+}
+
+function normalizeLanguageTag(value: string | null): string | null {
+  if (!value) return null;
+
+  const trimmed = value.trim();
+  if (!trimmed) return null;
+
+  const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
+  if (!primary) return null;
+
+  return primary.replace(/_/g, "-");
+}
+
+function flattenJsonLdItems(data: unknown): AnyRecord[] {
+  if (!data || typeof data !== "object") return [];
+  if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);
+
+  const item = data as AnyRecord;
+  if (Array.isArray(item["@graph"])) {
+    return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
+  }
+
+  return [item];
+}
+
+function parseJsonLdScripts(document: Document): AnyRecord[] {
+  const results: AnyRecord[] = [];
+  const scripts = document.querySelectorAll("script[type='application/ld+json']");
+
+  for (const script of scripts) {
+    try {
+      const data = JSON.parse(script.textContent ?? "");
+      results.push(...flattenJsonLdItems(data));
+    } catch {
+      // Ignore malformed blocks.
+    }
+  }
+
+  return results;
+}
+
+function isArticleType(item: AnyRecord): boolean {
+  const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
+  return typeof value === "string" && ARTICLE_TYPES.has(value);
+}
+
+function extractAuthorFromJsonLd(authorData: unknown): string | null {
+  if (typeof authorData === "string") return authorData;
+  if (!authorData || typeof authorData !== "object") return null;
+
+  if (Array.isArray(authorData)) {
+    const names = authorData
+      .map((author) => extractAuthorFromJsonLd(author))
+      .filter((name): name is string => Boolean(name));
+    return names.length > 0 ? names.join(", ") : null;
+  }
+
+  const author = authorData as AnyRecord;
+  return typeof author.name === "string" ? author.name : null;
+}
+
+function extractPrimaryJsonLdMeta(document: Document): Partial<PageMetadata> {
+  for (const item of parseJsonLdScripts(document)) {
+    if (!isArticleType(item)) continue;
+
+    return {
+      title: pickString(item.headline, item.name) ?? undefined,
+      description: pickString(item.description) ?? undefined,
+      author: extractAuthorFromJsonLd(item.author) ?? undefined,
+      published: pickString(item.datePublished, item.dateCreated) ?? undefined,
+      coverImage:
+        pickString(
+          item.image,
+          (item.image as AnyRecord | undefined)?.url,
+          (Array.isArray(item.image) ? item.image[0] : undefined) as unknown
+        ) ?? undefined,
+    };
+  }
+
+  return {};
+}
+
+export function extractPublishedTime(document: Document): string | null {
+  for (const selector of PUBLISHED_TIME_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (!el) continue;
+    const value = el.getAttribute("content") ?? el.getAttribute("datetime");
+    if (value && value.trim()) return value.trim();
+  }
+  return null;
+}
+
+export function extractTitle(document: Document): string | null {
+  const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content");
+  if (ogTitle && ogTitle.trim()) return ogTitle.trim();
+
+  const twitterTitle = document.querySelector("meta[name='twitter:title']")?.getAttribute("content");
+  if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim();
+
+  const title = document.querySelector("title")?.textContent?.trim();
+  if (title) {
+    const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim();
+    if (cleaned) return cleaned;
+  }
+
+  const h1 = document.querySelector("h1")?.textContent?.trim();
+  return h1 || null;
+}
+
+export function extractMetadataFromHtml(html: string, url: string, capturedAt: string): PageMetadata {
+  const document = parseDocument(html);
+  const jsonLd = extractPrimaryJsonLdMeta(document);
+  const timeEl = document.querySelector("time[datetime]");
+  const htmlLang = normalizeLanguageTag(document.documentElement?.getAttribute("lang"));
+  const metaLanguage = normalizeLanguageTag(
+    pickString(
+      getMetaContent(document, ["language", "content-language", "og:locale"]),
+      document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content")
+    )
+  );
+
+  return {
+    url,
+    title:
+      pickString(
+        getMetaContent(document, ["og:title", "twitter:title"]),
+        jsonLd.title,
+        document.querySelector("h1")?.textContent,
+        document.title
+      ) ?? "",
+    description:
+      pickString(
+        getMetaContent(document, ["description", "og:description", "twitter:description"]),
+        jsonLd.description
+      ) ?? undefined,
+    author:
+      pickString(
+        getMetaContent(document, ["author", "article:author", "twitter:creator"]),
+        jsonLd.author
+      ) ?? undefined,
+    published:
+      pickString(
+        timeEl?.getAttribute("datetime"),
+        getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
+        jsonLd.published,
+        extractPublishedTime(document)
+      ) ?? undefined,
+    coverImage:
+      pickString(
+        getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
+        jsonLd.coverImage
+      ) ?? undefined,
+    language: pickString(htmlLang, metaLanguage) ?? undefined,
+    captured_at: capturedAt,
+  };
+}
+
+export function isMarkdownUsable(markdown: string, html: string): boolean {
+  const normalized = normalizeMarkdown(markdown);
+  if (!normalized) return false;
+
+  const htmlTextLength = extractTextFromHtml(html).length;
+  if (htmlTextLength < MIN_CONTENT_LENGTH) return true;
+
+  if (normalized.length >= 80) return true;
+  return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
+}
+
+export function isYouTubeUrl(url: string): boolean {
+  try {
+    const hostname = new URL(url).hostname.toLowerCase();
+    return hostname === "youtu.be" || hostname.endsWith(".youtube.com") || hostname === "youtube.com";
+  } catch {
+    return false;
+  }
+}
+
+function escapeYamlValue(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\r?\n/g, "\\n");
+}
+
+export function formatMetadataYaml(meta: PageMetadata): string {
+  const lines = ["---"];
+  lines.push(`url: ${meta.url}`);
+  lines.push(`title: "${escapeYamlValue(meta.title)}"`);
+  if (meta.description) lines.push(`description: "${escapeYamlValue(meta.description)}"`);
+  if (meta.author) lines.push(`author: "${escapeYamlValue(meta.author)}"`);
+  if (meta.published) lines.push(`published: "${escapeYamlValue(meta.published)}"`);
+  if (meta.coverImage) lines.push(`coverImage: "${escapeYamlValue(meta.coverImage)}"`);
+  if (meta.language) lines.push(`language: "${escapeYamlValue(meta.language)}"`);
+  lines.push(`captured_at: "${escapeYamlValue(meta.captured_at)}"`);
+  lines.push("---");
+  return lines.join("\n");
+}
+
+export function createMarkdownDocument(result: ConversionResult): string {
+  const yaml = formatMetadataYaml(result.metadata);
+  const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
+  const hasTitle = titleRegex.test(result.markdown.trimStart());
+  const title = result.metadata.title && !hasTitle ? `\n\n# ${result.metadata.title}\n\n` : "\n\n";
+  return yaml + title + result.markdown;
+}