@s-hirano-ist/s-scripts 1.26.8 → 4.0.1

package/dist/update-json-articles.js

@@ -1,114 +1,118 @@
 #!/usr/bin/env node
-import { createPushoverService } from "@s-hirano-ist/s-notification";
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  decodeHtml
+} from "./chunk-UXGLXHQ6.js";
+import {
+  createPushoverService
+} from "./chunk-7PVU2Z32.js";
+import "./chunk-JZOR5GA2.js";
+
+// src/update-json-articles.ts
 import * as cheerio from "cheerio";
 import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { setTimeout } from "node:timers/promises";
-import { decodeHtml } from "./lib/html-charset.js";
 async function getOgTags(url) {
-    try {
-        console.log(`Fetching OG tags for: ${url}`);
-        const response = await fetch(url, {
-            headers: {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            },
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP error! status: ${response.status}`);
-        }
-        const buffer = Buffer.from(await response.arrayBuffer());
-        const html = decodeHtml(response.headers, buffer);
-        const $ = cheerio.load(html);
-        const ogTags = {};
-        $("meta[property]").each((_, el) => {
-            const property = $(el).attr("property")?.trim();
-            const content = $(el).attr("content");
-            if (property && content) {
-                ogTags[property] = content;
-            }
-        });
-        const ogImageUrl = ogTags["og:image"];
-        const ogTitle = ogTags["og:title"];
-        const ogDescription = ogTags["og:description"];
-        return { ogImageUrl, ogTitle, ogDescription };
-    }
-    catch (error) {
-        console.error(`Error fetching OG tags for ${url}:`, error);
-        return {};
+  try {
+    console.log(`Fetching OG tags for: ${url}`);
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+      }
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
     }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const html = decodeHtml(response.headers, buffer);
+    const $ = cheerio.load(html);
+    const ogTags = {};
+    $("meta[property]").each((_, el) => {
+      const property = $(el).attr("property")?.trim();
+      const content = $(el).attr("content");
+      if (property && content) {
+        ogTags[property] = content;
+      }
+    });
+    const ogImageUrl = ogTags["og:image"];
+    const ogTitle = ogTags["og:title"];
+    const ogDescription = ogTags["og:description"];
+    return { ogImageUrl, ogTitle, ogDescription };
+  } catch (error) {
+    console.error(`Error fetching OG tags for ${url}:`, error);
+    return {};
+  }
 }
 async function processArticleFile(filePath) {
-    try {
-        const jsonContent = readFileSync(filePath, "utf-8");
-        const articleData = JSON.parse(jsonContent);
-        console.log(`Processing ${filePath}...`);
-        for (const item of articleData.body) {
-            // Skip if OG tags already exist
-            if (item.ogImageUrl || item.ogTitle || item.ogDescription || item.skip) {
-                continue;
-            }
-            const ogTags = await getOgTags(item.url);
-            // Only add properties if they have values
-            if (ogTags.ogImageUrl)
-                item.ogImageUrl = ogTags.ogImageUrl;
-            else
-                console.warn(`No OG image found for ${item.url}`);
-            if (ogTags.ogTitle)
-                item.ogTitle = ogTags.ogTitle;
-            else
-                console.warn(`No OG title found for ${item.url}`);
-            if (ogTags.ogDescription)
-                item.ogDescription = ogTags.ogDescription;
-            else
-                console.warn(`No OG description found for ${item.url}`);
-            // Add a small delay to avoid overwhelming servers
-            await setTimeout(1000);
-        }
-        // Write back to file
-        writeFileSync(filePath, `${JSON.stringify(articleData, null, "\t")}\n`, "utf-8");
-        console.log(`Completed processing ${filePath}`);
-    }
-    catch (error) {
-        console.error(`Error processing ${filePath}:`, error);
+  try {
+    const jsonContent = readFileSync(filePath, "utf-8");
+    const articleData = JSON.parse(jsonContent);
+    console.log(`Processing ${filePath}...`);
+    for (const item of articleData.body) {
+      if (item.ogImageUrl || item.ogTitle || item.ogDescription || item.skip) {
+        continue;
+      }
+      const ogTags = await getOgTags(item.url);
+      if (ogTags.ogImageUrl) item.ogImageUrl = ogTags.ogImageUrl;
+      else console.warn(`No OG image found for ${item.url}`);
+      if (ogTags.ogTitle) item.ogTitle = ogTags.ogTitle;
+      else console.warn(`No OG title found for ${item.url}`);
+      if (ogTags.ogDescription) item.ogDescription = ogTags.ogDescription;
+      else console.warn(`No OG description found for ${item.url}`);
+      await setTimeout(1e3);
     }
+    writeFileSync(
+      filePath,
+      `${JSON.stringify(articleData, null, "	")}
+`,
+      "utf-8"
+    );
+    console.log(`Completed processing ${filePath}`);
+  } catch (error) {
+    console.error(`Error processing ${filePath}:`, error);
+  }
 }
 async function main() {
-    const env = {
-        PUSHOVER_URL: process.env.PUSHOVER_URL,
-        PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
-        PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
-    };
-    if (Object.values(env).some((v) => !v)) {
-        throw new Error("Required environment variables are not set.");
+  const env = {
+    PUSHOVER_URL: process.env.PUSHOVER_URL,
+    PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
+    PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN
+  };
+  if (Object.values(env).some((v) => !v)) {
+    throw new Error("Required environment variables are not set.");
+  }
+  const notificationService = createPushoverService({
+    url: env.PUSHOVER_URL ?? "",
+    userKey: env.PUSHOVER_USER_KEY ?? "",
+    appToken: env.PUSHOVER_APP_TOKEN ?? ""
+  });
+  const articleDir = join(process.cwd(), "json", "article");
+  try {
+    const files = readdirSync(articleDir).filter(
+      (file) => file.endsWith(".json")
+    );
+    console.log(`Found ${files.length} JSON files to process`);
+    for (const file of files) {
+      const filePath = join(articleDir, file);
+      await processArticleFile(filePath);
     }
-    const notificationService = createPushoverService({
-        url: env.PUSHOVER_URL ?? "",
-        userKey: env.PUSHOVER_USER_KEY ?? "",
-        appToken: env.PUSHOVER_APP_TOKEN ?? "",
+    console.log("All files processed successfully!");
+    await notificationService.notifyInfo("update-json-articles completed", {
+      caller: "update-json-articles"
     });
-    const articleDir = join(process.cwd(), "json", "article");
-    try {
-        const files = readdirSync(articleDir).filter((file) => file.endsWith(".json"));
-        console.log(`Found ${files.length} JSON files to process`);
-        for (const file of files) {
-            const filePath = join(articleDir, file);
-            await processArticleFile(filePath);
-        }
-        console.log("All files processed successfully!");
-        await notificationService.notifyInfo("update-json-articles completed", {
-            caller: "update-json-articles",
-        });
-    }
-    catch (error) {
-        console.error("Error in main process:", error);
-        await notificationService.notifyError(`update-json-articles failed: ${String(error)}`, {
-            caller: "update-json-articles",
-        });
-        process.exit(1);
-    }
+  } catch (error) {
+    console.error("Error in main process:", error);
+    await notificationService.notifyError(
+      `update-json-articles failed: ${String(error)}`,
+      {
+        caller: "update-json-articles"
+      }
+    );
+    process.exit(1);
+  }
 }
 main().catch((error) => {
-    console.error(error);
-    process.exit(1);
+  console.error(error);
+  process.exit(1);
 });
-//# sourceMappingURL=update-json-articles.js.map

package/dist/update-raw-articles.js

@@ -1,102 +1,104 @@
 #!/usr/bin/env node
-import { createPushoverService } from "@s-hirano-ist/s-notification";
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  decodeHtml
+} from "./chunk-UXGLXHQ6.js";
+import {
+  createPushoverService
+} from "./chunk-7PVU2Z32.js";
+import "./chunk-JZOR5GA2.js";
+
+// src/update-raw-articles.ts
 import { existsSync } from "node:fs";
 import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import TurndownService from "turndown";
-import { decodeHtml } from "./lib/html-charset.js";
-const JSON_DIR = "json/article";
-const OUTPUT_DIR = "raw/article";
+var JSON_DIR = "json/article";
+var OUTPUT_DIR = "raw/article";
 async function fetchWebsiteMarkdown(url) {
-    try {
-        const response = await fetch(url, {
-            headers: {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            },
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP error! status: ${response.status}`);
-        }
-        const buffer = Buffer.from(await response.arrayBuffer());
-        const html = decodeHtml(response.headers, buffer);
-        const turndownService = new TurndownService({
-            headingStyle: "atx",
-            codeBlockStyle: "fenced",
-            bulletListMarker: "-",
-            strongDelimiter: "**",
-            emDelimiter: "_",
-        });
-        turndownService.remove(["script", "style", "nav", "footer", "aside"]);
-        turndownService.addRule("preserveLinks", {
-            filter: ["a"],
-            replacement: (content, node) => {
-                const element = node;
-                const href = element.getAttribute("href");
-                if (!href)
-                    return content;
-                if (href.startsWith("http") || href.startsWith("https")) {
-                    return `[${content}](${href})`;
-                }
-                try {
-                    const absoluteUrl = new URL(href, url).href;
-                    return `[${content}](${absoluteUrl})`;
-                }
-                catch {
-                    return content;
-                }
-            },
-        });
-        turndownService.addRule("preserveImages", {
-            filter: ["img"],
-            replacement: (_content, node) => {
-                const element = node;
-                const src = element.getAttribute("src");
-                const alt = element.getAttribute("alt") || "";
-                if (!src)
-                    return "";
-                if (src.startsWith("http") || src.startsWith("https")) {
-                    return `![${alt}](${src})`;
-                }
-                try {
-                    const absoluteUrl = new URL(src, url).href;
-                    return `![${alt}](${absoluteUrl})`;
-                }
-                catch {
-                    return "";
-                }
-            },
-        });
-        const markdown = turndownService.turndown(html);
-        return markdown.trim();
-    }
-    catch (error) {
-        console.error(`Failed to fetch ${url}:`, error);
-        return "";
+  try {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+      }
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
     }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const html = decodeHtml(response.headers, buffer);
+    const turndownService = new TurndownService({
+      headingStyle: "atx",
+      codeBlockStyle: "fenced",
+      bulletListMarker: "-",
+      strongDelimiter: "**",
+      emDelimiter: "_"
+    });
+    turndownService.remove(["script", "style", "nav", "footer", "aside"]);
+    turndownService.addRule("preserveLinks", {
+      filter: ["a"],
+      replacement: (content, node) => {
+        const element = node;
+        const href = element.getAttribute("href");
+        if (!href) return content;
+        if (href.startsWith("http") || href.startsWith("https")) {
+          return `[${content}](${href})`;
+        }
+        try {
+          const absoluteUrl = new URL(href, url).href;
+          return `[${content}](${absoluteUrl})`;
+        } catch {
+          return content;
+        }
+      }
+    });
+    turndownService.addRule("preserveImages", {
+      filter: ["img"],
+      replacement: (_content, node) => {
+        const element = node;
+        const src = element.getAttribute("src");
+        const alt = element.getAttribute("alt") || "";
+        if (!src) return "";
+        if (src.startsWith("http") || src.startsWith("https")) {
+          return `![${alt}](${src})`;
+        }
+        try {
+          const absoluteUrl = new URL(src, url).href;
+          return `![${alt}](${absoluteUrl})`;
+        } catch {
+          return "";
+        }
+      }
+    });
+    const markdown = turndownService.turndown(html);
+    return markdown.trim();
+  } catch (error) {
+    console.error(`Failed to fetch ${url}:`, error);
+    return "";
+  }
 }
 async function jsonToMarkdown(jsonFile) {
-    const rawData = await readFile(jsonFile, "utf-8");
-    const data = JSON.parse(rawData);
-    // oxlint-disable-next-line typescript/no-unnecessary-condition -- data comes from an external JSON file; the runtime value may not match ArticlesData
-    const bodyList = data.body || [];
-    if (!existsSync(OUTPUT_DIR)) {
-        await mkdir(OUTPUT_DIR, { recursive: true });
-    }
-    for (const item of bodyList) {
-        const { title, quote, url, skip } = item;
-        try {
-            const strippedUrl = url.replace(/^https?:\/\//u, "");
-            const safeUrl = encodeURIComponent(strippedUrl);
-            const outputFilename = `${safeUrl}.md`;
-            const outputPath = path.join(OUTPUT_DIR, outputFilename);
-            if (existsSync(outputPath) || skip) {
-                continue;
-            }
-            const websiteText = await fetchWebsiteMarkdown(url);
-            if (!websiteText) {
-                continue;
-            }
-            const markdownContent = `# [${title}](${url})
+  const rawData = await readFile(jsonFile, "utf-8");
+  const data = JSON.parse(rawData);
+  const bodyList = data.body || [];
+  if (!existsSync(OUTPUT_DIR)) {
+    await mkdir(OUTPUT_DIR, { recursive: true });
+  }
+  for (const item of bodyList) {
+    const { title, quote, url, skip } = item;
+    try {
+      const strippedUrl = url.replace(/^https?:\/\//u, "");
+      const safeUrl = encodeURIComponent(strippedUrl);
+      const outputFilename = `${safeUrl}.md`;
+      const outputPath = path.join(OUTPUT_DIR, outputFilename);
+      if (existsSync(outputPath) || skip) {
+        continue;
+      }
+      const websiteText = await fetchWebsiteMarkdown(url);
+      if (!websiteText) {
+        continue;
+      }
+      const markdownContent = `# [${title}](${url})
 
 ---
 
@@ -108,51 +110,49 @@ ${quote}
 
 ${websiteText}
 `;
-            await writeFile(outputPath, markdownContent, "utf-8");
-            console.log(`Exported: ${outputPath}`);
-        }
-        catch {
-            console.error("Error on file:", url);
-        }
+      await writeFile(outputPath, markdownContent, "utf-8");
+      console.log(`Exported: ${outputPath}`);
+    } catch {
+      console.error("Error on file:", url);
     }
+  }
 }
 async function main() {
-    const env = {
-        PUSHOVER_URL: process.env.PUSHOVER_URL,
-        PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
-        PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
-    };
-    if (Object.values(env).some((v) => !v)) {
-        throw new Error("Required environment variables are not set.");
+  const env = {
+    PUSHOVER_URL: process.env.PUSHOVER_URL,
+    PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
+    PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN
+  };
+  if (Object.values(env).some((v) => !v)) {
+    throw new Error("Required environment variables are not set.");
+  }
+  const notificationService = createPushoverService({
+    url: env.PUSHOVER_URL ?? "",
+    userKey: env.PUSHOVER_USER_KEY ?? "",
+    appToken: env.PUSHOVER_APP_TOKEN ?? ""
+  });
+  try {
+    const jsonFiles = await readdir(JSON_DIR);
+    const jsonFilePaths = jsonFiles.filter((file) => file.endsWith(".json")).map((file) => path.join(JSON_DIR, file));
+    for (const jsonFile of jsonFilePaths) {
+      console.log(`Processing: ${jsonFile}`);
+      await jsonToMarkdown(jsonFile);
     }
-    const notificationService = createPushoverService({
-        url: env.PUSHOVER_URL ?? "",
-        userKey: env.PUSHOVER_USER_KEY ?? "",
-        appToken: env.PUSHOVER_APP_TOKEN ?? "",
+    await notificationService.notifyInfo("update-raw-articles completed", {
+      caller: "update-raw-articles"
     });
-    try {
-        const jsonFiles = await readdir(JSON_DIR);
-        const jsonFilePaths = jsonFiles
-            .filter((file) => file.endsWith(".json"))
-            .map((file) => path.join(JSON_DIR, file));
-        for (const jsonFile of jsonFilePaths) {
-            console.log(`Processing: ${jsonFile}`);
-            await jsonToMarkdown(jsonFile);
-        }
-        await notificationService.notifyInfo("update-raw-articles completed", {
-            caller: "update-raw-articles",
-        });
-    }
-    catch (error) {
-        console.error("Error in main process:", error);
-        await notificationService.notifyError(`update-raw-articles failed: ${String(error)}`, {
-            caller: "update-raw-articles",
-        });
-        process.exit(1);
-    }
+  } catch (error) {
+    console.error("Error in main process:", error);
+    await notificationService.notifyError(
+      `update-raw-articles failed: ${String(error)}`,
+      {
+        caller: "update-raw-articles"
+      }
+    );
+    process.exit(1);
+  }
 }
 main().catch((error) => {
-    console.error(error);
-    process.exit(1);
+  console.error(error);
+  process.exit(1);
 });
-//# sourceMappingURL=update-raw-articles.js.map