h56-github-scrapper 1.0.3 → 1.0.5

package/main-scrapping.js

@@ -1,32 +1,30 @@
 /**
- * main-scrapping.js
+ * main-scrapping.js (ESM)
  *
- * NPM package style single-file scraper for GitHub user data.
- * Package name suggestion: h56-github-scrapper
+ * Converted to ECMAScript Modules (ESM) for Node.js.
+ * - Preserves original logic and features (scraping, retry/backoff, spinner, CLI).
+ * - Adds optional translator integration (uses translate-engine/translate.js as ESM).
+ * - Uses dynamic import for third-party runtime deps after optional auto-install.
  *
- * Features:
- * - Exports programmatic functions: scrapeProfile, scrapeRepos, scrapeUser, calculateStats
- * - CLI entry when run directly: node main-scrapping.js <username> [--json] [--output=file]
- * - Optional translator integration (h56-translator) with selectable fields to translate
- * - Automatic install of missing npm runtime dependencies (asks for consent when needed)
- * - Robust retry/backoff, polite scraping delay, spinner (ora) fallback
- * - Well-structured results and JSON output support
  *
- * NOTE:
- * - The recommended (proper) approach for publishing as an npm package is to list dependencies
- *   in package.json. The runtime auto-installer is implemented as a convenience only.
- * - Scraping HTML may break if GitHub changes markup. Consider using GitHub API for production.
+ * This file is ready to copy/paste. It intentionally preserves behavior and CLI options
+ * while modernizing module semantics to ESM.
  */
 
-const fs = require("fs");
-const path = require("path");
-const { spawnSync } = require("child_process");
-const os = require("os");
+import fs from "fs";
+import path from "path";
+import { spawnSync } from "child_process";
+import os from "os";
+import { createRequire } from "module";
+import readline from "readline";
+import { fileURLToPath } from "url";
+
+const require = createRequire(import.meta.url);
 
 // -------------------------
-// Ensure runtime deps
+// Ensure runtime deps (async)
 // -------------------------
-function ensureDependencies(deps = []) {
+async function ensureDependencies(deps = []) {
   const missing = deps.filter((d) => {
     try {
       require.resolve(d);
@@ -50,45 +48,29 @@ function ensureDependencies(deps = []) {
     consent = true;
     console.log("Non-interactive environment detected, installing automatically...");
   } else {
-    const rl = require("readline").createInterface({
+    const rl = readline.createInterface({
       input: process.stdin,
       output: process.stdout,
     });
 
-    const answer = new Promise((resolve) =>
+    // Prompt with 10s timeout; default to yes
+    const questionPromise = new Promise((resolve) =>
       rl.question(`Install missing dependencies now? (Y/n): `, (a) => {
         rl.close();
-        resolve(a.trim().toLowerCase());
+        resolve(String(a || "").trim().toLowerCase());
       })
     );
 
-    // synchronous wait via spawnSync is not used here; use await-like pattern:
-    // To keep file runnable without top-level await, use de-facto blocking by reading from stdin synchronously not available.
-    // We'll perform synchronous prompt by checking answer resolved synchronously above (it won't), so instead do a simpler default:
-    // If user doesn't answer quickly, default to yes after 10s.
-    // Implementing a small blocking wait using child_process to call 'bash -c read -t 10' is platform-dependent.
-    // For simplicity here, assume consent if user presses Enter quickly — we'll read synchronously via question with callback, then block via a small busy loop until resolved.
-    let resolved = false;
-    let ansValue = "";
-    answer.then((v) => {
-      resolved = true;
-      ansValue = v;
-    });
-
-    // Wait (busy-loop) until resolved — acceptable for a small prompt in CLI tool.
-    const waitUntil = Date.now() + 10000; // 10s timeout
-    while (!resolved && Date.now() < waitUntil) {
-      // small sleep
-      const start = Date.now();
-      while (Date.now() - start < 50) {}
-    }
+    const ans = await Promise.race([
+      questionPromise,
+      new Promise((resolve) => setTimeout(() => resolve(null), 10000)),
+    ]);
 
-    if (!resolved) {
-      // default to yes
+    if (ans === null) {
       consent = true;
       console.log("\nNo answer, defaulting to install.\n");
     } else {
-      consent = !ansValue || ansValue === "y" || ansValue === "yes";
+      consent = !ans || ans === "y" || ans === "yes";
     }
   }
 
@@ -99,8 +81,9 @@ function ensureDependencies(deps = []) {
 
   // Run npm install for missing deps
   console.log(`Installing: ${missing.join(" ")} ...`);
+  const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
   const args = ["install", "--save", ...missing];
-  const result = spawnSync(process.platform === "win32" ? "npm.cmd" : "npm", args, {
+  const result = spawnSync(npmCmd, args, {
     stdio: "inherit",
     shell: false,
   });
@@ -114,14 +97,21 @@ function ensureDependencies(deps = []) {
   console.log("Dependencies installed, continuing...");
 }
 
-// required runtime modules
-ensureDependencies(["axios", "cheerio", "ora", "yargs"]);
+// Ensure runtime modules (same set as original)
+await ensureDependencies(["axios", "cheerio", "ora", "yargs"]);
 
-// now require installed modules
-const axios = require("axios");
-const cheerio = require("cheerio");
-const ora = require("ora");
-const yargs = require("yargs");
+// Dynamic import of runtime libraries (so they are loaded after potential install)
+const axiosModule = await import("axios");
+const axios = axiosModule.default || axiosModule;
+
+const cheerioModule = await import("cheerio");
+const cheerio = cheerioModule.default || cheerioModule;
+
+const oraModule = await import("ora");
+const ora = oraModule.default || oraModule;
+
+const yargsModule = await import("yargs");
+const yargs = yargsModule.default || yargsModule;
 
 // -------------------------
 // Utilities & Config
@@ -168,21 +158,38 @@ function validateUsername(username) {
 }
 
 // -------------------------
-// Translator loader (optional)
+// Translator loader (optional, lazy)
 // -------------------------
-// Provide a safe, synchronous attempt to load the local CommonJS wrapper
-// translate-engine/translate.js. If not present, translatorFn will be null.
-// Consumers of translation should handle null translatorFn.
-let translatorFn = null;
-try {
-  // prefer the local wrapper shipped with the package (does dynamic require to h56-translator)
-  const tmod = require("./translate-engine/translate");
-  if (tmod && typeof tmod.translate === "function") {
-    translatorFn = tmod.translate;
+// We keep a lazy loader so translator is imported only if requested.
+let _translatorModule = null; // module namespace
+let _translatorLoadAttempted = false;
+
+async function loadTranslatorModule() {
+  if (_translatorModule) return _translatorModule;
+  if (_translatorLoadAttempted) return null;
+  _translatorLoadAttempted = true;
+  try {
+    // prefer the package-local wrapper (ESM) if present
+    // path relative to this file
+    const wrapperPath = new URL("./translate-engine/translate.js", import.meta.url).pathname;
+    // attempt dynamic import; if module not present or h56-translator not installed, it will throw
+    const mod = await import(wrapperPath);
+    if (mod && typeof mod.translate === "function") {
+      _translatorModule = mod;
+      return _translatorModule;
+    }
+    // fallback: try importing the package directly
+    const pkg = await import("h56-translator");
+    if (pkg && (typeof pkg.translate === "function" || typeof pkg.default === "function")) {
+      const translateFn = typeof pkg.translate === "function" ? pkg.translate : pkg.default;
+      _translatorModule = { translate: translateFn };
+      return _translatorModule;
+    }
+    return null;
+  } catch (err) {
+    // translator not available — return null and let callers handle
+    return null;
   }
-} catch (e) {
-  // not installed / wrapper not available; keep translatorFn null
-  translatorFn = null;
 }
 
 // -------------------------
@@ -391,9 +398,9 @@ class GithubScraper {
       }
     }
 
-    // translator function must exist
-    const tfn = translatorFn;
-    if (!tfn) {
+    // translator function must exist (lazy load)
+    const mod = await loadTranslatorModule();
+    if (!mod || typeof mod.translate !== "function") {
       const msg =
         "Optional translator is not available. Install 'h56-translator' (and ensure translate-engine/translate.js is present) to enable translations.";
       if (opts.failOnMissing) {
@@ -409,6 +416,7 @@ class GithubScraper {
         return result;
       }
     }
+    const tfn = mod.translate;
 
     // perform profile translation
     try {
@@ -559,48 +567,52 @@ class GithubScraper {
 }
 
 // -------------------------
-// Exports (for npm usage)
+// Exports (for usage as package/module)
 // -------------------------
 const defaultScraper = new GithubScraper();
 
-// Export translator helper (attempt to load; provide helpful fallback if not available)
-let h56translate;
-try {
-  const tmod = require("./translate-engine/translate");
-  if (tmod && typeof tmod.translate === "function") {
-    h56translate = tmod.translate;
-  } else {
-    h56translate = async function () {
-      throw new Error(
-        "Optional translator module loaded but export shape is not recognized. Ensure 'h56-translator' is installed and compatible."
-      );
-    };
-  }
-} catch (e) {
-  h56translate = async function () {
+// h56translate helper: lazy-call translator module when invoked
+export async function h56translate(text, targetLang, options) {
+  const mod = await loadTranslatorModule();
+  if (!mod || typeof mod.translate !== "function") {
     throw new Error(
-      "Optional translator is not available. Install it with `npm install h56-translator` or run `npm install` to trigger postinstall."
+      "Optional translator is not available. Install it with `npm install h56-translator` or ensure translate-engine/translate.js exists and is usable."
     );
-  };
+  }
+  return await mod.translate(text, targetLang, options);
 }
 
-module.exports = {
+export {
   GithubScraper,
   defaultScraper,
-  scrapeProfile: (username, opts) => defaultScraper.scrapeProfile(username, opts),
-  scrapeRepos: (username, opts) => defaultScraper.scrapeRepos(username, opts),
-  scrapeUser: (username, opts) => defaultScraper.scrapeUser(username, opts),
-  calculateStats: (repos) => defaultScraper.calculateStats(repos),
-  printResult: GithubScraper.printResult,
-  // new optional helper: h56translate(text, targetLang, options?)
-  h56translate,
+  // convenience wrappers
+  // note: keep same signatures as before (username, opts)
+  async function scrapeProfile(username, opts) {
+    return defaultScraper.scrapeProfile(username, opts);
+  },
+  async function scrapeRepos(username, opts) {
+    return defaultScraper.scrapeRepos(username, opts);
+  },
+  async function scrapeUser(username, opts) {
+    return defaultScraper.scrapeUser(username, opts);
+  },
+  function calculateStats(repos) {
+    return defaultScraper.calculateStats(repos);
+  },
+  GithubScraper.printResult as printResult,
 };
 
+// The above export block uses named exports compatible with ESM consumers.
+// For backwards compatibility with consumers expecting a default CommonJS export,
+// also write a small CJS-compatible default export file (optional) when packaging.
+
 // -------------------------
 // CLI behavior when run directly
 // -------------------------
-if (require.main === module) {
+const __filename = fileURLToPath(import.meta.url);
+if (process.argv[1] === __filename) {
   (async () => {
+    // build argv using yargs (same API as original)
     const argv = yargs(process.argv.slice(2))
       .usage("Usage: $0 <username> [options]")
       .option("json", {

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "h56-github-scrapper",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "GitHub user scraper",
+  "type": "module",
   "main": "main-scrapping.js",
   "bin": {
     "h56-github-scrapper": "main-scrapping.js"

package/script/ensure-external-deps.js

@@ -1,44 +1,75 @@
 #!/usr/bin/env node
-// scripts/ensure-external-deps.js
-// Postinstall helper: idempotent installer for optional external deps.
-// - Will attempt to install packages listed in optionalDeps if missing.
-// - Skips automatic install in CI unless H56_FORCE_POSTINSTALL=1
-// - Does not throw on failure to avoid breaking `npm install` completely.
+/**
+ * scripts/ensure-external-deps.js (ESM)
+ *
+ * Postinstall helper: idempotent installer for optional external deps.
+ * - Implemented as a full ESM module using top-level await.
+ * - Detects missing packages using dynamic `import()` (avoids createRequire / CJS).
+ * - Skips automatic install in CI by default; set H56_FORCE_POSTINSTALL=1 to override.
+ * - Uses spawnSync to run npm install and keeps the script resilient (won't throw on failure).
+ *
+ */
 
-const { spawnSync } = require("child_process");
+import { spawnSync } from "child_process";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
 const optionalDeps = ["h56-translator"];
 
-function isInstalled(name) {
+/**
+ * Check whether a package is resolvable via dynamic import.
+ * Uses import() which will attempt to load the package; this is the most portable
+ * ESM-compatible way to test presence without using CJS helpers.
+ *
+ * We deliberately avoid `require.resolve` / createRequire to keep file pure ESM.
+ */
+async function isInstalled(name) {
   try {
-    require.resolve(name);
+    // dynamic import of a bare specifier will resolve via Node's resolver
+    await import(name);
     return true;
-  } catch (_) {
+  } catch (err) {
+    // import failed => treat as missing
     return false;
   }
 }
 
+/**
+ * Run npm install for the given list of packages (synchronous to keep postinstall simple).
+ */
 function installDeps(deps) {
-  if (!deps.length) return;
+  if (!deps || deps.length === 0) return;
   const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
   const args = ["install", "--no-audit", "--no-fund", "--save", ...deps];
-  console.log("Installing optional dependencies:", deps.join(", "));
-  const res = spawnSync(npmCmd, args, { stdio: "inherit" });
+  console.log(`Installing optional dependencies: ${deps.join(", ")}`);
+  const res = spawnSync(npmCmd, args, { stdio: "inherit", shell: false, cwd: process.cwd() });
   if (res.error || res.status !== 0) {
-    console.error("Failed to install optional dependencies. You can run manually:");
+    console.error("Failed to install optional dependencies automatically. Please run manually:");
     console.error("  npm install " + deps.join(" "));
-    // do not throw to keep npm install resilient
+    // keep postinstall resilient: do not throw/exit non-zero to avoid breaking npm install
   } else {
     console.log("Optional dependencies installed.");
   }
 }
 
-(function main() {
+async function main() {
   try {
-    const toInstall = optionalDeps.filter((d) => !isInstalled(d));
-    if (toInstall.length === 0) return;
+    const toInstall = [];
+    for (const dep of optionalDeps) {
+      const present = await isInstalled(dep);
+      if (!present) toInstall.push(dep);
+    }
+
+    if (toInstall.length === 0) {
+      // nothing to do
+      // console.log("All optional dependencies present.");
+      return;
+    }
 
-    // Skip auto-install in CI by default to avoid surprises; allow override.
+    // Safety: skip auto-install in CI unless explicitly forced
     if (process.env.CI && !process.env.H56_FORCE_POSTINSTALL) {
       console.log(
         "CI environment detected — skipping automatic installation of optional dependencies.",
@@ -49,7 +80,9 @@ function installDeps(deps) {
 
     installDeps(toInstall);
   } catch (err) {
+    // Keep postinstall resilient; log error but don't fail the install process.
     console.error("Postinstall check encountered an error:", err && err.message ? err.message : err);
-    // do not exit non-zero; keep postinstall resilient
   }
-})();
+}
+
+await main();

package/translate-engine/translate.js

@@ -1,18 +1,30 @@
-// translate-engine/translate.js
-// CommonJS runtime wrapper for optional dependency 'h56-translator'.
-// Exports: { translate(text, targetLang, options?) }
-
-"use strict";
+/**
+ * translate-engine/translate.js (ESM)
+ *
+ * ESM wrapper for optional dependency 'h56-translator'.
+ * Exports a single named function `translate(text, targetLang, options?)`.
+ *
+ * This file replaces the previous CommonJS wrapper and is ready for ESM usage.
+ *
+ * NOTE: If you publish this package as ESM (package.json "type": "module"),
+ * keep this file as-is. It will attempt to import 'h56-translator' and normalize shapes.
+ */
 
-function loadTranslator() {
+export async function translate(text, targetLang, options) {
   try {
-    const mod = require("h56-translator");
-    // handle shapes: named export, default export function, or default object with translate
-    if (mod && typeof mod.translate === "function") return { translate: mod.translate };
-    if (typeof mod === "function") return { translate: mod };
-    if (mod && mod.default && typeof mod.default === "function") return { translate: mod.default };
-    if (mod && mod.default && typeof mod.default.translate === "function")
-      return { translate: mod.default.translate };
+    // dynamic import of the translator package (supports ESM or CJS shapes)
+    const mod = await import("h56-translator");
+    const m = mod && (mod.default || mod); // handle default interop
+    if (typeof m.translate === "function") {
+      return await m.translate(text, targetLang, options);
+    }
+    if (typeof m === "function") {
+      // package export is a function
+      return await m(text, targetLang, options);
+    }
+    if (m && typeof m.default === "function") {
+      return await m.default(text, targetLang, options);
+    }
     throw new Error("h56-translator export shape not recognized");
   } catch (err) {
     const e = new Error(
@@ -21,15 +33,4 @@ function loadTranslator() {
     e.cause = err;
     throw e;
   }
-}
-
-/**
- * translate(text, targetLang, options?)
- * Delegates directly to h56-translator.
- */
-async function translate(text, targetLang, options) {
-  const t = loadTranslator();
-  return await t.translate(text, targetLang, options);
-}
-
-module.exports = { translate };
+}