h56-github-scrapper 1.0.4 → 1.0.6

package/main-scrapping.js

@@ -1,120 +1,76 @@
+#!/usr/bin/env node
 /**
- * main-scrapping.js (ESM)
+ * main-scrapping.js (ESM, improved)
  *
- * Converted to ECMAScript Modules (ESM) for Node.js.
- * - Preserves original logic and features (scraping, retry/backoff, spinner, CLI).
- * - Adds optional translator integration (uses translate-engine/translate.js as ESM).
- * - Uses dynamic import for third-party runtime deps after optional auto-install.
- *
- *
- * This file is ready to copy/paste. It intentionally preserves behavior and CLI options
- * while modernizing module semantics to ESM.
+ * - Fully ESM, no createRequire usage.
+ * - Does NOT auto-install dependencies. Instead it detects missing runtime deps
+ *   and prints a clear instruction to install them (safer for CI and terminals).
+ * - Keeps previous scraping, retry/backoff, translator lazy-loading, CLI and public API.
+ * - Improved export block and CLI detection.
  */
 
 import fs from "fs";
 import path from "path";
-import { spawnSync } from "child_process";
 import os from "os";
-import { createRequire } from "module";
 import readline from "readline";
 import { fileURLToPath } from "url";
 
-const require = createRequire(import.meta.url);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
-// -------------------------
-// Ensure runtime deps (async)
-// -------------------------
-async function ensureDependencies(deps = []) {
-  const missing = deps.filter((d) => {
+// Runtime packages required for operation
+const RUNTIME_PKGS = ["axios", "cheerio", "ora", "yargs"];
+
+/**
+ * Dynamically import runtime packages and return a map of package -> module.
+ * If any package is missing, behave differently depending on whether this module
+ * is being run as CLI or imported as a library:
+ *  - CLI: print a helpful message and exit(1)
+ *  - Library: throw an Error (so consumer can handle)
+ */
+async function loadRuntimes() {
+  const results = {};
+  const missing = [];
+
+  for (const name of RUNTIME_PKGS) {
     try {
-      require.resolve(d);
-      return false;
-    } catch (e) {
-      return true;
+      const mod = await import(name);
+      results[name] = mod && mod.default ? mod.default : mod;
+    } catch (err) {
+      missing.push(name);
     }
-  });
-
-  if (!missing.length) return;
-
-  console.log(
-    `Dependencies missing: ${missing.join(
-      ", "
-    )}. The script can install them automatically.`
-  );
-
-  // If running non-interactive environment, install automatically.
-  let consent = false;
-  if (process.env.CI || !process.stdin.isTTY) {
-    consent = true;
-    console.log("Non-interactive environment detected, installing automatically...");
-  } else {
-    const rl = readline.createInterface({
-      input: process.stdin,
-      output: process.stdout,
-    });
+  }
 
-    // Prompt with 10s timeout; default to yes
-    const questionPromise = new Promise((resolve) =>
-      rl.question(`Install missing dependencies now? (Y/n): `, (a) => {
-        rl.close();
-        resolve(String(a || "").trim().toLowerCase());
-      })
-    );
+  if (missing.length) {
+    const cmd = `npm install ${missing.join(" ")}`;
+    const msg =
+      `Missing runtime dependencies: ${missing.join(", ")}.\n` +
+      `Please install them before running this script:\n\n  ${cmd}\n\n` +
+      `If you are in CI and want deterministic installs, declare these dependencies explicitly in your pipeline or package.json.`;
 
-    const ans = await Promise.race([
-      questionPromise,
-      new Promise((resolve) => setTimeout(() => resolve(null), 10000)),
-    ]);
+    // Detect CLI execution (rough check)
+    const isCli = process.argv && process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
 
-    if (ans === null) {
-      consent = true;
-      console.log("\nNo answer, defaulting to install.\n");
+    if (isCli) {
+      console.error(msg);
+      process.exit(1);
     } else {
-      consent = !ans || ans === "y" || ans === "yes";
+      throw new Error(msg);
     }
   }
 
-  if (!consent) {
-    console.error("Cannot proceed without required dependencies. Exiting.");
-    process.exit(1);
-  }
-
-  // Run npm install for missing deps
-  console.log(`Installing: ${missing.join(" ")} ...`);
-  const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
-  const args = ["install", "--save", ...missing];
-  const result = spawnSync(npmCmd, args, {
-    stdio: "inherit",
-    shell: false,
-  });
-
-  if (result.error || result.status !== 0) {
-    console.error("Automatic installation failed. Please run:");
-    console.error(`  npm install ${missing.join(" ")}`);
-    process.exit(1);
-  }
-
-  console.log("Dependencies installed, continuing...");
+  return results;
 }
 
-// Ensure runtime modules (same set as original)
-await ensureDependencies(["axios", "cheerio", "ora", "yargs"]);
-
-// Dynamic import of runtime libraries (so they are loaded after potential install)
-const axiosModule = await import("axios");
-const axios = axiosModule.default || axiosModule;
-
-const cheerioModule = await import("cheerio");
-const cheerio = cheerioModule.default || cheerioModule;
-
-const oraModule = await import("ora");
-const ora = oraModule.default || oraModule;
-
-const yargsModule = await import("yargs");
-const yargs = yargsModule.default || yargsModule;
+// Load runtimes before continuing
+const runtimes = await loadRuntimes();
+const axios = runtimes["axios"];
+const cheerio = runtimes["cheerio"];
+const ora = runtimes["ora"];
+const yargs = runtimes["yargs"];
 
 // -------------------------
-// Utilities & Config
+// Utilities & Config (unchanged behavior)
 // -------------------------
 const DEFAULT_CONFIG = {
   BASE_URL: "https://github.com",
@@ -158,9 +114,8 @@ function validateUsername(username) {
 }
 
 // -------------------------
-// Translator loader (optional, lazy)
+// Translator loader (optional, lazy) - unchanged behavior
 // -------------------------
-// We keep a lazy loader so translator is imported only if requested.
 let _translatorModule = null; // module namespace
 let _translatorLoadAttempted = false;
 
@@ -169,22 +124,27 @@ async function loadTranslatorModule() {
   if (_translatorLoadAttempted) return null;
   _translatorLoadAttempted = true;
   try {
-    // prefer the package-local wrapper (ESM) if present
-    // path relative to this file
-    const wrapperPath = new URL("./translate-engine/translate.js", import.meta.url).pathname;
-    // attempt dynamic import; if module not present or h56-translator not installed, it will throw
-    const mod = await import(wrapperPath);
-    if (mod && typeof mod.translate === "function") {
-      _translatorModule = mod;
-      return _translatorModule;
+    // prefer package-local wrapper (ESM)
+    const wrapperUrl = new URL("./translate-engine/translate.js", import.meta.url).href;
+    try {
+      const mod = await import(wrapperUrl);
+      if (mod && typeof mod.translate === "function") {
+        _translatorModule = mod;
+        return _translatorModule;
+      }
+    } catch (_) {
+      // wrapper not present or failed - fallthrough to package import
     }
-    // fallback: try importing the package directly
+
+    // fallback: try importing the optional package directly
     const pkg = await import("h56-translator");
-    if (pkg && (typeof pkg.translate === "function" || typeof pkg.default === "function")) {
-      const translateFn = typeof pkg.translate === "function" ? pkg.translate : pkg.default;
+    const impl = pkg && (pkg.default || pkg);
+    if (impl && (typeof impl.translate === "function" || typeof impl === "function")) {
+      const translateFn = typeof impl.translate === "function" ? impl.translate : impl;
       _translatorModule = { translate: translateFn };
       return _translatorModule;
     }
+
     return null;
   } catch (err) {
     // translator not available — return null and let callers handle
@@ -193,7 +153,7 @@ async function loadTranslatorModule() {
 }
 
 // -------------------------
-// Scraper class
+// Scraper class (logic preserved)
 // -------------------------
 class GithubScraper {
   constructor(opts = {}) {
@@ -210,12 +170,10 @@ class GithubScraper {
       return res.data;
     } catch (err) {
       if (attempt >= this.config.MAX_RETRY) {
-        // wrap error with url info
         const e = new Error(`Failed to fetch ${url}: ${err.message}`);
         e.cause = err;
         throw e;
       }
-      // backoff
       await sleep(1000 * attempt);
       return this.requestWithRetry(url, attempt + 1);
     }
@@ -245,7 +203,6 @@ class GithubScraper {
       $('div[itemprop="description"]').text().trim() ||
       "";
 
-    // Pulled from header counters (structure may vary by locale/markup)
     const followersText = $(
       'a[href$="?tab=followers"], a[href$="?tab=followers"] .text-bold'
     )
@@ -286,7 +243,6 @@ class GithubScraper {
     while (true) {
       const url = `${this.config.BASE_URL}/${username}?page=${page}&tab=repositories`;
       const $ = await this.fetchPage(url);
-      // older layout: li[itemprop='owns'], new layout: div[id^=user-repositories-list] li
       const repoItems =
         $("li[itemprop='owns']").length > 0
           ? $("li[itemprop='owns']")
@@ -360,24 +316,6 @@ class GithubScraper {
     };
   }
 
-  /**
-   * applyTranslations(result, translateOptions)
-   *
-   * Mutates the result object by adding translated fields.
-   *
-   * translateOptions (optional) shape:
-   * {
-   *   lang: string,                 // target language code (required to perform translations)
-   *   fields?: string[],            // list of fields to translate; supported values:
-   *                                 //   'bio' (profile.bio)
-   *                                 //   'repo_descriptions' (repo.description)
-   *                                 //   'repo_names' (repo.name)
-   *                                 //   'all_repos' (alias for repo_descriptions + repo_names)
-   *                                 // default: ['bio', 'repo_descriptions']
-   *   perRepoDelay?: number,        // ms delay between repo translations (default 120)
-   *   failOnMissing?: boolean       // if true, throw when translator is not available (default false)
-   * }
-   */
   async applyTranslations(result, translateOptions = {}) {
     if (!translateOptions || !translateOptions.lang) return result;
     const opts = {
@@ -387,7 +325,6 @@ class GithubScraper {
       ...translateOptions,
     };
 
-    // normalize fields
     const fields = new Set();
     for (const f of opts.fields) {
       if (f === "all_repos") {
@@ -398,7 +335,6 @@ class GithubScraper {
       }
     }
 
-    // translator function must exist (lazy load)
     const mod = await loadTranslatorModule();
     if (!mod || typeof mod.translate !== "function") {
       const msg =
@@ -408,7 +344,6 @@ class GithubScraper {
         e.code = "TRANSLATOR_MISSING";
         throw e;
       } else {
-        // attach a note and skip translations
         result._translation_note = {
           skipped: true,
           reason: msg,
@@ -418,7 +353,6 @@ class GithubScraper {
     }
     const tfn = mod.translate;
 
-    // perform profile translation
     try {
       if (fields.has("bio") && result.profile && result.profile.bio) {
         try {
@@ -433,11 +367,9 @@ class GithubScraper {
         }
       }
     } catch (e) {
-      // defensive: any translator error should not break the main flow
       result._translation_profile_error = e && e.message ? e.message : String(e);
     }
 
-    // perform repository translations sequentially (safer)
     if (Array.isArray(result.repos) && result.repos.length > 0) {
       for (const repo of result.repos) {
         try {
@@ -466,7 +398,6 @@ class GithubScraper {
             }
           }
         } catch (e) {
-          // attach per-repo error but continue
           repo.translation_internal_error = e && e.message ? e.message : String(e);
         }
         await sleep(opts.perRepoDelay);
@@ -476,19 +407,6 @@ class GithubScraper {
     return result;
   }
 
-  // high-level helper
-  /**
-   * scrapeUser(username, opts)
-   *
-   * opts:
-   *   spinner: boolean (default true)
-   *   translate: {
-   *     lang: 'en',                // target language code (required to enable translations)
-   *     fields: ['bio','repo_descriptions'], // which fields to translate
-   *     perRepoDelay: 120,         // ms
-   *     failOnMissing: false       // if true, throw when translator missing
-   *   }
-   */
   async scrapeUser(username, opts = {}) {
     if (!validateUsername(username)) {
       const e = new Error("Invalid GitHub username format");
@@ -507,18 +425,15 @@ class GithubScraper {
 
       let result = { profile, repos, stats };
 
-      // If translation options provided, attempt to apply translations.
       if (opts.translate && opts.translate.lang) {
         if (spinner) spinner.text = "Applying translations...";
         try {
           result = await this.applyTranslations(result, opts.translate);
         } catch (e) {
-          // translator errors: if failOnMissing requested, rethrow; otherwise attach note
           if (opts.translate && opts.translate.failOnMissing) {
             if (spinner) spinner.fail("Failed");
             throw e;
           } else {
-            // attach note and continue
             result._translation_error = e && e.message ? e.message : String(e);
           }
         }
@@ -531,7 +446,6 @@ class GithubScraper {
     }
   }
 
-  // CLI pretty print
   static printResult(profile, stats, repos = []) {
     console.log("\n========== GITHUB ACCOUNT ==========\n");
     console.log("Username  :", profile.username);
@@ -567,11 +481,10 @@ class GithubScraper {
 }
 
 // -------------------------
-// Exports (for usage as package/module)
+// Exports (clean ESM)
 // -------------------------
 const defaultScraper = new GithubScraper();
 
-// h56translate helper: lazy-call translator module when invoked
 export async function h56translate(text, targetLang, options) {
   const mod = await loadTranslatorModule();
   if (!mod || typeof mod.translate !== "function") {
@@ -582,35 +495,26 @@ export async function h56translate(text, targetLang, options) {
   return await mod.translate(text, targetLang, options);
 }
 
-export {
-  GithubScraper,
-  defaultScraper,
-  // convenience wrappers
-  // note: keep same signatures as before (username, opts)
-  async function scrapeProfile(username, opts) {
-    return defaultScraper.scrapeProfile(username, opts);
-  },
-  async function scrapeRepos(username, opts) {
-    return defaultScraper.scrapeRepos(username, opts);
-  },
-  async function scrapeUser(username, opts) {
-    return defaultScraper.scrapeUser(username, opts);
-  },
-  function calculateStats(repos) {
-    return defaultScraper.calculateStats(repos);
-  },
-  GithubScraper.printResult as printResult,
-};
+export { GithubScraper, defaultScraper };
 
-// The above export block uses named exports compatible with ESM consumers.
-// For backwards compatibility with consumers expecting a default CommonJS export,
-// also write a small CJS-compatible default export file (optional) when packaging.
+export async function scrapeProfile(username) {
+  return defaultScraper.scrapeProfile(username);
+}
+export async function scrapeRepos(username) {
+  return defaultScraper.scrapeRepos(username);
+}
+export async function scrapeUser(username, opts) {
+  return defaultScraper.scrapeUser(username, opts);
+}
+export function calculateStats(repos) {
+  return defaultScraper.calculateStats(repos);
+}
+export const printResult = GithubScraper.printResult;
 
 // -------------------------
 // CLI behavior when run directly
 // -------------------------
-const __filename = fileURLToPath(import.meta.url);
-if (process.argv[1] === __filename) {
+if (fileURLToPath(import.meta.url) === process.argv[1]) {
   (async () => {
     // build argv using yargs (same API as original)
     const argv = yargs(process.argv.slice(2))
@@ -649,7 +553,6 @@ if (process.argv[1] === __filename) {
       process.exit(1);
     }
 
-    // build translate options if requested
     const translateOpt = argv.lang
       ? {
           lang: argv.lang,
@@ -678,7 +581,6 @@ if (process.argv[1] === __filename) {
       } else {
         GithubScraper.printResult(result.profile, result.stats, result.repos);
         if (argv.output) {
-          // also write JSON file if requested
           fs.writeFileSync(path.resolve(argv.output), JSON.stringify(result, null, 2) + os.EOL, "utf8");
           console.log("Written JSON to", argv.output);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "h56-github-scrapper",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "GitHub user scraper",
   "type": "module",
   "main": "main-scrapping.js",
@@ -18,8 +18,5 @@
   },
   "optionalDependencies": {
     "h56-translator": "^1.0.0"
-  },
-  "scripts": {
-    "postinstall": "node ./scripts/ensure-external-deps.js"
   }
 }

package/readme.md

@@ -274,14 +274,6 @@ console.log(r.translatedText);
 - `h56-translator` adalah dependency opsional. Paket menyediakan:
   - `translate-engine/translate.ts` (typed wrapper) untuk development/TS.
   - `translate-engine/translate.js` (CJS wrapper) untuk runtime require().
-  - `scripts/ensure-external-deps.js` — postinstall helper yang berusaha memasang `h56-translator` jika tidak ada, kecuali di CI (safety).
-- Jika Anda menginginkan pemasangan otomatis di CI:
-  - jalankan: `H56_FORCE_POSTINSTALL=1 npm install`
-- Jika translator tidak terpasang:
-  - `h56translate(...)` akan melempar Error informatif.
-  - `scrapeUser(..., { translate: {...} })` akan:
-    - menambahkan `_translation_note` dan melanjutkan (default), atau
-    - melempar error jika `failOnMissing: true` disetel.
 
 ---
 

package/script/ensure-external-deps.js

@@ -1,57 +0,0 @@
-#!/usr/bin/env node
-/**
- * scripts/ensure-external-deps.js (ESM)
- *
- * Postinstall helper (ESM). Attempts to install optional dependencies if missing.**/
-
-import { spawnSync } from "child_process";
-import { createRequire } from "module";
-
-const require = createRequire(import.meta.url);
-
-const optionalDeps = ["h56-translator"];
-
-function isInstalled(name) {
-  try {
-    require.resolve(name);
-    return true;
-  } catch (_) {
-    return false;
-  }
-}
-
-function installDeps(deps) {
-  if (!deps.length) return;
-  const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
-  const args = ["install", "--no-audit", "--no-fund", "--save", ...deps];
-  console.log("Installing optional dependencies:", deps.join(", "));
-  const res = spawnSync(npmCmd, args, { stdio: "inherit" });
-  if (res.error || res.status !== 0) {
-    console.error("Failed to install optional dependencies. You can run manually:");
-    console.error("  npm install " + deps.join(" "));
-    // do not throw to keep npm install resilient
-  } else {
-    console.log("Optional dependencies installed.");
-  }
-}
-
-(function main() {
-  try {
-    const toInstall = optionalDeps.filter((d) => !isInstalled(d));
-    if (toInstall.length === 0) return;
-
-    // Skip auto-install in CI by default to avoid surprises; allow override.
-    if (process.env.CI && !process.env.H56_FORCE_POSTINSTALL) {
-      console.log(
-        "CI environment detected — skipping automatic installation of optional dependencies.",
-        "Set H56_FORCE_POSTINSTALL=1 to force installation in CI."
-      );
-      return;
-    }
-
-    installDeps(toInstall);
-  } catch (err) {
-    console.error("Postinstall check encountered an error:", err && err.message ? err.message : err);
-    // do not exit non-zero; keep postinstall resilient
-  }
-})();