h56-github-scrapper 1.0.0

package/main-scrapping.js

@@ -0,0 +1,684 @@
+/**
+ * main-scrapping.js
+ *
+ * NPM package style single-file scraper for GitHub user data.
+ * Package name suggestion: h56-github-scrapper
+ *
+ * Features:
+ * - Exports programmatic functions: scrapeProfile, scrapeRepos, scrapeUser, calculateStats
+ * - CLI entry when run directly: node main-scrapping.js <username> [--json] [--output=file]
+ * - Optional translator integration (h56-translator) with selectable fields to translate
+ * - Automatic install of missing npm runtime dependencies (asks for consent when needed)
+ * - Robust retry/backoff, polite scraping delay, spinner (ora) fallback
+ * - Well-structured results and JSON output support
+ *
+ * NOTE:
+ * - The recommended (proper) approach for publishing as an npm package is to list dependencies
+ *   in package.json. The runtime auto-installer is implemented as a convenience only.
+ * - Scraping HTML may break if GitHub changes markup. Consider using GitHub API for production.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { spawnSync } = require("child_process");
+const os = require("os");
+
+// -------------------------
+// Ensure runtime deps
+// -------------------------
+function ensureDependencies(deps = []) {
+  const missing = deps.filter((d) => {
+    try {
+      require.resolve(d);
+      return false;
+    } catch (e) {
+      return true;
+    }
+  });
+
+  if (!missing.length) return;
+
+  console.log(
+    `Dependencies missing: ${missing.join(
+      ", "
+    )}. The script can install them automatically.`
+  );
+
+  // If running non-interactive environment, install automatically.
+  let consent = false;
+  if (process.env.CI || !process.stdin.isTTY) {
+    consent = true;
+    console.log("Non-interactive environment detected, installing automatically...");
+  } else {
+    const rl = require("readline").createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+
+    const answer = new Promise((resolve) =>
+      rl.question(`Install missing dependencies now? (Y/n): `, (a) => {
+        rl.close();
+        resolve(a.trim().toLowerCase());
+      })
+    );
+
+    // synchronous wait via spawnSync is not used here; use await-like pattern:
+    // To keep file runnable without top-level await, use de-facto blocking by reading from stdin synchronously not available.
+    // We'll perform synchronous prompt by checking answer resolved synchronously above (it won't), so instead do a simpler default:
+    // If user doesn't answer quickly, default to yes after 10s.
+    // Implementing a small blocking wait using child_process to call 'bash -c read -t 10' is platform-dependent.
+    // For simplicity here, assume consent if user presses Enter quickly — we'll read synchronously via question with callback, then block via a small busy loop until resolved.
+    let resolved = false;
+    let ansValue = "";
+    answer.then((v) => {
+      resolved = true;
+      ansValue = v;
+    });
+
+    // Wait (busy-loop) until resolved — acceptable for a small prompt in CLI tool.
+    const waitUntil = Date.now() + 10000; // 10s timeout
+    while (!resolved && Date.now() < waitUntil) {
+      // small sleep
+      const start = Date.now();
+      while (Date.now() - start < 50) {}
+    }
+
+    if (!resolved) {
+      // default to yes
+      consent = true;
+      console.log("\nNo answer, defaulting to install.\n");
+    } else {
+      consent = !ansValue || ansValue === "y" || ansValue === "yes";
+    }
+  }
+
+  if (!consent) {
+    console.error("Cannot proceed without required dependencies. Exiting.");
+    process.exit(1);
+  }
+
+  // Run npm install for missing deps
+  console.log(`Installing: ${missing.join(" ")} ...`);
+  const args = ["install", "--save", ...missing];
+  const result = spawnSync(process.platform === "win32" ? "npm.cmd" : "npm", args, {
+    stdio: "inherit",
+    shell: false,
+  });
+
+  if (result.error || result.status !== 0) {
+    console.error("Automatic installation failed. Please run:");
+    console.error(`  npm install ${missing.join(" ")}`);
+    process.exit(1);
+  }
+
+  console.log("Dependencies installed, continuing...");
+}
+
+// required runtime modules
+ensureDependencies(["axios", "cheerio", "ora", "yargs"]);
+
+// now require installed modules
+const axios = require("axios");
+const cheerio = require("cheerio");
+const ora = require("ora");
+const yargs = require("yargs");
+
+// -------------------------
+// Utilities & Config
+// -------------------------
+const DEFAULT_CONFIG = {
+  BASE_URL: "https://github.com",
+  REQUEST_TIMEOUT: 15000,
+  MAX_RETRY: 3,
+  SCRAPE_DELAY: 400, // ms between page fetches
+  USER_AGENT:
+    "Mozilla/5.0 (compatible; h56-github-scrapper/1.0; +https://github.com/)",
+  PER_PAGE: 30,
+};
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function parseGithubNumber(text = "") {
+  if (!text) return 0;
+  text = String(text).toLowerCase().replace(/\s+/g, "").replace(/,/g, "");
+  // common cases: 1k 1.2k 1.2m 1,200 1200
+  const m = text.match(/^([\d,.]*\d)([km])?$/);
+  if (!m) {
+    const n = Number(text.replace(/[^0-9.]/g, ""));
+    return Number.isFinite(n) ? Math.round(n) : 0;
+  }
+  const val = parseFloat(m[1]);
+  if (m[2] === "k") return Math.round(val * 1000);
+  if (m[2] === "m") return Math.round(val * 1000000);
+  return Math.round(val);
+}
+
+function formatNumber(num) {
+  try {
+    return new Intl.NumberFormat("en-US").format(num);
+  } catch (e) {
+    return String(num);
+  }
+}
+
+function validateUsername(username) {
+  return /^[a-zA-Z0-9-]{1,39}$/.test(username);
+}
+
+// -------------------------
+// Translator loader (optional)
+// -------------------------
+// Provide a safe, synchronous attempt to load the local CommonJS wrapper
+// translate-engine/translate.js. If not present, translatorFn will be null.
+// Consumers of translation should handle null translatorFn.
+let translatorFn = null;
+try {
+  // prefer the local wrapper shipped with the package (does dynamic require to h56-translator)
+  const tmod = require("./translate-engine/translate");
+  if (tmod && typeof tmod.translate === "function") {
+    translatorFn = tmod.translate;
+  }
+} catch (e) {
+  // not installed / wrapper not available; keep translatorFn null
+  translatorFn = null;
+}
+
+// -------------------------
+// Scraper class
+// -------------------------
+class GithubScraper {
+  constructor(opts = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...(opts || {}) };
+    this.axios = axios.create({
+      timeout: this.config.REQUEST_TIMEOUT,
+      headers: { "User-Agent": this.config.USER_AGENT, Accept: "text/html" },
+    });
+  }
+
+  async requestWithRetry(url, attempt = 1) {
+    try {
+      const res = await this.axios.get(url);
+      return res.data;
+    } catch (err) {
+      if (attempt >= this.config.MAX_RETRY) {
+        // wrap error with url info
+        const e = new Error(`Failed to fetch ${url}: ${err.message}`);
+        e.cause = err;
+        throw e;
+      }
+      // backoff
+      await sleep(1000 * attempt);
+      return this.requestWithRetry(url, attempt + 1);
+    }
+  }
+
+  async fetchPage(url) {
+    const html = await this.requestWithRetry(url);
+    return cheerio.load(html);
+  }
+
+  async scrapeProfile(username) {
+    const $ = await this.fetchPage(`${this.config.BASE_URL}/${username}`);
+
+    if ($("title").text().includes("Not Found")) {
+      const e = new Error("Username not found");
+      e.code = "NOT_FOUND";
+      throw e;
+    }
+
+    const name =
+      $('h1[class*="vcard-names"] .p-name').text().trim() ||
+      $(".p-name.vcard-fullname").text().trim() ||
+      "";
+
+    const bio =
+      $('div[class*="p-note"]').text().trim() ||
+      $('div[itemprop="description"]').text().trim() ||
+      "";
+
+    // Pulled from header counters (structure may vary by locale/markup)
+    const followersText = $(
+      'a[href$="?tab=followers"], a[href$="?tab=followers"] .text-bold'
+    )
+      .first()
+      .text()
+      .trim();
+    const followingText = $(
+      'a[href$="?tab=following"], a[href$="?tab=following"] .text-bold'
+    )
+      .first()
+      .text()
+      .trim();
+    const reposText = $(
+      'a[href$="?tab=repositories"], a[href$="?tab=repositories"] .Counter'
+    )
+      .first()
+      .text()
+      .trim();
+
+    const followers = parseGithubNumber(followersText);
+    const following = parseGithubNumber(followingText);
+    const public_repos = parseGithubNumber(reposText);
+
+    return {
+      username,
+      name,
+      bio,
+      followers,
+      following,
+      public_repos,
+      profile_url: `${this.config.BASE_URL}/${username}`,
+    };
+  }
+
+  async scrapeRepos(username) {
+    const repos = [];
+    let page = 1;
+    while (true) {
+      const url = `${this.config.BASE_URL}/${username}?page=${page}&tab=repositories`;
+      const $ = await this.fetchPage(url);
+      // older layout: li[itemprop='owns'], new layout: div[id^=user-repositories-list] li
+      const repoItems =
+        $("li[itemprop='owns']").length > 0
+          ? $("li[itemprop='owns']")
+          : $("#user-repositories-list li");
+
+      if (!repoItems.length) break;
+
+      repoItems.each((_, el) => {
+        const el$ = $(el);
+        const repoName =
+          el$.find("a[itemprop='name codeRepository']").text().trim() ||
+          el$.find("h3 a").text().trim();
+
+        const starText =
+          el$.find("a[href$='/stargazers']").text().trim() ||
+          el$.find("svg[aria-label='star'] + span").text().trim();
+        const forkText =
+          el$.find("a[href$='/network/members']").text().trim() ||
+          el$.find("svg[aria-label='fork'] + span").text().trim();
+
+        const language =
+          el$.find("[itemprop='programmingLanguage']").text().trim() ||
+          el$.find(".repo-language-color + span").text().trim() ||
+          "Unknown";
+
+        const description =
+          el$.find("p[itemprop='description']").text().trim() ||
+          el$.find("p.col-9").text().trim() ||
+          "";
+
+        const updated = el$.find("relative-time").attr("datetime") || "";
+
+        repos.push({
+          name: repoName,
+          description,
+          stars: parseGithubNumber(starText),
+          forks: parseGithubNumber(forkText),
+          language: language || "Unknown",
+          updated_at: updated,
+        });
+      });
+
+      page++;
+      await sleep(this.config.SCRAPE_DELAY);
+    }
+
+    return repos;
+  }
+
+  calculateStats(repos) {
+    const languageMap = {};
+    let totalStars = 0;
+    let totalForks = 0;
+
+    repos.forEach((r) => {
+      totalStars += r.stars || 0;
+      totalForks += r.forks || 0;
+      const lang = r.language || "Unknown";
+      languageMap[lang] = (languageMap[lang] || 0) + 1;
+    });
+
+    const top_languages = Object.entries(languageMap)
+      .sort((a, b) => b[1] - a[1])
+      .map(([language, count]) => ({ language, repos: count }));
+
+    return {
+      total_repositories: repos.length,
+      total_stars: totalStars,
+      total_forks: totalForks,
+      top_languages,
+    };
+  }
+
+  /**
+   * applyTranslations(result, translateOptions)
+   *
+   * Mutates the result object by adding translated fields.
+   *
+   * translateOptions (optional) shape:
+   * {
+   *   lang: string,                 // target language code (required to perform translations)
+   *   fields?: string[],            // list of fields to translate; supported values:
+   *                                 //   'bio' (profile.bio)
+   *                                 //   'repo_descriptions' (repo.description)
+   *                                 //   'repo_names' (repo.name)
+   *                                 //   'all_repos' (alias for repo_descriptions + repo_names)
+   *                                 // default: ['bio', 'repo_descriptions']
+   *   perRepoDelay?: number,        // ms delay between repo translations (default 120)
+   *   failOnMissing?: boolean       // if true, throw when translator is not available (default false)
+   * }
+   */
+  async applyTranslations(result, translateOptions = {}) {
+    if (!translateOptions || !translateOptions.lang) return result;
+    const opts = {
+      fields: ["bio", "repo_descriptions"],
+      perRepoDelay: 120,
+      failOnMissing: false,
+      ...translateOptions,
+    };
+
+    // normalize fields
+    const fields = new Set();
+    for (const f of opts.fields) {
+      if (f === "all_repos") {
+        fields.add("repo_descriptions");
+        fields.add("repo_names");
+      } else {
+        fields.add(f);
+      }
+    }
+
+    // translator function must exist
+    const tfn = translatorFn;
+    if (!tfn) {
+      const msg =
+        "Optional translator is not available. Install 'h56-translator' (and ensure translate-engine/translate.js is present) to enable translations.";
+      if (opts.failOnMissing) {
+        const e = new Error(msg);
+        e.code = "TRANSLATOR_MISSING";
+        throw e;
+      } else {
+        // attach a note and skip translations
+        result._translation_note = {
+          skipped: true,
+          reason: msg,
+        };
+        return result;
+      }
+    }
+
+    // perform profile translation
+    try {
+      if (fields.has("bio") && result.profile && result.profile.bio) {
+        try {
+          const t = await tfn(result.profile.bio, opts.lang, { timeoutMs: 5000 });
+          if (t && typeof t.translatedText === "string") {
+            result.profile.bio_translated = t.translatedText;
+            result.profile.bio_source_lang = t.sourceLang || null;
+            result.profile.bio_translation_meta = { serviceStatus: t.serviceStatus || "ok" };
+          }
+        } catch (e) {
+          result.profile.bio_translation_error = e && e.message ? e.message : String(e);
+        }
+      }
+    } catch (e) {
+      // defensive: any translator error should not break the main flow
+      result._translation_profile_error = e && e.message ? e.message : String(e);
+    }
+
+    // perform repository translations sequentially (safer)
+    if (Array.isArray(result.repos) && result.repos.length > 0) {
+      for (const repo of result.repos) {
+        try {
+          if (fields.has("repo_descriptions") && repo.description) {
+            try {
+              const t = await tfn(repo.description, opts.lang, { timeoutMs: 5000 });
+              if (t && typeof t.translatedText === "string") {
+                repo.description_translated = t.translatedText;
+                repo.description_source_lang = t.sourceLang || null;
+                repo.description_translation_meta = { serviceStatus: t.serviceStatus || "ok" };
+              }
+            } catch (e) {
+              repo.description_translation_error = e && e.message ? e.message : String(e);
+            }
+          }
+          if (fields.has("repo_names") && repo.name) {
+            try {
+              const t2 = await tfn(repo.name, opts.lang, { timeoutMs: 3000 });
+              if (t2 && typeof t2.translatedText === "string") {
+                repo.name_translated = t2.translatedText;
+                repo.name_source_lang = t2.sourceLang || null;
+                repo.name_translation_meta = { serviceStatus: t2.serviceStatus || "ok" };
+              }
+            } catch (e) {
+              repo.name_translation_error = e && e.message ? e.message : String(e);
+            }
+          }
+        } catch (e) {
+          // attach per-repo error but continue
+          repo.translation_internal_error = e && e.message ? e.message : String(e);
+        }
+        await sleep(opts.perRepoDelay);
+      }
+    }
+
+    return result;
+  }
+
+  // high-level helper
+  /**
+   * scrapeUser(username, opts)
+   *
+   * opts:
+   *   spinner: boolean (default true)
+   *   translate: {
+   *     lang: 'en',                // target language code (required to enable translations)
+   *     fields: ['bio','repo_descriptions'], // which fields to translate
+   *     perRepoDelay: 120,         // ms
+   *     failOnMissing: false       // if true, throw when translator missing
+   *   }
+   */
+  async scrapeUser(username, opts = {}) {
+    if (!validateUsername(username)) {
+      const e = new Error("Invalid GitHub username format");
+      e.code = "INVALID_USERNAME";
+      throw e;
+    }
+
+    const spinner = (opts.spinner !== false) ? ora({ text: `Scraping ${username}...`, spinner: "dots" }).start() : null;
+
+    try {
+      const profile = await this.scrapeProfile(username);
+      if (spinner) spinner.text = "Fetching repositories...";
+      const repos = await this.scrapeRepos(username);
+      if (spinner) spinner.succeed("Scraping completed");
+      const stats = this.calculateStats(repos);
+
+      let result = { profile, repos, stats };
+
+      // If translation options provided, attempt to apply translations.
+      if (opts.translate && opts.translate.lang) {
+        if (spinner) spinner.text = "Applying translations...";
+        try {
+          result = await this.applyTranslations(result, opts.translate);
+        } catch (e) {
+          // translator errors: if failOnMissing requested, rethrow; otherwise attach note
+          if (opts.translate && opts.translate.failOnMissing) {
+            if (spinner) spinner.fail("Failed");
+            throw e;
+          } else {
+            // attach note and continue
+            result._translation_error = e && e.message ? e.message : String(e);
+          }
+        }
+      }
+
+      return result;
+    } catch (err) {
+      if (spinner) spinner.fail("Failed");
+      throw err;
+    }
+  }
+
+  // CLI pretty print
+  static printResult(profile, stats, repos = []) {
+    console.log("\n========== GITHUB ACCOUNT ==========\n");
+    console.log("Username  :", profile.username);
+    console.log("Name      :", profile.name || "-");
+    console.log("Bio       :", profile.bio || "-");
+    if (profile.bio_translated) console.log("Bio (translated):", profile.bio_translated);
+    console.log("Followers :", formatNumber(profile.followers));
+    console.log("Following :", formatNumber(profile.following));
+    console.log("Repos     :", formatNumber(profile.public_repos));
+    console.log("Profile   :", profile.profile_url);
+
+    console.log("\n------- Repository Statistics -------\n");
+    console.log("Total Repository :", formatNumber(stats.total_repositories));
+    console.log("Total Stars      :", formatNumber(stats.total_stars));
+    console.log("Total Forks      :", formatNumber(stats.total_forks));
+
+    console.log("\nTop Languages:");
+    stats.top_languages.forEach((l) => console.log(`• ${l.language} (${l.repos})`));
+
+    if (repos && repos.length) {
+      console.log("\nSample repositories:");
+      repos.slice(0, 10).forEach((r) =>
+        console.log(
+          `- ${r.name} (${r.language}) ★${formatNumber(r.stars)} Forks:${formatNumber(
+            r.forks
+          )}${r.description_translated ? `\n    → ${r.description_translated}` : ""}`
+        )
+      );
+    }
+
+    console.log("\n====================================\n");
+  }
+}
+
+// -------------------------
+// Exports (for npm usage)
+// -------------------------
+const defaultScraper = new GithubScraper();
+
+// Export translator helper (attempt to load; provide helpful fallback if not available)
+let h56translate;
+try {
+  const tmod = require("./translate-engine/translate");
+  if (tmod && typeof tmod.translate === "function") {
+    h56translate = tmod.translate;
+  } else {
+    h56translate = async function () {
+      throw new Error(
+        "Optional translator module loaded but export shape is not recognized. Ensure 'h56-translator' is installed and compatible."
+      );
+    };
+  }
+} catch (e) {
+  h56translate = async function () {
+    throw new Error(
+      "Optional translator is not available. Install it with `npm install h56-translator` or run `npm install` to trigger postinstall."
+    );
+  };
+}
+
+module.exports = {
+  GithubScraper,
+  defaultScraper,
+  scrapeProfile: (username, opts) => defaultScraper.scrapeProfile(username, opts),
+  scrapeRepos: (username, opts) => defaultScraper.scrapeRepos(username, opts),
+  scrapeUser: (username, opts) => defaultScraper.scrapeUser(username, opts),
+  calculateStats: (repos) => defaultScraper.calculateStats(repos),
+  printResult: GithubScraper.printResult,
+  // new optional helper: h56translate(text, targetLang, options?)
+  h56translate,
+};
+
+// -------------------------
+// CLI behavior when run directly
+// -------------------------
+if (require.main === module) {
+  (async () => {
+    const argv = yargs(process.argv.slice(2))
+      .usage("Usage: $0 <username> [options]")
+      .option("json", {
+        alias: "j",
+        type: "boolean",
+        description: "Output raw JSON",
+      })
+      .option("output", {
+        alias: "o",
+        type: "string",
+        description: "Write JSON output to file",
+      })
+      .option("no-spinner", {
+        type: "boolean",
+        description: "Disable spinner output",
+      })
+      .option("lang", {
+        alias: "l",
+        type: "string",
+        description: "Optional: translate selected text fields to this language (e.g. en, id)",
+      })
+      .option("translate-fields", {
+        type: "string",
+        description:
+          "Comma-separated fields to translate (bio,repo_descriptions,repo_names,all_repos). Default: bio,repo_descriptions",
+      })
+      .demandCommand(1, "Github username is required")
+      .help().argv;
+
+    const username = argv._[0];
+
+    if (!validateUsername(username)) {
+      console.error("Invalid GitHub username format.");
+      process.exit(1);
+    }
+
+    // build translate options if requested
+    const translateOpt = argv.lang
+      ? {
+          lang: argv.lang,
+          fields: argv["translate-fields"]
+            ? argv["translate-fields"].split(",").map((s) => s.trim())
+            : undefined,
+          perRepoDelay: 120,
+          failOnMissing: false,
+        }
+      : undefined;
+
+    try {
+      const result = await defaultScraper.scrapeUser(username, {
+        spinner: !argv["no-spinner"],
+        translate: translateOpt,
+      });
+
+      if (argv.json) {
+        const out = JSON.stringify(result, null, 2);
+        if (argv.output) {
+          fs.writeFileSync(path.resolve(argv.output), out + os.EOL, "utf8");
+          console.log("Written JSON to", argv.output);
+        } else {
+          console.log(out);
+        }
+      } else {
+        GithubScraper.printResult(result.profile, result.stats, result.repos);
+        if (argv.output) {
+          // also write JSON file if requested
+          fs.writeFileSync(path.resolve(argv.output), JSON.stringify(result, null, 2) + os.EOL, "utf8");
+          console.log("Written JSON to", argv.output);
+        }
+      }
+
+      process.exit(0);
+    } catch (err) {
+      console.error("Error:", err.message || err);
+      if (err.cause && err.cause.message) {
+        console.error("Cause:", err.cause.message);
+      }
+      process.exit(1);
+    }
+  })();
+}

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "h56-github-scrapper",
+  "version": "1.0.0",
+  "description": "GitHub user scraper (programmatic + CLI)",
+  "main": "main-scrapping.js",
+  "bin": {
+    "h56-github-scrapper": "main-scrapping.js"
+  },
+  "keywords": ["github","scraper","scraping"],
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.0.0",
+    "cheerio": "^1.0.0-rc.12",
+    "ora": "^6.0.0",
+    "yargs": "^17.0.0"
+  },
+  "optionalDependencies": {
+    "h56-translator": "^1.0.0"
+  },
+  "scripts": {
+    "postinstall": "node ./scripts/ensure-external-deps.js"
+  }
+}

package/readme.md

@@ -0,0 +1,320 @@
+# h56-github-scrapper
+
+[![npm version](https://img.shields.io/npm/v/h56-github-scrapper.svg)](https://www.npmjs.com/package/h56-github-scrapper)
+[![Downloads/month](https://img.shields.io/npm/dm/h56-github-scrapper.svg)](https://www.npmjs.com/package/h56-github-scrapper)
+[![Node](https://img.shields.io/badge/node-%3E%3D16-brightgreen.svg?logo=node.js)](https://nodejs.org/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
+[![TypeScript friendly](https://img.shields.io/badge/types-TypeScript-blue.svg?logo=typescript)](https://www.typescriptlang.org/)
+[![Translator: optional](https://img.shields.io/badge/translator-h56--translator-lightgrey.svg?logo=googletranslate)](https://www.npmjs.com/package/h56-translator)
+
+Ringkasan: h56-github-scrapper adalah paket Node.js ringan untuk mengambil (scrape) informasi profil publik dan repositori pengguna GitHub. Paket ini berfungsi sebagai CLI dan juga dapat diimpor sebagai library programatik. Versi ini menambahkan integrasi opsional dengan layanan terjemahan (`h56-translator`) sehingga Anda dapat memilih teks mana yang ingin diterjemahkan pada output (bio, nama repo, deskripsi repo, dsb).
+
+> Peringatan: paket melakukan scraping HTML publik GitHub. Struktur HTML dapat berubah sewaktu-waktu — untuk produksi/skala besar gunakan GitHub REST API (dengan autentikasi). Selalu patuhi Terms of Service GitHub dan etika scraping.
+
+---
+
+## Daftar isi
+
+- [Fitur utama](#fitur-utama)  
+- [Badge & status](#badge--status)  
+- [Persyaratan & instalasi](#persyaratan--instalasi)  
+- [Quick start — CLI](#quick-start--cli)  
+- [Opsi terjemahan (CLI & programatik)](#opsi-terjemahan-cli--programatik)  
+- [API Reference (singkat) — TypeScript interfaces](#api-reference-singkat---typescript-interfaces)  
+- [Contoh penggunaan (CommonJS / ESM / TS)](#contoh-penggunaan-commonjs--esm--ts)  
+- [Behavior translator opsional & postinstall](#behavior-translator-opsional--postinstall)  
+- [Best practices & etika scraping](#best-practices--etika-scraping)  
+- [Troubleshooting](#troubleshooting)  
+- [Contributing & changelog singkat](#contributing--changelog-singkat)  
+- [License](#license)
+
+---
+
+## Fitur utama
+
+- Ambil data profil publik: username, nama, bio, followers, following, jumlah repo publik, profile_url.
+- Ambil daftar repositori publik: name, description, language, stars, forks, updated_at.
+- Hitung statistik agregat: total_repositories, total_stars, total_forks, top_languages.
+- CLI interaktif + opsi JSON output.
+- API programatik: `scrapeUser`, `scrapeProfile`, `scrapeRepos`, `calculateStats`, `GithubScraper` class.
+- Integrasi terjemahan opsional via `h56-translator` (wrapper tersedia: `translate-engine/translate.(ts|js)`).
+- Retry/backoff, polite delay antar-request, spinner (ora) untuk UX.
+
+---
+
+## Badge & status
+
+- npm package: lihat badge versi & download di bagian atas.
+- Node: target minimum Node.js >= 16.
+- License: MIT.
+- Translator: opsi integrasi ditandai sebagai optional; install manual `npm install h56-translator` untuk mengaktifkan fitur terjemahan.
+
+---
+
+## Persyaratan & instalasi
+
+- Node.js >= 16.x direkomendasikan.
+- Instal dari npm:
+
+```bash
+npm install h56-github-scrapper
+# atau
+yarn add h56-github-scrapper
+```
+
+Jika Anda ingin menggunakan fitur terjemahan, pasang package terjemahan opsional:
+
+```bash
+npm install h56-translator
+```
+
+Catatan: paket menyediakan skrip `postinstall` yang berusaha memasang `h56-translator` secara otomatis kecuali di environment CI. Untuk memaksa install di CI gunakan:
+
+```bash
+H56_FORCE_POSTINSTALL=1 npm install
+```
+
+Namun untuk determinisme CI/CD sebaiknya deklarasikan dependency secara eksplisit di pipeline Anda.
+
+---
+
+## Quick start — CLI
+
+Sintaks dasar:
+
+```bash
+node main-scrapping.js <username> [--json] [--output=path] [--no-spinner]
+```
+
+Contoh:
+
+```bash
+# Ringkasan human readable
+node main-scrapping.js HASYIM56
+
+# Output JSON ke STDOUT
+node main-scrapping.js HASYIM56 --json
+
+# Output JSON ke file
+node main-scrapping.js HASYIM56 --json --output=HASYIM56.json
+```
+
+### Opsi terjemahan CLI (baru)
+
+- `--lang, -l <code>` — target bahasa (mis. `en`, `id`, `fr`)
+- `--translate-fields <comma-separated>` — fields yang ingin diterjemahkan: `bio`, `repo_descriptions`, `repo_names`, `all_repos`  
+  Default: `bio,repo_descriptions`
+- `--no-spinner` — non-aktifkan spinner (berguna pada CI)
+- `--json` — output JSON
+
+Contoh:
+
+```bash
+node main-scrapping.js HASYIM56 --lang=en --translate-fields=bio,repo_descriptions --json --output=HASYIM56-en.json
+```
+
+Jika translator tidak terpasang, CLI tetap berjalan dan akan menambahkan `_translation_note` pada hasil JSON (default fail-safe). Untuk membuat proses gagal ketika translator tidak ada, gunakan opsi programatik `failOnMissing: true` dengan `scrapeUser`.
+
+---
+
+## Opsi terjemahan (CLI & programatik)
+
+Terjemahan bersifat opsional dan dikontrol melalui:
+
+- CLI flags (`--lang`, `--translate-fields`)
+- Programatik: `scrapeUser(username, { translate: { lang, fields, perRepoDelay, failOnMissing } })`
+- Helper langsung: `h56translate(text, targetLang, options?)`
+
+Default behavior:
+- Jika `translate` tidak diberikan -> tidak ada terjemahan.
+- Jika `translate.lang` diberikan tapi `h56-translator` tidak ada:
+  - Default: tetap kembalikan hasil asli dan tambahkan `_translation_note`.
+  - Jika `failOnMissing: true` -> lempar error `TRANSLATOR_MISSING`.
+
+Rekomendasi: lakukan translate secara sequential untuk mengurangi beban, atau lakukan paralelisasi terbatas + cache untuk skala.
+
+---
+
+## API Reference singkat — TypeScript interfaces
+
+Berikut ringkasan interface yang relevan (copas ke .d.ts atau file dokumentasi Anda):
+
+```ts
+// translate-engine/translate.ts (contract)
+export interface TranslationResult {
+  translatedText: string;
+  sourceLang: string;    // kode bahasa terdeteksi (service-defined)
+  targetLang: string;    // bahasa target yang diminta
+  serviceStatus: 'ok' | 'error';
+  raw?: any;             // payload mentah dari service (opsional)
+}
+
+export interface TranslateOptions {
+  endpoint?: string;               // default jika disediakan oleh service
+  signal?: AbortSignal;
+  fetch?: typeof globalThis.fetch;
+  timeoutMs?: number;              // helper timeout, opsional
+}
+
+declare function translate(
+  text: string,
+  targetLang: string,
+  options?: TranslateOptions
+): Promise<TranslationResult>;
+```
+
+Core scraping types:
+
+```ts
+export interface Profile {
+  username: string;
+  name: string;
+  bio: string;
+  followers: number;
+  following: number;
+  public_repos: number;
+  profile_url: string;
+  // optional translation fields added dynamically:
+  bio_translated?: string;
+  bio_source_lang?: string | null;
+  bio_translation_meta?: any;
+  bio_translation_error?: string;
+}
+
+export interface Repo {
+  name: string;
+  description: string;
+  language: string;
+  stars: number;
+  forks: number;
+  updated_at?: string;
+  // optional translated fields:
+  description_translated?: string;
+  description_source_lang?: string | null;
+  description_translation_meta?: any;
+  description_translation_error?: string;
+  name_translated?: string;
+  name_source_lang?: string | null;
+}
+
+export interface Stats {
+  total_repositories: number;
+  total_stars: number;
+  total_forks: number;
+  top_languages: { language: string; repos: number }[];
+}
+
+export function scrapeUser(
+  username: string,
+  opts?: {
+    spinner?: boolean;
+    translate?: {
+      lang: string;
+      fields?: string[];
+      perRepoDelay?: number;
+      failOnMissing?: boolean;
+    };
+  }
+): Promise<{ profile: Profile; repos: Repo[]; stats: Stats; _translation_note?: any }>;
+```
+
+---
+
+## Contoh penggunaan
+
+### CommonJS (Node.js)
+
+```js
+const {
+  scrapeUser,
+  h56translate, // optional helper; may throw if not installed
+} = require("h56-github-scrapper");
+
+(async () => {
+  // tanpa terjemahan
+  const { profile, repos, stats } = await scrapeUser("HASYIM56");
+
+  // dengan terjemahan via opsi
+  const translated = await scrapeUser("HASYIM56", {
+    translate: { lang: "en", fields: ["bio", "repo_descriptions"], perRepoDelay: 120 }
+  });
+  console.log(translated.profile.bio_translated);
+
+  // menggunakan helper langsung (opsional)
+  try {
+    const r = await h56translate("Halo dunia", "en");
+    console.log(r.translatedText);
+  } catch (err) {
+    console.warn("Translator unavailable:", err.message);
+  }
+})();
+```
+
+### ESM (dynamic import)
+
+```js
+const pkg = await import("h56-github-scrapper");
+const { scrapeUser, h56translate } = pkg;
+
+const res = await scrapeUser("HASYIM56", { translate: { lang: "en" } });
+```
+
+### Contoh TypeScript (development)
+
+```ts
+import { translate } from "./translate-engine/translate";
+const r = await translate("Halo dunia", "en");
+console.log(r.translatedText);
+```
+
+---
+
+## Behavior translator opsional & postinstall
+
+- `h56-translator` adalah dependency opsional. Paket menyediakan:
+  - `translate-engine/translate.ts` (typed wrapper) untuk development/TS.
+  - `translate-engine/translate.js` (CJS wrapper) untuk runtime require().
+  - `scripts/ensure-external-deps.js` — postinstall helper yang berusaha memasang `h56-translator` jika tidak ada, kecuali di CI (safety).
+- Jika Anda menginginkan pemasangan otomatis di CI:
+  - jalankan: `H56_FORCE_POSTINSTALL=1 npm install`
+- Jika translator tidak terpasang:
+  - `h56translate(...)` akan melempar Error informatif.
+  - `scrapeUser(..., { translate: {...} })` akan:
+    - menambahkan `_translation_note` dan melanjutkan (default), atau
+    - melempar error jika `failOnMissing: true` disetel.
+
+---
+
+## Best practices & etika scraping
+
+- Jangan paralelisasi scraping untuk banyak akun tanpa jeda; gunakan `SCRAPE_DELAY` dan `MAX_RETRY` yang konservatif.
+- Untuk skala besar/penggunaan produksi, gunakan GitHub API (REST) dengan otentikasi.
+- Untuk terjemahan massal:
+  - Perhatikan rate limit dan biaya pada layanan penerjemah.
+  - Tambahkan cache (memory/file/db) untuk hasil terjemahan agar tidak berulang.
+  - Batasi concurrency ketika melakukan banyak permintaan terjemahan.
+- Gunakan logger terpusat (winston/pino) untuk memantau error, retries, dan metrik.
+
+---
+
+## Troubleshooting
+
+- "Optional dependency 'h56-translator' is not available":
+  - Jalankan: `npm install h56-translator`
+  - Atau jalankan `npm install` ulang dengan `H56_FORCE_POSTINSTALL=1` jika menggunakan postinstall di CI.
+- Parsing kosong/field hilang:
+  - GitHub mungkin mengubah markup; periksa selector di `main-scrapping.js`.
+- Performance / timeout:
+  - Atur `REQUEST_TIMEOUT`, `MAX_RETRY`, dan `SCRAPE_DELAY` saat membuat `new GithubScraper({...})`.
+
+---
+
+## Contributing & changelog singkat
+
+Kontribusi disambut. Silakan:
+- Buka issue jika akan mengubah API publik.
+- Sertakan test untuk fitur baru.
+- Ikuti style guide dan sertakan deskripsi perubahan pada PR.
+
+Changelog singkat (ringkasan):
+- v1.0.0 — Core scraper + optional translator support (h56-translator) + CLI translate flags.

package/script/ensure-external-deps.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+// scripts/ensure-external-deps.js
+// Postinstall helper: idempotent installer for optional external deps.
+// - Will attempt to install packages listed in optionalDeps if missing.
+// - Skips automatic install in CI unless H56_FORCE_POSTINSTALL=1
+// - Does not throw on failure to avoid breaking `npm install` completely.
+
+const { spawnSync } = require("child_process");
+
+const optionalDeps = ["h56-translator"];
+
+function isInstalled(name) {
+  try {
+    require.resolve(name);
+    return true;
+  } catch (_) {
+    return false;
+  }
+}
+
+function installDeps(deps) {
+  if (!deps.length) return;
+  const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
+  const args = ["install", "--no-audit", "--no-fund", "--save", ...deps];
+  console.log("Installing optional dependencies:", deps.join(", "));
+  const res = spawnSync(npmCmd, args, { stdio: "inherit" });
+  if (res.error || res.status !== 0) {
+    console.error("Failed to install optional dependencies. You can run manually:");
+    console.error("  npm install " + deps.join(" "));
+    // do not throw to keep npm install resilient
+  } else {
+    console.log("Optional dependencies installed.");
+  }
+}
+
+(function main() {
+  try {
+    const toInstall = optionalDeps.filter((d) => !isInstalled(d));
+    if (toInstall.length === 0) return;
+
+    // Skip auto-install in CI by default to avoid surprises; allow override.
+    if (process.env.CI && !process.env.H56_FORCE_POSTINSTALL) {
+      console.log(
+        "CI environment detected — skipping automatic installation of optional dependencies.",
+        "Set H56_FORCE_POSTINSTALL=1 to force installation in CI."
+      );
+      return;
+    }
+
+    installDeps(toInstall);
+  } catch (err) {
+    console.error("Postinstall check encountered an error:", err && err.message ? err.message : err);
+    // do not exit non-zero; keep postinstall resilient
+  }
+})();

package/translate-engine/translate.js

@@ -0,0 +1,35 @@
+// translate-engine/translate.js
+// CommonJS runtime wrapper for optional dependency 'h56-translator'.
+// Exports: { translate(text, targetLang, options?) }
+
+"use strict";
+
+function loadTranslator() {
+  try {
+    const mod = require("h56-translator");
+    // handle shapes: named export, default export function, or default object with translate
+    if (mod && typeof mod.translate === "function") return { translate: mod.translate };
+    if (typeof mod === "function") return { translate: mod };
+    if (mod && mod.default && typeof mod.default === "function") return { translate: mod.default };
+    if (mod && mod.default && typeof mod.default.translate === "function")
+      return { translate: mod.default.translate };
+    throw new Error("h56-translator export shape not recognized");
+  } catch (err) {
+    const e = new Error(
+      "Optional dependency 'h56-translator' is not available. Install it with `npm install h56-translator`."
+    );
+    e.cause = err;
+    throw e;
+  }
+}
+
+/**
+ * translate(text, targetLang, options?)
+ * Delegates directly to h56-translator.
+ */
+async function translate(text, targetLang, options) {
+  const t = loadTranslator();
+  return await t.translate(text, targetLang, options);
+}
+
+module.exports = { translate };

package/translate-engine/translate.ts

@@ -0,0 +1,72 @@
+// translate-engine/translate.ts
+// TypeScript typed wrapper for optional dependency 'h56-translator'.
+// This file provides a well-typed `translate` function for TypeScript consumers.
+// Usage (TS):
+//   import { translate } from "./translate-engine/translate";
+//   const r = await translate("Halo dunia", "en");
+
+export interface TranslationResult {
+  translatedText: string;
+  sourceLang: string;    // kode bahasa terdeteksi (service-defined)
+  targetLang: string;    // nilai yang diminta
+  serviceStatus: "ok" | "error";
+  raw?: any;
+}
+
+export interface TranslateOptions {
+  endpoint?: string;
+  signal?: AbortSignal;
+  fetch?: typeof globalThis.fetch;
+  timeoutMs?: number;
+}
+
+type UnderlyingTranslate = (
+  text: string,
+  targetLang: string,
+  options?: TranslateOptions
+) => Promise<TranslationResult>;
+
+async function loadTranslator(): Promise<{ translate: UnderlyingTranslate }> {
+  try {
+    // dynamic import to support both ESM and CJS consumers
+    const mod = await import("h56-translator");
+    const anyMod: any = mod;
+    if (typeof anyMod.translate === "function") {
+      return { translate: anyMod.translate as UnderlyingTranslate };
+    }
+    if (typeof anyMod.default === "function") {
+      return { translate: anyMod.default as UnderlyingTranslate };
+    }
+    if (anyMod.default && typeof anyMod.default.translate === "function") {
+      return { translate: anyMod.default.translate as UnderlyingTranslate };
+    }
+    throw new Error("h56-translator export shape not recognized");
+  } catch (err) {
+    throw new Error(
+      "Optional dependency 'h56-translator' is not available. Install it with `npm install h56-translator`."
+    );
+  }
+}
+
+/**
+ * translate(text, targetLang, options?)
+ * Thin typed wrapper around h56-translator. Normalizes minimal payload.
+ */
+export async function translate(
+  text: string,
+  targetLang: string,
+  options?: TranslateOptions
+): Promise<TranslationResult> {
+  const tmod = await loadTranslator();
+  const raw = await (tmod.translate as UnderlyingTranslate)(text, targetLang, options);
+  if (!raw || typeof raw.translatedText !== "string") {
+    throw new Error("Translation service returned unexpected payload");
+  }
+  return {
+    translatedText: raw.translatedText,
+    sourceLang: raw.sourceLang || "",
+    targetLang: raw.targetLang || targetLang,
+    serviceStatus: raw.serviceStatus || "ok",
+    raw: raw.raw || raw,
+  };
+}