itchlib 0.0.0-stage → 1.0.0

package/src/parsers/game.js

@@ -0,0 +1,164 @@
+import {
+  absoluteUrl,
+  embedUrls,
+  findMeta,
+  imageUrls,
+  parseHtml,
+  readLinks,
+  textOf,
+  unique,
+  videoUrls,
+} from "./html.js";
+
+function parseDetails($, baseUrl) {
+  const details = [];
+  $(".game_info_panel_widget tr").each((_, row) => {
+    const cells = $(row).children("th, td");
+    if (cells.length < 2) return;
+    const label = textOf(cells.first())
+      .replace(/\s*\|\s*$/, "")
+      .trim();
+    if (!label) return;
+    const values = cells.slice(1);
+    details.push({ label, text: textOf(values), links: readLinks($, values, baseUrl) });
+  });
+  return details;
+}
+
+function parseSize(value) {
+  const match = String(value || "").match(/([\d.]+)\s*(B|KB|MB|GB|TB)\b/i);
+  if (!match) return 0;
+  const units = { B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3, TB: 1024 ** 4 };
+  return Math.round(Number(match[1]) * units[match[2].toUpperCase()]);
+}
+
+export function parseUploads(markup) {
+  const $ = parseHtml(markup);
+  const uploads = [];
+  $(".upload").each((_, element) => {
+    const row = $(element);
+    const nameNode = row.find(".name").first();
+    const name = textOf(nameNode);
+    if (!name) return;
+    const id =
+      row.attr("data-upload_id") ||
+      row.attr("data-upload-id") ||
+      row.find('[name="upload_id"]').attr("value") ||
+      null;
+    const size = textOf(row.find(".file_size").first());
+    const version = textOf(row).match(/\bVersion\s+([\d][\w.-]*)/i)?.[1] || null;
+    const platforms = [];
+    row.find(".download_platforms [title]").each((__, platform) => platforms.push($(platform).attr("title")));
+    uploads.push({
+      id: id && /^\d+$/.test(id) ? id : null,
+      name,
+      filename: nameNode.attr("title") || name,
+      size: parseSize(size),
+      version,
+      platforms: unique(platforms),
+    });
+  });
+  return uploads;
+}
+
+function parseComments($, baseUrl) {
+  const comments = [];
+  $(".community_post").each((_, element) => {
+    const post = $(element);
+    const rawData = post.attr("data-post") || "{}";
+    let postData = {};
+    try {
+      postData = JSON.parse(rawData);
+    } catch {
+      /* Some page versions provide non-JSON values. */
+    }
+    const id = post.attr("id")?.match(/post-(\d+)/)?.[1] || post.attr("data-post_id") || null;
+    const authorLink = post.find(".post_author a, a.user_name").first();
+    const author = textOf(authorLink) || null;
+    const authorUrl = absoluteUrl(authorLink.attr("href"), baseUrl);
+    const date = post.find(".post_date").first();
+    const createdAt =
+      date.attr("title") || date.find("time").attr("datetime") || date.attr("datetime") || null;
+    const body = post.find(".post_body, .community_post_body, .community_post_content").first();
+    const text = textOf(body.length ? body : post);
+    if (!text) return;
+    const postUrl = absoluteUrl(date.find("a[href]").first().attr("href"), baseUrl);
+    comments.push({
+      id,
+      userId: Number(postData.user_id) || null,
+      isReply: post.hasClass("is_reply"),
+      author,
+      authorUrl,
+      createdAt,
+      text,
+      html: body.length ? body.html() : null,
+      url: postUrl,
+    });
+  });
+  return comments;
+}
+
+export function parseGame(markup, pageUrl) {
+  const $ = parseHtml(markup);
+  const metaTitle = findMeta($, "og:title");
+  const metaDescription = findMeta($, "og:description") || findMeta($, "description");
+  const title = metaTitle || textOf($(".game_title").first()) || textOf($("title").first());
+  const descriptionNode = $(".formatted_description").first();
+  const description = metaDescription || textOf(descriptionNode);
+  const descriptionHtml = descriptionNode.html()?.trim() || null;
+  const authorLink = $("a.user_name").first();
+  const author = textOf(authorLink) || null;
+  const authorUrl = absoluteUrl(authorLink.attr("href"), pageUrl);
+  const screenshotContainer = $(".screenshot_list").first();
+  const screenshotSource = screenshotContainer.length ? screenshotContainer : $("body");
+  const images = unique([
+    absoluteUrl(findMeta($, "og:image"), pageUrl),
+    ...imageUrls($, screenshotSource, pageUrl),
+  ]);
+  const details = parseDetails($, pageUrl);
+  const detail = (label) => details.find((item) => item.label.toLowerCase() === label.toLowerCase());
+  const purchase = $("a.buy_btn, a.donate_btn").first();
+  const itchPath = findMeta($, "itch:path");
+  const numericGameId = itchPath.match(/(?:^|\/)games\/(\d+)/)?.[1];
+  const cleanedTitle = String(title).replace(/\s+/g, " ").trim();
+  const authors = detail("Authors")?.links || (author ? [{ url: authorUrl, text: author }] : []);
+  const tags = detail("Tags")?.links || [];
+  const uploads = parseUploads(markup);
+
+  return {
+    id: numericGameId ? Number(numericGameId) : null,
+    slug: new URL(pageUrl).pathname.replace(/\/$/, "").split("/").pop() || null,
+    url: new URL(pageUrl).href,
+    title: cleanedTitle,
+    author,
+    authorUrl,
+    authors,
+    description,
+    descriptionHtml,
+    cover: images[0] || null,
+    images,
+    screenshots: images.slice(1),
+    videos: videoUrls($, $("body"), pageUrl),
+    embeds: embedUrls($, $("body"), pageUrl),
+    links: readLinks($, $("body"), pageUrl),
+    price: textOf($(".buy_message").first()) || null,
+    purchase: purchase.length
+      ? {
+          url: absoluteUrl(purchase.attr("href"), pageUrl),
+          label: textOf(purchase),
+        }
+      : null,
+    details,
+    status: detail("Status")?.text || null,
+    pagePlatforms: detail("Platforms")?.links || [],
+    genres: detail("Genre")?.links || [],
+    tags,
+    madeWith: detail("Made with")?.links || [],
+    languages: detail("Languages")?.links || [],
+    inputs: detail("Inputs")?.links || [],
+    accessibility: detail("Accessibility")?.links || [],
+    developmentLogs: readLinks($, $(".devlog_list").first(), pageUrl),
+    comments: parseComments($, pageUrl),
+    uploads,
+  };
+}

package/src/parsers/html.js

@@ -0,0 +1,114 @@
+import { load } from "cheerio";
+
+export function parseHtml(markup, options = {}) {
+  return load(String(markup ?? ""), { decodeEntities: true, ...options });
+}
+
+export function textOf(element) {
+  if (!element?.length) return "";
+  const copy = element.clone();
+  copy.find("script, style, template").remove();
+  return copy.text().replace(/\s+/g, " ").trim();
+}
+
+export function absoluteUrl(value, base) {
+  if (typeof value !== "string" || !value.trim() || /^(?:data|javascript|mailto|tel):/i.test(value.trim()))
+    return null;
+  try {
+    const url = new URL(value, base);
+    return ["https:", "http:"].includes(url.protocol) ? url.href : null;
+  } catch {
+    return null;
+  }
+}
+
+function linkUrl(value, base) {
+  if (typeof value !== "string" || !value.trim()) return null;
+  const trimmed = value.trim();
+  if (/^(?:mailto|tel|itch):/i.test(trimmed)) return trimmed;
+  return absoluteUrl(trimmed, base);
+}
+
+export function findMeta($, key) {
+  const wanted = key.toLowerCase();
+  let result = "";
+  $("meta[property], meta[name], meta[itemprop]").each((_, element) => {
+    const meta = $(element);
+    const name = (meta.attr("property") || meta.attr("name") || meta.attr("itemprop") || "").toLowerCase();
+    if (!result && name === wanted) result = meta.attr("content") || "";
+  });
+  return result;
+}
+
+export function readLinks($, scope, base) {
+  const links = [];
+  const seen = new Set();
+  scope.find("a[href]").each((_, element) => {
+    const anchor = $(element);
+    const url = linkUrl(anchor.attr("href"), base);
+    if (!url) return;
+    const text = textOf(anchor);
+    const title = anchor.attr("title");
+    const key = `${url}\n${text}\n${title || ""}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    links.push({ url, text, ...(title ? { title } : {}) });
+  });
+  return links;
+}
+
+export function imageUrls($, scope, base) {
+  const urls = [];
+  scope
+    .find('a[data-image_lightbox="true"][href]')
+    .each((_, el) => urls.push(absoluteUrl($(el).attr("href"), base)));
+  scope.find("img").each((_, el) => {
+    const img = $(el);
+    const source = img.attr("src") || img.attr("data-src") || img.attr("data-original") || img.attr("poster");
+    if (source) urls.push(absoluteUrl(source, base));
+    const srcset = img.attr("srcset") || img.attr("data-srcset") || img.attr("data-lazy_srcset");
+    if (srcset)
+      for (const candidate of srcset.split(","))
+        urls.push(absoluteUrl(candidate.trim().split(/\s+/)[0], base));
+  });
+  return unique(urls);
+}
+
+export function videoUrls($, scope, base) {
+  const urls = [];
+  scope.find("video, source").each((_, el) => {
+    const node = $(el);
+    const value = node.attr("src") || node.attr("data-src") || node.attr("data-video-url");
+    const url = absoluteUrl(value, base);
+    if (
+      url &&
+      (el.tagName === "video" ||
+        /video\//i.test(node.attr("type") || "") ||
+        /\.(?:mp4|webm|mov|m4v)(?:$|\?)/i.test(url))
+    )
+      urls.push(url);
+  });
+  scope.find("iframe").each((_, el) => {
+    const url = absoluteUrl($(el).attr("src") || $(el).attr("data-src"), base);
+    if (
+      url &&
+      /(?:youtube(?:-nocookie)?\.com|youtu\.be|vimeo\.com|dailymotion\.com|player\.twitch\.tv|streamable\.com)/i.test(
+        url,
+      )
+    )
+      urls.push(url);
+  });
+  return unique(urls);
+}
+
+export function embedUrls($, scope, base) {
+  const urls = [];
+  scope
+    .find("iframe")
+    .each((_, el) => urls.push(absoluteUrl($(el).attr("src") || $(el).attr("data-src"), base)));
+  return unique(urls);
+}
+
+export function unique(values) {
+  return [...new Set(values.filter(Boolean))];
+}

package/src/parsers/profile.js

@@ -0,0 +1,33 @@
+import { absoluteUrl, findMeta, parseHtml, readLinks, textOf } from "./html.js";
+import { parseGameCards } from "./cards.js";
+
+export function parseProfile(markup, pageUrl) {
+  const $ = parseHtml(markup);
+  const canonicalUrl = new URL(pageUrl);
+  const title = findMeta($, "og:title") || textOf($("h1").first()) || textOf($("title").first());
+  const avatarNode = $(".user_avatar img, .user_cover img, .user_avatar, .user_cover").first();
+  const styleUrl = avatarNode.attr("style")?.match(/url\(["']?([^"')]+)["']?\)/i)?.[1];
+  const avatar = absoluteUrl(
+    findMeta($, "og:image") || avatarNode.attr("src") || avatarNode.attr("data-src") || styleUrl,
+    canonicalUrl,
+  );
+  const bioNode = $(".user_bio").first();
+  const description = findMeta($, "og:description") || textOf(bioNode);
+  const itchPath = findMeta($, "itch:path");
+  const profilePathUserId = itchPath.match(/users\/(\d+)/)?.[1];
+  const username = canonicalUrl.hostname.endsWith(".itch.io")
+    ? canonicalUrl.hostname.slice(0, -".itch.io".length)
+    : canonicalUrl.pathname.split("/").filter(Boolean).at(-1) || null;
+
+  return {
+    id: Number(profilePathUserId) || null,
+    username,
+    displayName: title.replace(/\s*(?:\||-)\s*itch\.io\s*$/i, "") || username || "",
+    url: canonicalUrl.href,
+    avatar,
+    bio: description,
+    bioHtml: bioNode.html()?.trim() || null,
+    links: readLinks($, $(".user_links").first(), canonicalUrl),
+    games: parseGameCards(markup, canonicalUrl),
+  };
+}

package/src/routes.js

@@ -0,0 +1,26 @@
+export const ITCH_BROWSE_PAGES = Object.freeze({
+  games: "/games",
+  store: "/games/store",
+  freeGames: "/games/price-free",
+  funGames: "/games/fun",
+  demos: "/games/price-demo",
+  onSale: "/games/price-sale",
+  topRated: "/games/top-rated",
+  topSellers: "/games/top-sellers",
+  newestGames: "/games/newest",
+  newAndPopular: "/games/new-and-popular",
+  tools: "/tools",
+  assets: "/game-assets",
+  soundtracks: "/soundtracks",
+  physicalGames: "/physical-games",
+  comics: "/comics",
+  books: "/books",
+  jams: "/jams",
+  community: "/community",
+  devlogs: "/devlogs",
+  tags: "/tags",
+  bundles: "/bundles",
+  blog: "/blog",
+  developers: "/developers",
+  search: "/search",
+});

package/src/urls.js

@@ -0,0 +1,69 @@
+import { ItchError } from "./errors.js";
+
+export function validateGameUrl(value) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch (cause) {
+    throw new ItchError("Expected a valid itch.io game URL", { code: "ITCH_INVALID_URL", cause });
+  }
+  if (
+    url.protocol !== "https:" ||
+    !/(^|\.)itch\.io$/i.test(url.hostname) ||
+    url.hostname.toLowerCase() === "itch.io"
+  ) {
+    throw new ItchError("Expected an HTTPS game page on an itch.io subdomain", {
+      url: url.href,
+      code: "ITCH_INVALID_URL",
+    });
+  }
+  return url;
+}
+
+export function normalizeProfileUrl(profile) {
+  if (typeof profile !== "string" || !profile.trim()) {
+    throw new TypeError("profile must be a username or itch.io profile URL");
+  }
+  if (/^https?:\/\//i.test(profile)) {
+    let url;
+    try {
+      url = new URL(profile);
+    } catch (cause) {
+      throw new ItchError("Expected a valid itch.io profile URL", { code: "ITCH_INVALID_URL", cause });
+    }
+    if (url.protocol !== "https:" || !/(^|\.)itch\.io$/i.test(url.hostname)) {
+      throw new ItchError("Only HTTPS itch.io profile URLs are supported", {
+        url: url.href,
+        code: "ITCH_INVALID_URL",
+      });
+    }
+    return url;
+  }
+  if (profile.includes("/") || /[?#\s]/.test(profile))
+    throw new ItchError("Expected a username or an itch.io profile URL", { code: "ITCH_INVALID_URL" });
+  return new URL(`https://itch.io/profile/${encodeURIComponent(profile)}`);
+}
+
+export function resolveGamePath(pageUrl, path) {
+  const page = new URL(pageUrl.href);
+  page.pathname = `${page.pathname.replace(/\/$/, "")}/`;
+  page.search = "";
+  page.hash = "";
+  return new URL(`./${path}`, page);
+}
+
+export function assertItchUrl(value, { allowSubdomains = false } = {}) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch (cause) {
+    throw new ItchError("Expected an HTTPS itch.io URL", { code: "ITCH_INVALID_URL", cause });
+  }
+  const validHost = allowSubdomains
+    ? /(^|\.)itch\.io$/i.test(url.hostname)
+    : url.hostname.toLowerCase() === "itch.io";
+  if (url.protocol !== "https:" || !validHost) {
+    throw new ItchError("URL must target HTTPS itch.io", { url: url.href, code: "ITCH_INVALID_URL" });
+  }
+  return url;
+}