itchlib 0.0.0-stage → 1.0.0

package/src/downloads.js

@@ -0,0 +1,131 @@
+import { load } from "cheerio";
+import { assertResponse, ItchError, readJson } from "./errors.js";
+import { parseUploads } from "./parsers/game.js";
+import { resolveGamePath, validateGameUrl } from "./urls.js";
+
+async function getCsrfToken(client, gameUrl, signal) {
+  const purchaseUrl = resolveGamePath(gameUrl, "purchase");
+  const response = assertResponse(
+    await client.request(purchaseUrl, { signal }),
+    "Itch.io purchase page failed",
+    purchaseUrl,
+  );
+  const purchaseHtml = await response.text();
+  const $ = load(purchaseHtml);
+  const token = $('meta[name="csrf_token"]').attr("value") || $('meta[name="csrf_token"]').attr("content");
+  if (!token)
+    throw new ItchError("Itch.io download token is unavailable", {
+      url: purchaseUrl.href,
+      code: "ITCH_ACCESS_ERROR",
+    });
+  return { purchaseUrl, token };
+}
+
+async function getDownloadPageUrl(client, gameUrl, token, purchaseUrl, signal) {
+  const endpoint = resolveGamePath(gameUrl, "download_url");
+  const response = assertResponse(
+    await client.request(endpoint, {
+      method: "POST",
+      signal,
+      headers: {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "X-Requested-With": "XMLHttpRequest",
+        Referer: purchaseUrl.href,
+      },
+      body: new URLSearchParams({ csrf_token: token, reward_id: "" }),
+    }),
+    "Itch.io download page request failed",
+    endpoint,
+  );
+  const result = await readJson(response, "Itch.io download endpoint returned invalid JSON", endpoint);
+  const rawUrl = result?.url;
+  if (!rawUrl)
+    throw new ItchError("Itch.io did not return a download page URL", {
+      url: endpoint.href,
+      code: "ITCH_ACCESS_ERROR",
+    });
+  let downloadPageUrl;
+  try {
+    downloadPageUrl = new URL(rawUrl, gameUrl);
+  } catch (cause) {
+    throw new ItchError("Itch.io returned an invalid download page URL", {
+      url: endpoint.href,
+      code: "ITCH_INVALID_RESPONSE",
+      cause,
+    });
+  }
+  if (downloadPageUrl.protocol !== "https:")
+    throw new ItchError("Itch.io returned an unsupported download page URL", {
+      url: downloadPageUrl.href,
+      code: "ITCH_INVALID_RESPONSE",
+    });
+  return downloadPageUrl;
+}
+
+export async function getDownloadUploads(client, gameValue, { signal } = {}) {
+  const gameUrl = validateGameUrl(gameValue);
+  const { purchaseUrl, token } = await getCsrfToken(client, gameUrl, signal);
+  const downloadPageUrl = await getDownloadPageUrl(client, gameUrl, token, purchaseUrl, signal);
+  const pageResponse = assertResponse(
+    await client.request(downloadPageUrl, { signal }),
+    "Itch.io download page failed",
+    downloadPageUrl,
+  );
+  return parseUploads(await pageResponse.text());
+}
+
+export async function getDownloadUrl(client, gameValue, uploadId, { signal } = {}) {
+  const gameUrl = validateGameUrl(gameValue);
+  if (!/^\d+$/.test(String(uploadId))) throw new TypeError("uploadId must be a numeric itch.io upload ID");
+  const { purchaseUrl, token } = await getCsrfToken(client, gameUrl, signal);
+  const downloadPageUrl = await getDownloadPageUrl(client, gameUrl, token, purchaseUrl, signal);
+  const pageResponse = assertResponse(
+    await client.request(downloadPageUrl, { signal }),
+    "Itch.io download page failed",
+    downloadPageUrl,
+  );
+  const upload = parseUploads(await pageResponse.text()).find((item) => item.id === String(uploadId));
+  if (!upload)
+    throw new ItchError(`Itch.io upload ${uploadId} is unavailable`, {
+      url: downloadPageUrl.href,
+      code: "ITCH_ACCESS_ERROR",
+    });
+
+  const fileEndpoint = resolveGamePath(gameUrl, `file/${encodeURIComponent(uploadId)}?source=game_download`);
+  const fileResponse = assertResponse(
+    await client.request(fileEndpoint, {
+      method: "POST",
+      signal,
+      headers: {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "X-Requested-With": "XMLHttpRequest",
+        Referer: purchaseUrl.href,
+      },
+      body: new URLSearchParams({ csrf_token: token }),
+    }),
+    "Itch.io file download request failed",
+    fileEndpoint,
+  );
+  const result = await readJson(fileResponse, "Itch.io file endpoint returned invalid JSON", fileEndpoint);
+  if (!result?.url)
+    throw new ItchError("Itch.io did not return a download URL", {
+      url: fileEndpoint.href,
+      code: "ITCH_ACCESS_ERROR",
+    });
+  let fileUrl;
+  try {
+    fileUrl = new URL(result.url, gameUrl);
+  } catch (cause) {
+    throw new ItchError("Itch.io returned an invalid download URL", {
+      url: fileEndpoint.href,
+      code: "ITCH_INVALID_RESPONSE",
+      cause,
+    });
+  }
+  if (!/^https?:$/.test(fileUrl.protocol))
+    throw new ItchError("Itch.io returned an unsupported download URL", {
+      url: fileUrl.href,
+      code: "ITCH_INVALID_RESPONSE",
+    });
+  return fileUrl.href;
+}

package/src/errors.js

@@ -0,0 +1,33 @@
+export class ItchError extends Error {
+  constructor(message, { status, url, code = "ITCH_ERROR", cause } = {}) {
+    super(message, { cause });
+    this.name = "ItchError";
+    this.code = code;
+    if (status !== undefined) this.status = status;
+    if (url) this.url = url;
+  }
+}
+
+export function assertResponse(response, message, url) {
+  if (!response.ok) {
+    throw new ItchError(`${message} (HTTP ${response.status})`, {
+      status: response.status,
+      url: String(url),
+      code: "ITCH_HTTP_ERROR",
+    });
+  }
+  return response;
+}
+
+export async function readJson(response, message, url) {
+  try {
+    return await response.json();
+  } catch (cause) {
+    throw new ItchError(message, {
+      status: response.status,
+      url: String(url),
+      code: "ITCH_INVALID_RESPONSE",
+      cause,
+    });
+  }
+}

package/src/http.js

@@ -0,0 +1,103 @@
+import { ItchError } from "./errors.js";
+
+const DEFAULT_TIMEOUT_MS = 15_000;
+const DEFAULT_ATTEMPTS = 3;
+const DEFAULT_MAX_RETRY_DELAY_MS = 30_000;
+const RETRYABLE_STATUS = new Set([408, 429]);
+
+function retryDelay(response, attempt) {
+  const retryAfter = response.headers?.get?.("retry-after");
+  const seconds = Number(retryAfter);
+  if (retryAfter && Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
+  if (retryAfter) {
+    const timestamp = Date.parse(retryAfter);
+    if (Number.isFinite(timestamp)) return Math.max(0, timestamp - Date.now());
+  }
+  const base = response.status === 429 ? 2_000 : 500;
+  return Math.round(base * 2 ** (attempt - 1) * (0.8 + Math.random() * 0.4));
+}
+
+function isRetryableStatus(status) {
+  return RETRYABLE_STATUS.has(status) || status >= 500;
+}
+
+export class ItchTransport {
+  constructor({
+    fetch: fetchImpl = globalThis.fetch,
+    timeout = DEFAULT_TIMEOUT_MS,
+    headers = {},
+    userAgent = "itchlib/0.1.0",
+    maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,
+  } = {}) {
+    if (typeof fetchImpl !== "function") throw new TypeError("A Fetch API implementation is required");
+    if (!Number.isFinite(timeout) || timeout <= 0)
+      throw new TypeError("timeout must be a positive number of milliseconds");
+    if (!Number.isFinite(maxRetryDelay) || maxRetryDelay < 0)
+      throw new TypeError("maxRetryDelay must be a non-negative number of milliseconds");
+    this.fetch = fetchImpl;
+    this.timeout = timeout;
+    this.maxRetryDelay = maxRetryDelay;
+    this.headers = { "User-Agent": userAgent, Accept: "text/html,application/xhtml+xml", ...headers };
+  }
+
+  async request(url, options = {}) {
+    const timeout = options.timeout ?? this.timeout;
+    const { timeout: _timeout, maxRetryDelay: _maxRetryDelay, ...fetchOptions } = options;
+    let lastError;
+
+    for (let attempt = 1; attempt <= DEFAULT_ATTEMPTS; attempt += 1) {
+      const controller = new AbortController();
+      const externalSignal = fetchOptions.signal;
+      const abortFromCaller = () => controller.abort(externalSignal.reason);
+      if (externalSignal?.aborted)
+        throw new ItchError("Itch.io request was aborted", { url: String(url), code: "ITCH_ABORTED" });
+      externalSignal?.addEventListener("abort", abortFromCaller, { once: true });
+      const timer = setTimeout(
+        () => controller.abort(new Error(`Request timed out after ${timeout} ms`)),
+        timeout,
+      );
+
+      try {
+        const response = await this.fetch(url, {
+          ...fetchOptions,
+          headers: { ...this.headers, ...fetchOptions.headers },
+          signal: controller.signal,
+        });
+        if (!isRetryableStatus(response.status) || attempt === DEFAULT_ATTEMPTS) return response;
+        const delay = retryDelay(response, attempt);
+        if (delay > this.maxRetryDelay) return response;
+        try {
+          await response.body?.cancel?.();
+        } catch {
+          /* A body may already have been consumed or closed. */
+        }
+        lastError = new ItchError(`Itch.io request failed with HTTP ${response.status}`, {
+          status: response.status,
+          url: String(url),
+          code: "ITCH_HTTP_ERROR",
+        });
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      } catch (cause) {
+        if (externalSignal?.aborted)
+          throw new ItchError("Itch.io request was aborted", {
+            url: String(url),
+            code: "ITCH_ABORTED",
+            cause,
+          });
+        lastError = new ItchError(`Itch.io request failed: ${cause.message}`, {
+          url: String(url),
+          code: "ITCH_NETWORK_ERROR",
+          cause,
+        });
+        if (attempt === DEFAULT_ATTEMPTS) throw lastError;
+        await new Promise((resolve) =>
+          setTimeout(resolve, Math.round(500 * 2 ** (attempt - 1) * (0.8 + Math.random() * 0.4))),
+        );
+      } finally {
+        clearTimeout(timer);
+        externalSignal?.removeEventListener("abort", abortFromCaller);
+      }
+    }
+    throw lastError;
+  }
+}

package/src/index.js

@@ -0,0 +1,40 @@
+export { ItchClient } from "./client.js";
+export { ItchError } from "./errors.js";
+export { ITCH_BROWSE_PAGES } from "./routes.js";
+
+import { ItchClient } from "./client.js";
+
+export function createItchClient(options) {
+  return new ItchClient(options);
+}
+
+export async function getGame(gameUrl, options) {
+  return new ItchClient(options).getGame(gameUrl);
+}
+export async function getUploads(gameUrl, options) {
+  return new ItchClient(options).getUploads(gameUrl);
+}
+export async function getDownloadUploads(gameUrl, options) {
+  return new ItchClient(options).getDownloadUploads(gameUrl);
+}
+export async function getDownloadUrl(gameUrl, uploadId, options) {
+  return new ItchClient(options).getDownloadUrl(gameUrl, uploadId);
+}
+export async function getComments(gameUrl, options) {
+  return new ItchClient(options).getComments(gameUrl);
+}
+export async function getProfile(profile, options) {
+  return new ItchClient(options).getProfile(profile);
+}
+export async function searchGames(query, options, clientOptions) {
+  return new ItchClient(clientOptions).searchGames(query, options);
+}
+export async function getSuggestions(query, options) {
+  return new ItchClient(options).getSuggestions(query);
+}
+export async function getBrowsePage(section, browseOptions, clientOptions) {
+  return new ItchClient(clientOptions).getBrowsePage(section, browseOptions);
+}
+export async function getFeed(feed, options) {
+  return new ItchClient(options).getFeed(feed);
+}

package/src/parsers/browse.js

@@ -0,0 +1,247 @@
+import { absoluteUrl, parseHtml, readLinks, textOf } from "./html.js";
+import { parseGameCards } from "./cards.js";
+
+function parseFilterPickers($, baseUrl) {
+  const pickers = [];
+  $(".filter_picker_widget").each((_, element) => {
+    const picker = $(element);
+    const options = [];
+    picker.find(".filter_option").each((__, optionElement) => {
+      const option = $(optionElement);
+      options.push({
+        label: textOf(option),
+        url: absoluteUrl(option.attr("href"), baseUrl),
+        value: option.attr("data-value") || null,
+        selected: option.hasClass("selected"),
+      });
+    });
+    if (options.length)
+      pickers.push({
+        id: picker.attr("id") || null,
+        selected: textOf(picker.find(".value_label").first()) || null,
+        options,
+      });
+  });
+  return pickers;
+}
+
+function parseFilterGroups($, baseUrl) {
+  const groups = [];
+  $(".browse_filter_group_widget").each((_, element) => {
+    const group = $(element);
+    const options = [];
+    group.find("a[href]").each((__, optionElement) => {
+      const option = $(optionElement);
+      const href = option.attr("href");
+      if (!href || href === "#") return;
+      options.push({
+        label: textOf(option),
+        url: absoluteUrl(href, baseUrl),
+        value: option.attr("data-value") || null,
+        selected: option.hasClass("selected"),
+      });
+    });
+    const label = textOf(group.find(".filter_group_label").first());
+    if (label && options.length) groups.push({ id: group.attr("id") || null, label, options });
+  });
+  return groups;
+}
+
+function parseFacets($, baseUrl) {
+  const facets = [];
+  $(".facet_group").each((_, element) => {
+    const group = $(element);
+    const label = textOf(group.find(".facet_group_label").first());
+    const values = [];
+    group.find("a.facet_value[href]").each((__, valueElement) => {
+      const value = $(valueElement);
+      const countText = textOf(value.find(".facet_count").first()).replace(/,/g, "");
+      values.push({
+        label: textOf(value.find(".facet_name").first()) || textOf(value),
+        count: Number(countText) || null,
+        url: absoluteUrl(value.attr("href"), baseUrl),
+      });
+    });
+    if (label && values.length) facets.push({ label, values });
+  });
+  return facets;
+}
+
+function parseJams($, baseUrl) {
+  const jams = [];
+  $(".jam_cell").each((_, element) => {
+    const cell = $(element);
+    const anchor = cell.find("a[href]").first();
+    const url = absoluteUrl(anchor.attr("href"), baseUrl);
+    if (!url) return;
+    const count = textOf(cell.find(".joined_count").first()).match(/[\d,]+/)?.[0];
+    jams.push({ title: textOf(anchor), url, joinedCount: count ? Number(count.replace(/,/g, "")) : null });
+  });
+  return jams;
+}
+
+function parseTags($, baseUrl) {
+  const tags = [];
+  $(".tag_cell").each((_, element) => {
+    const cell = $(element);
+    const anchor = cell.find(".tag_title[href]").first().length
+      ? cell.find(".tag_title[href]").first()
+      : cell.find("a[href]").first();
+    const url = absoluteUrl(anchor.attr("href"), baseUrl);
+    if (!url) return;
+    const image = cell.find("img").first();
+    tags.push({
+      title: textOf(anchor),
+      url,
+      cover: absoluteUrl(image.attr("src") || image.attr("data-lazy_src"), baseUrl),
+      count: textOf(cell.find(".games_count").first()) || null,
+      description: textOf(cell.find(".tag_description").first()) || null,
+    });
+  });
+  return tags;
+}
+
+function parseBlogPosts($, baseUrl, allLinks) {
+  const posts = [];
+  $(".post_preview").each((_, element) => {
+    const preview = $(element);
+    const summaryLink = preview.find("a.summary_link[href]").first();
+    const localTitle = preview.find("a.title[href]").first();
+    const url = absoluteUrl(localTitle.attr("href") || summaryLink.attr("href"), baseUrl);
+    if (!url) return;
+    const title =
+      textOf(localTitle) ||
+      allLinks.find((link) => link.url === url && !/continue reading/i.test(link.text))?.text;
+    if (!title) return;
+    posts.push({ title, url, summary: textOf(preview.find(".summary_text").first()) || null });
+  });
+  return posts;
+}
+
+function parseDevlogs($, baseUrl) {
+  const posts = [];
+  $(".post_title").each((_, element) => {
+    const anchor = $(element).find("a[href]").first();
+    const url = absoluteUrl(anchor.attr("href"), baseUrl);
+    const title = textOf(anchor);
+    if (url && title) posts.push({ title, url });
+  });
+  return posts;
+}
+
+function parseTopics($, allLinks) {
+  const topics = [];
+  $(".topic_title").each((_, element) => {
+    const title = textOf($(element));
+    const link = allLinks.find((candidate) => candidate.text.includes(title) && /\/t\//.test(candidate.url));
+    if (title && link) topics.push({ title, url: link.url });
+  });
+  return topics;
+}
+
+function parseCategories($, baseUrl) {
+  const categories = [];
+  $(".category_row").each((_, element) => {
+    const row = $(element);
+    const titleLink = row.find(".category_title a[href]").first();
+    const url = absoluteUrl(titleLink.attr("href"), baseUrl);
+    const title = textOf(titleLink);
+    if (!url || !title) return;
+    const latestTopicLink = row.find(".last_topic[href], .last_topic a[href]").first();
+    const latestTopicUrl = absoluteUrl(latestTopicLink.attr("href"), baseUrl);
+    const latestTopicTitle = textOf(latestTopicLink.find(".topic_title").first());
+    const topicCount = textOf(row.find(".topics_count").first()).match(/[\d,]+/)?.[0];
+    categories.push({
+      title,
+      url,
+      description: textOf(row.find(".short_description").first()) || null,
+      topicCount: topicCount ? Number(topicCount.replace(/,/g, "")) : null,
+      latestTopic:
+        latestTopicUrl && latestTopicTitle
+          ? {
+              title: latestTopicTitle,
+              url: latestTopicUrl,
+              author: textOf(row.find(".last_topic_author").first()) || null,
+            }
+          : null,
+    });
+  });
+  return categories;
+}
+
+function parseSales($, baseUrl) {
+  const sales = [];
+  $(".sale_row").each((_, element) => {
+    const row = $(element);
+    const titleLink = row.find(".sale_title a[href]").first();
+    const url = absoluteUrl(titleLink.attr("href"), baseUrl);
+    const title = textOf(titleLink);
+    if (!url || !title) return;
+    const authorLink = row.find(".sale_author a[href]").first();
+    const itemCount = textOf(row.find(".game_count").first()).match(/[\d,]+/)?.[0];
+    const endsAt = row.find(".sale_end .date_format").first().attr("title") || null;
+    const platforms = [];
+    row.find(".game_platforms [title]").each((__, label) => platforms.push($(label).attr("title")));
+    sales.push({
+      id: row.attr("data-sale_id") || null,
+      type: row.attr("data-sale_type") || null,
+      title,
+      url,
+      author: authorLink.length
+        ? { url: absoluteUrl(authorLink.attr("href"), baseUrl), text: textOf(authorLink) }
+        : null,
+      itemCount: itemCount ? Number(itemCount.replace(/,/g, "")) : null,
+      endsAt,
+      platforms: [...new Set(platforms.filter(Boolean))],
+      covers: row
+        .find("img.game_thumb")
+        .map((__, image) => absoluteUrl($(image).attr("data-lazy_src") || $(image).attr("src"), baseUrl))
+        .get()
+        .filter(Boolean),
+    });
+  });
+  return sales;
+}
+
+export function parseBrowsePage(markup, pageUrl) {
+  const $ = parseHtml(markup);
+  const allLinks = readLinks($, $("body"), pageUrl);
+  const sortRoot = $(".browse_sort_options_widget .sorts").first();
+  const mobileSort = $(".sort_options_mobile .filter_option.selected").first();
+  const activeButton = sortRoot.find("button.active").first();
+  const activeSort = textOf(mobileSort) || textOf(activeButton) || null;
+  const sortOptions = readLinks($, sortRoot, pageUrl).map((option) => ({
+    ...option,
+    selected: option.text === activeSort,
+  }));
+  if (activeSort && !sortOptions.some((option) => option.selected))
+    sortOptions.unshift({ text: activeSort, url: pageUrl, selected: true });
+  const countText = textOf($(".game_count").first());
+  const nextHref = $("a[rel~='next']").first().attr("href");
+  const sales = parseSales($, pageUrl);
+  const bundleUrls = new Set(sales.filter((sale) => sale.type === "bundle").map((sale) => sale.url));
+  const bundleLinks = allLinks.filter((link) => /\/b\/\d+\//.test(link.url) && !bundleUrls.has(link.url));
+
+  return {
+    url: pageUrl,
+    title: textOf($("title").first()),
+    query: new URL(pageUrl).searchParams.get("q"),
+    resultCount: Number(countText.match(/[\d,]+/)?.[0]?.replace(/,/g, "")) || null,
+    games: parseGameCards(markup, pageUrl),
+    jams: parseJams($, pageUrl),
+    categories: parseCategories($, pageUrl),
+    tags: parseTags($, pageUrl),
+    bundles: [...sales.filter((sale) => sale.type === "bundle"), ...bundleLinks],
+    sales,
+    devlogs: parseDevlogs($, pageUrl),
+    blogPosts: parseBlogPosts($, pageUrl, allLinks),
+    topics: parseTopics($, allLinks),
+    filters: parseFilterGroups($, pageUrl),
+    filterPickers: parseFilterPickers($, pageUrl),
+    facets: parseFacets($, pageUrl),
+    sortOptions,
+    activeSort,
+    relatedTags: readLinks($, $(".browse_top_tags").first(), pageUrl),
+    nextPageUrl: absoluteUrl(nextHref, pageUrl),
+  };
+}

package/src/parsers/cards.js

@@ -0,0 +1,36 @@
+import { absoluteUrl, parseHtml, textOf } from "./html.js";
+
+export function parseGameCards(markup, baseUrl) {
+  const $ = parseHtml(markup);
+  const games = [];
+  $(".game_cell").each((_, element) => {
+    const card = $(element);
+    const anchor = card.find("a.title[href]").first().length
+      ? card.find("a.title[href]").first()
+      : card.find("a[href]").first();
+    const url = absoluteUrl(anchor.attr("href"), baseUrl);
+    const image = card.find("img").first();
+    const title = textOf(anchor) || image.attr("alt") || "";
+    if (!url || !title) return;
+    const id = Number(card.attr("data-game_id")) || null;
+    const cover = absoluteUrl(
+      image.attr("src") || image.attr("data-src") || image.attr("data-lazy_src"),
+      baseUrl,
+    );
+    const description = textOf(card.find(".game_text").first());
+    const author = textOf(card.find(".game_author").first());
+    const genre = textOf(card.find(".game_genre").first());
+    const platforms = textOf(card.find(".game_platform").first());
+    games.push({
+      id,
+      title,
+      url,
+      cover,
+      description: description || null,
+      author: author || null,
+      genre: genre || null,
+      platforms: platforms || null,
+    });
+  });
+  return games;
+}

package/src/parsers/feed.js

@@ -0,0 +1,31 @@
+import { load } from "cheerio";
+import { parseHtml, textOf } from "./html.js";
+
+function elementText($, scope, name) {
+  const localName = name.includes(":") ? name.split(":").at(-1) : name;
+  let value;
+  scope.find("*").each((_, element) => {
+    if (!value && element.name?.split(":").at(-1)?.toLowerCase() === localName.toLowerCase()) {
+      value = textOf($(element));
+    }
+  });
+  return value || "";
+}
+
+export function parseFeed(markup, feed, url) {
+  const $ = load(String(markup ?? ""), { xmlMode: true, decodeEntities: true });
+  const items = [];
+  $("item").each((_, element) => {
+    const item = $(element);
+    const field = (selector, fallbackName = selector) =>
+      textOf(item.find(selector).first()) || elementText($, item, fallbackName);
+    items.push({
+      title: field("title"),
+      url: field("link"),
+      description: textOf(parseHtml(field("description"))("body")),
+      publishedAt: field("pubDate"),
+      author: field("dc\\:creator", "dc:creator") || null,
+    });
+  });
+  return { feed, url, items };
+}