litescrape-sdk 0.1.0

package/src/runtime.ts

@@ -0,0 +1,255 @@
+import { createHash } from "node:crypto";
+import { setTimeout as delay } from "node:timers/promises";
+import { APIError, LitescrapeError, TransportError, apiError, isObject } from "./errors.js";
+import { VERSION } from "./version.js";
+
+export const DEFAULT_BASE_URL = "https://api.litescrape.com";
+export const USER_AGENT = `litescrape-sdk/${VERSION} (Node/${process.versions.node}; ${process.platform}; TypeScript)`;
+export const CLIENT_HEADER_VALUE = `typescript-sdk/${VERSION}`;
+export type Fetch = typeof globalThis.fetch;
+
+export interface Outcome {
+  data: Record<string, unknown> | null;
+  error: LitescrapeError | null;
+  statusCode: number | null;
+  requestId: string;
+  attempts: number;
+}
+export const emptyOutcome = (): Outcome => ({
+  data: null,
+  error: null,
+  statusCode: null,
+  requestId: "",
+  attempts: 0,
+});
+
+/** FIFO gate. Waiting and backoff do not hold a network slot. */
+export class Semaphore {
+  private active = 0;
+  private readonly queue: Array<{ start: () => void; cancel: () => void }> = [];
+  constructor(public limit: number) {}
+
+  async acquire(signal?: AbortSignal): Promise<() => void> {
+    signal?.throwIfAborted();
+    return new Promise((resolve, reject) => {
+      const waiter = {
+        start: () => {
+          signal?.removeEventListener("abort", waiter.cancel);
+          this.active++;
+          let released = false;
+          resolve(() => {
+            if (released) return;
+            released = true;
+            this.active--;
+            this.drain();
+          });
+        },
+        cancel: () => {
+          const index = this.queue.indexOf(waiter);
+          if (index !== -1) this.queue.splice(index, 1);
+          reject(signal?.reason);
+        },
+      };
+      if (this.active < this.limit) waiter.start();
+      else {
+        this.queue.push(waiter);
+        signal?.addEventListener("abort", waiter.cancel, { once: true });
+      }
+    });
+  }
+
+  resize(limit: number): void {
+    this.limit = limit;
+    this.drain();
+  }
+  private drain(): void {
+    while (this.active < this.limit && this.queue.length) this.queue.shift()!.start();
+  }
+}
+
+// One key-wide cap shared by overlapping scrape calls. Per-call limits are separate,
+// so two callers with different overrides cannot accidentally multiply the key limit.
+const sharedLimits = new Map<string, { semaphore: Semaphore; users: number }>();
+export function keySemaphore(
+  key: string,
+  baseUrl: string,
+  limit: number,
+): { semaphore: Semaphore; release: () => void } {
+  const identity = createHash("sha256")
+    .update(JSON.stringify([key, baseUrl]))
+    .digest("hex");
+  let entry = sharedLimits.get(identity);
+  if (!entry) {
+    entry = { semaphore: new Semaphore(limit), users: 0 };
+    sharedLimits.set(identity, entry);
+  } else entry.semaphore.resize(limit);
+  entry.users++;
+  const held = entry;
+  return {
+    semaphore: held.semaphore,
+    release: () => {
+      held.users--;
+      if (held.users === 0) sharedLimits.delete(identity);
+    },
+  };
+}
+
+export function shouldRetry(error: LitescrapeError): boolean {
+  if (error instanceof TransportError) return error.retryable;
+  return (
+    error instanceof APIError &&
+    (error.retryable ||
+      error.statusCode === 429 ||
+      (error.statusCode ?? 0) >= 500 ||
+      ["proxy_capacity_unavailable", "upstream_session_unavailable", "service_unavailable"].includes(
+        error.errorCode,
+      ))
+  );
+}
+
+export function backoff(attempt: number, retryAfter: number | null = null): number {
+  if (retryAfter !== null) return Math.min(Math.max(retryAfter, 0), 30);
+  const ceiling = Math.min(30, 2 ** Math.min(Math.max(attempt - 1, 0), 6));
+  return ceiling * (0.5 + Math.random() / 2);
+}
+
+export async function sleep(seconds: number, signal?: AbortSignal): Promise<void> {
+  signal?.throwIfAborted();
+  try {
+    await delay(seconds * 1000, undefined, { signal });
+  } catch (error) {
+    signal?.throwIfAborted();
+    throw error;
+  }
+}
+
+export interface RuntimeOptions {
+  baseUrl: string;
+  apiKey: string;
+  attempts: number;
+  timeout: number;
+  fetch: Fetch;
+  signal?: AbortSignal;
+}
+interface RequestOptions extends RuntimeOptions {
+  params?: Record<string, string>;
+  method?: "GET" | "POST";
+  body?: Record<string, string>;
+  headers?: Record<string, string>;
+  acceptedStatuses?: number[];
+  semaphores?: Semaphore[];
+  gate?: () => APIError | null;
+  onOutcome?: (outcome: Outcome) => void;
+}
+
+export async function requestWithRetries(path: string, options: RequestOptions): Promise<Outcome> {
+  const { signal } = options;
+  const outcome = emptyOutcome();
+  for (let attempt = 1; attempt <= options.attempts; attempt++) {
+    signal?.throwIfAborted();
+    const release: Array<() => void> = [];
+    try {
+      for (const semaphore of options.semaphores ?? []) release.push(await semaphore.acquire(signal));
+      signal?.throwIfAborted();
+      const blocked = options.gate?.();
+      if (blocked) {
+        outcome.error = blocked;
+        outcome.statusCode = blocked.statusCode;
+        outcome.requestId = blocked.requestId;
+        return outcome;
+      }
+      Object.assign(outcome, emptyOutcome(), { attempts: attempt });
+      const controller = new AbortController();
+      const timeout = setTimeout(
+        () => controller.abort(new DOMException("HTTP transport timed out", "TimeoutError")),
+        options.timeout * 1000,
+      );
+      const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+      try {
+        const url = new URL(options.baseUrl + path);
+        for (const [key, value] of Object.entries(options.params ?? {})) url.searchParams.set(key, value);
+        const response = await options.fetch(url, {
+          method: options.method ?? "GET",
+          redirect: "manual",
+          signal: combined,
+          headers: {
+            Accept: "application/json",
+            "User-Agent": USER_AGENT,
+            "X-Litescrape-Client": CLIENT_HEADER_VALUE,
+            Authorization: `Bearer ${options.apiKey}`,
+            ...(options.body ? { "Content-Type": "application/json" } : {}),
+            ...options.headers,
+          },
+          ...(options.body ? { body: JSON.stringify(options.body) } : {}),
+        });
+        outcome.statusCode = response.status;
+        outcome.requestId = response.headers.get("x-request-id") ?? "";
+        let data: unknown;
+        let bodyError: unknown;
+        try {
+          data = await response.json();
+        } catch (error) {
+          bodyError = error;
+        }
+        signal?.throwIfAborted();
+        if (!(options.acceptedStatuses ?? [200]).includes(response.status)) {
+          outcome.error = apiError(response.status, data, response.headers);
+          outcome.requestId = outcome.error instanceof APIError ? outcome.error.requestId : outcome.requestId;
+        } else if (bodyError) {
+          outcome.error = new TransportError("Response body was not JSON", {
+            retryable: true,
+            cause: bodyError,
+          });
+        } else if (!isObject(data)) {
+          outcome.error = new TransportError("Response body was not a JSON object", { retryable: true });
+        } else outcome.data = data;
+      } catch (cause) {
+        signal?.throwIfAborted();
+        const retryable =
+          controller.signal.aborted ||
+          cause instanceof TypeError ||
+          (cause instanceof DOMException && ["AbortError", "TimeoutError"].includes(cause.name));
+        outcome.error = new TransportError(cause instanceof Error ? cause.message : String(cause), {
+          retryable,
+          cause,
+        });
+      } finally {
+        clearTimeout(timeout);
+      }
+      // Set the fatal gate while the slot is held, before queued requests can start.
+      options.onOutcome?.(outcome);
+    } finally {
+      for (const unlock of release.reverse()) unlock();
+    }
+    if (!outcome.error || attempt === options.attempts || !shouldRetry(outcome.error)) return outcome;
+    await sleep(
+      backoff(attempt, outcome.error instanceof APIError ? outcome.error.retryAfter : null),
+      signal,
+    );
+  }
+  return outcome;
+}
+
+/** A bounded worker pool; errors stop new work and settle all workers before returning. */
+export async function each<T>(
+  items: readonly T[],
+  limit: number,
+  operation: (item: T) => Promise<void>,
+): Promise<void> {
+  let next = 0;
+  let failed = false;
+  const worker = async () => {
+    while (!failed && next < items.length) {
+      const item = items[next++]!;
+      try {
+        await operation(item);
+      } catch (error) {
+        failed = true;
+        throw error;
+      }
+    }
+  };
+  const settled = await Promise.allSettled(Array.from({ length: Math.min(limit, items.length) }, worker));
+  const failure = settled.find((result) => result.status === "rejected");
+  if (failure?.status === "rejected") throw failure.reason;
+}

package/src/stores.ts

@@ -0,0 +1,206 @@
+import { isDeepStrictEqual } from "node:util";
+import { z } from "zod";
+import { isObject } from "./errors.js";
+import { optional as opt, has, issue, type Rule } from "./validation.js";
+
+const cleanText = z
+  .string()
+  .refine((value) => !/[\x00-\x1f\x7f]/.test(value), "Control characters are not allowed")
+  .refine(
+    (value) => !Array.from(value).some((c) => c.length === 1 && /[\ud800-\udfff]/.test(c)),
+    "Use valid Unicode text",
+  );
+const query = cleanText
+  .refine((value) => Buffer.byteLength(value.trim(), "utf8") <= 2048, "Use at most 2,048 UTF-8 bytes")
+  .trim()
+  .min(1)
+  .max(2048);
+const whole = z
+  .union([
+    z.number().int(),
+    cleanText
+      .trim()
+      .regex(/^[+-]?\d+$/)
+      .transform(Number),
+  ])
+  .pipe(z.number().int());
+const category = cleanText
+  .min(1)
+  .max(128)
+  .regex(/^[a-zA-Z0-9_-]+$/);
+const identifier = cleanText
+  .min(1)
+  .max(512)
+  .regex(/^[a-zA-Z0-9_.-]+$/);
+const token = cleanText
+  .min(1)
+  .max(65536)
+  .regex(/^[a-zA-Z0-9_=+/:.-]+$/);
+const age = z.enum(["AGE_RANGE1", "AGE_RANGE2", "AGE_RANGE3"]);
+const store = z.enum(["apps", "books", "audiobooks", "movies", "tv"]).default("apps");
+const playLocale = {
+  hl: cleanText
+    .max(32)
+    .regex(/^[a-zA-Z]{2,3}(?:[-_][a-zA-Z0-9]{2,8}){0,3}$/)
+    .default("en"),
+  gl: cleanText
+    .regex(/^[a-zA-Z]{2}$/)
+    .toLowerCase()
+    .default("us"),
+};
+const listing = {
+  ...playLocale,
+  q: opt(query),
+  chart: opt(category),
+  next_page_token: opt(token),
+  section_page_token: opt(token),
+  see_more_token: opt(token),
+};
+export const playApps = {
+  ...listing,
+  apps_category: opt(category),
+  age: opt(age),
+  store_device: opt(z.enum(["phone", "tablet", "tv", "chromebook", "watch", "car"])),
+};
+export const playGames = {
+  ...listing,
+  games_category: opt(category),
+  store_device: opt(z.enum(["phone", "tablet", "tv", "chromebook", "watch", "windows"])),
+};
+export const playBooks = {
+  ...listing,
+  books_category: opt(category),
+  age: opt(age),
+  price: opt(whole.pipe(z.number().min(1).max(2))),
+};
+export const playMovies = { ...listing, movies_category: opt(category), age: opt(age) };
+export const playProduct = { ...playLocale, product_id: identifier, store, season_id: opt(identifier) };
+export const playReviews = {
+  ...playLocale,
+  product_id: identifier,
+  store,
+  platform: z.enum(["phone", "tablet", "watch", "chromebook", "tv"]).default("phone"),
+  rating: opt(whole.pipe(z.number().min(1).max(5))),
+  sort_by: whole.pipe(z.number().min(1).max(3)).default(1),
+  num: whole.pipe(z.number().min(1).max(199)).default(40),
+  next_page_token: opt(token),
+};
+
+const tokenFields = ["next_page_token", "section_page_token", "see_more_token"];
+export const playRules = (operation: string): Rule[] => [
+  (p, c) => {
+    if (["apps", "games", "books", "movies"].includes(operation)) {
+      if ([...tokenFields, "chart"].filter((f) => p[f]).length > 1)
+        issue(c, "chart and pagination selectors are mutually exclusive");
+      if (p.chart && p.q) issue(c, "chart and q are mutually exclusive");
+      if (p.chart && has(p, "store_device") && p.store_device !== "phone")
+        issue(c, "Device-specific storefronts do not support charts; omit chart or use phone");
+      const categoryValue = p[`${operation}_category`];
+      if (p.q && categoryValue) issue(c, "q and category are mutually exclusive");
+      if (has(p, "store_device") && (p.q || categoryValue)) issue(c, "store_device excludes q and category");
+      if (p.age && categoryValue !== (operation === "books" ? "coll_1689" : "FAMILY"))
+        issue(c, "age requires the children's category");
+      if (has(p, "price") && !p.q) issue(c, "price requires q");
+    }
+    if (p.season_id && p.store !== "tv") issue(c, "season_id requires store=tv");
+
+    const context = Object.fromEntries(
+      Object.entries(p).filter(([key, value]) => value != null && ![...tokenFields, "chart"].includes(key)),
+    );
+    if (["apps", "games"].includes(operation) && !p.q && !p[`${operation}_category`])
+      context.store_device ??= "phone";
+    for (const field of tokenFields) {
+      const encoded = p[field];
+      if (typeof encoded !== "string") continue;
+      const kind =
+        field === "next_page_token"
+          ? operation === "reviews"
+            ? "reviews"
+            : "page"
+          : field === "section_page_token"
+            ? "section"
+            : "collection";
+      try {
+        if (!/^[a-zA-Z0-9+/_-]*={0,2}$/.test(encoded) || encoded.replace(/=+$/, "").length % 4 === 1)
+          throw new Error();
+        const decoded: unknown = JSON.parse(
+          new TextDecoder("utf8", { fatal: true }).decode(Buffer.from(encoded, "base64url")),
+        );
+        if (
+          !isObject(decoded) ||
+          decoded.v !== 1 ||
+          decoded.operation !== operation ||
+          decoded.kind !== kind ||
+          !isDeepStrictEqual(decoded.context, context) ||
+          typeof decoded.token !== "string" ||
+          !/^[a-zA-Z0-9_=+/:.-]{1,12000}$/.test(decoded.token)
+        )
+          throw new Error();
+      } catch {
+        issue(c, "Use a returned token with the same operation and search parameters", [field]);
+      }
+    }
+  },
+];
+
+const storefronts = new Set(
+  "ae af ag ai al am ao ar at au az ba bb be bf bg bh bj bm bn bo br bs bt bw by bz ca cd cg ch ci cl cm cn co cr cv cy cz de dk dm do dz ec ee eg es fi fj fm fr ga gb gd ge gh gm gr gt gw gy hk hn hr hu id ie il in iq is it jm jo jp ke kg kh kn kr kw ky kz la lb lc lk lr lt lu lv ly ma md me mg mk ml mm mn mo mr ms mt mu mv mw mx my mz na ne ng ni nl no np nr nz om pa pe pg ph pk pl pt pw py qa ro rs ru rw sa sb sc se sg si sk sl sn sr st sv sz tc td th tj tm tn to tr tt tw tz ua ug us uy uz vc ve vg vn vu xk ye za zm zw".split(
+    " ",
+  ),
+);
+const appleLocale = {
+  country: cleanText
+    .regex(/^[a-zA-Z]{2}$/)
+    .toLowerCase()
+    .transform((value) => (value === "uk" ? "gb" : value))
+    .refine((value) => storefronts.has(value), "Choose a supported Apple storefront country")
+    .default("us"),
+};
+const productId = cleanText.regex(/^[1-9][0-9]{0,19}$/);
+const quotePlus = (value: string) =>
+  encodeURIComponent(value)
+    .replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)
+    .replace(/%20/g, "+");
+export const appleSearch = {
+  ...appleLocale,
+  term: query.refine((value) => {
+    try {
+      return quotePlus(value).length <= 4096;
+    } catch {
+      return false;
+    }
+  }, "Use at most 4,096 URL-encoded bytes"),
+  lang: cleanText
+    .regex(/^[a-zA-Z]{2,3}-[a-zA-Z]{2}$/)
+    .toLowerCase()
+    .default("en-us"),
+  num: whole.pipe(z.number().min(1).max(200)).default(10),
+  disallow_explicit: z
+    .union([z.boolean(), z.enum(["true", "false"])])
+    .transform((value) => value === true || value === "true")
+    .default(false),
+  property: opt(z.literal("developer")),
+  category_id: opt(
+    whole.pipe(
+      z
+        .number()
+        .min(1)
+        .max(2 ** 31 - 1),
+    ),
+  ),
+  device: z.enum(["mobile", "tablet", "desktop"]).default("mobile"),
+};
+export const appleProduct = { ...appleLocale, product_id: productId, type: z.literal("app").default("app") };
+export const appleReviews = {
+  ...appleLocale,
+  product_id: productId,
+  sort: z.enum(["mostrecent", "mosthelpful"]).default("mostrecent"),
+  page: whole
+    .pipe(
+      z
+        .number()
+        .min(1)
+        .max(2 ** 31 - 1),
+    )
+    .default(1),
+};

package/src/validation.ts

@@ -0,0 +1,140 @@
+import { z } from "zod";
+import { ValidationError } from "./errors.js";
+
+export const optional = <S extends z.ZodType>(schema: S) => schema.nullish();
+export const text = z.union([z.string(), z.number().finite(), z.bigint()]).transform(String);
+export const number = z
+  .union([
+    z.number(),
+    z
+      .string()
+      .trim()
+      .regex(/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/)
+      .transform(Number),
+    z.boolean().transform(Number),
+  ])
+  .pipe(z.number().finite());
+export const integer = number.pipe(z.number().int());
+export const boolean = z
+  .union([
+    z.boolean(),
+    z.literal(0),
+    z.literal(1),
+    z
+      .string()
+      .toLowerCase()
+      .pipe(z.enum(["0", "1", "true", "false", "yes", "no", "on", "off", "t", "f", "y", "n"])),
+  ])
+  .transform((value) => [true, 1, "1", "true", "yes", "on", "t", "y"].includes(value));
+export const flag = z
+  .union([z.enum(["0", "1"]), z.literal(0), z.literal(1), z.boolean()])
+  .transform((value) => (value === true || value === 1 || value === "1" ? ("1" as const) : ("0" as const)));
+export const strictBoolean = z.union([z.boolean(), z.enum(["true", "false"])]);
+export const timeoutSeconds = z.number().finite().gt(0).max(90);
+export const device = z.enum(["desktop", "tablet", "mobile"]);
+
+type Params = Record<string, unknown>;
+export type Rule = (params: Params, context: z.RefinementCtx) => void;
+export const issue = (context: z.RefinementCtx, message: string, path: string[] = []) =>
+  context.addIssue({ code: "custom", message, path });
+export const has = (params: Params, field: string) => params[field] !== null && params[field] !== undefined;
+export const requireAny =
+  (...fields: string[]): Rule =>
+  (p, c) => {
+    if (!fields.some((f) => has(p, f))) issue(c, `one of ${fields.join(", ")} is required`);
+  };
+export const exactlyOne =
+  (...fields: string[]): Rule =>
+  (p, c) => {
+    if (fields.filter((f) => has(p, f)).length !== 1)
+      issue(c, `exactly one of ${fields.join(", ")} is required`);
+  };
+export const atMostOne =
+  (...fields: string[]): Rule =>
+  (p, c) => {
+    const found = fields.filter((f) => has(p, f));
+    if (found.length > 1) issue(c, `${found.join(" and ")} cannot be combined`);
+  };
+export const paired =
+  (a: string, b: string): Rule =>
+  (p, c) => {
+    if (has(p, a) !== has(p, b)) issue(c, `${a} and ${b} must be given together`);
+  };
+export const requires =
+  (field: string, dependency: string): Rule =>
+  (p, c) => {
+    if (has(p, field) && !has(p, dependency)) issue(c, `${field} requires ${dependency}`);
+  };
+
+/** Validated, immutable request. Construct an endpoint class or pass a plain object to scrape. */
+export class ScrapeRequest<E extends string = string> {
+  declare readonly endpoint: E;
+  declare readonly timeout?: number | null;
+  readonly #path: string;
+
+  constructor(path: string, params: object & { endpoint: E }, prototype?: object) {
+    this.#path = path;
+    Object.assign(this, params);
+    if (prototype) Object.setPrototypeOf(this, prototype);
+    Object.freeze(this);
+  }
+
+  get path(): string {
+    return this.#path;
+  }
+
+  queryParams(): Record<string, string> {
+    return Object.fromEntries(
+      Object.entries(this)
+        .filter(([key, value]) => key !== "endpoint" && value != null)
+        .map(([key, value]) => [key, String(value)]),
+    );
+  }
+
+  query_params(): Record<string, string> {
+    return this.queryParams();
+  }
+
+  /** Copy a validated request with a default deadline without revalidating its cursor context. */
+  withTimeout(timeout: number): this {
+    const checked = timeoutSeconds.parse(timeout);
+    return new ScrapeRequest(this.path, { ...this, timeout: checked }, Object.getPrototypeOf(this)) as this;
+  }
+}
+
+export interface RequestClass<E extends string, Input, Output> {
+  new (params: Input): ScrapeRequest<E> & Readonly<Output>;
+  readonly endpoint: E;
+  readonly path: string;
+  readonly parameters: readonly string[];
+}
+
+export function defineRequest<E extends string, Shape extends z.ZodRawShape>(
+  endpoint: E,
+  path: string,
+  shape: Shape,
+  rules: Rule[] = [],
+): RequestClass<E, z.input<z.ZodObject<Shape>>, z.output<z.ZodObject<Shape>>> {
+  const schema = z.strictObject(shape).superRefine((params, ctx) => {
+    for (const rule of rules) rule(params, ctx);
+  });
+  class Request extends ScrapeRequest<E> {
+    static readonly endpoint = endpoint;
+    static readonly path = path;
+    static readonly parameters = Object.freeze(Object.keys(shape));
+    constructor(params: z.input<typeof schema>) {
+      const result = schema.safeParse(params);
+      if (!result.success)
+        throw new ValidationError([
+          [
+            0,
+            result.error.issues
+              .map((i) => `${i.path.join(".")}${i.path.length ? ": " : ""}${i.message}`)
+              .join("; "),
+          ],
+        ]);
+      super(path, { ...result.data, endpoint });
+    }
+  }
+  return Request as unknown as RequestClass<E, z.input<z.ZodObject<Shape>>, z.output<z.ZodObject<Shape>>>;
+}

package/src/version.ts

@@ -0,0 +1 @@
+export const VERSION = "0.1.0";