@robertying/thu-learn-lib 4.1.0 → 5.0.0

package/src/notices.ts

@@ -4,21 +4,31 @@ import { Base64 } from 'js-base64';
 
 import type { Session } from './request.ts';
 import { $, YES } from './request.ts';
-import type { INotification, INotificationDetail, Notification } from './types.ts';
+import type { INotification, Notification, NotificationListOptions, RemoteFile } from './types.ts';
 import { ContentType, CourseType, FailReason } from './types.ts';
 import * as URLS from './urls.ts';
-import { decodeHTML, trimAndDefine } from './utils.ts';
+import { decodeHTML, isSelected, trimAndDefine } from './utils.ts';
 
-/** Get all notifications （课程公告） of the specified course. */
+/**
+ * Get all notifications （课程公告） of the specified course: current and expired lists.
+ * Attachments only for the notices `options.attachments` selects.
+ */
 export async function getNotificationList(
   s: Session,
   courseID: string,
   courseType: CourseType = CourseType.STUDENT,
+  options: NotificationListOptions = {},
 ): Promise<Notification[]> {
-  return Promise.all([
+  const lists = await Promise.all([
     getNotificationListKind(s, courseID, courseType, false),
     getNotificationListKind(s, courseID, courseType, true),
-  ]).then((r) => r.flat());
+  ]);
+  return Promise.all(
+    lists.flat().map(async (n): Promise<Notification> => {
+      if (!n.attachmentName || !isSelected(options.attachments, n)) return n;
+      return { ...n, attachment: await getNotificationAttachment(s, n, courseType) };
+    }),
+  );
 }
 
 async function getNotificationListKind(
@@ -26,7 +36,7 @@ async function getNotificationListKind(
   courseID: string,
   courseType: CourseType,
   expired: boolean,
-): Promise<Notification[]> {
+): Promise<INotification[]> {
   const json = await (
     await s.fetchWithToken(URLS.LEARN_NOTIFICATION_LIST(courseType, expired), {
       method: 'POST',
@@ -42,38 +52,35 @@ async function getNotificationListKind(
 
   const result = (json.object?.aaData ?? json.object?.resultsList ?? []) as any[];
 
-  return Promise.all(
-    result.map(async (n) => {
-      const notification: INotification = {
-        id: n.ggid,
-        content: Base64.decode(n.ggnr ?? ''),
-        title: decodeHTML(n.bt),
-        url: URLS.LEARN_NOTIFICATION_DETAIL(courseID, n.ggid, courseType),
-        publisher: n.fbrxm,
-        hasRead: n.sfyd === YES,
-        markedImportant: Number(n.sfqd) === 1, // n.sfqd could be string '1' (teacher mode) or number 1 (student mode)
-        publishTime: n.fbsj && typeof n.fbsj === 'string' ? n.fbsj : n.fbsjStr,
-        expireTime: n.jzsj ?? undefined,
-        isFavorite: n.sfsc === YES,
-        comment: n.bznr ?? undefined,
-      };
-      let detail: INotificationDetail = {};
-      const attachmentName = courseType === CourseType.STUDENT ? n.fjmc : n.fjbt;
-      if (attachmentName) {
-        detail = await parseNotificationDetail(s, courseID, notification.id, courseType, attachmentName);
-      }
-      return { ...notification, ...detail } satisfies Notification;
-    }),
-  );
+  return result.map((n) => {
+    const attachmentName: unknown = courseType === CourseType.STUDENT ? n.fjmc : n.fjbt;
+    return {
+      id: n.ggid,
+      content: Base64.decode(n.ggnr ?? ''),
+      title: decodeHTML(n.bt),
+      url: URLS.LEARN_NOTIFICATION_DETAIL(courseID, n.ggid, courseType),
+      publisher: n.fbrxm,
+      hasRead: n.sfyd === YES,
+      markedImportant: Number(n.sfqd) === 1, // n.sfqd could be string '1' (teacher mode) or number 1 (student mode)
+      publishTime: n.fbsj && typeof n.fbsj === 'string' ? n.fbsj : n.fbsjStr,
+      expireTime: n.jzsj ?? undefined,
+      isFavorite: n.sfsc === YES,
+      comment: n.bznr ?? undefined,
+      attachmentName: typeof attachmentName === 'string' && attachmentName ? attachmentName : undefined,
+    } satisfies INotification;
+  });
 }
 
-async function parseNotificationDetail(
+/**
+ * Get the attachment of a notice from its detail page (1 request), or `undefined` without a request if
+ * `attachmentName` says it has none.
+ */
+export async function getNotificationAttachment(
   s: Session,
-  courseID: string,
-  id: string,
-  courseType: CourseType,
-  attachmentName: string,
-): Promise<INotificationDetail> {
+  notification: Pick<INotification, 'url' | 'attachmentName'>,
+  courseType: CourseType = CourseType.STUDENT,
+): Promise<RemoteFile | undefined> {
+  if (!notification.attachmentName) return undefined;
   /// from JSON (backup, currently not used)
   // const postParams = new FormData();
   // postParams.append('id', id);
@@ -84,7 +91,7 @@ async function parseNotificationDetail(
   // })).json();
   // const attachmentId = metadata.ggfjid as string;
   /// parsed from HTML
-  const response = await s.fetchWithToken(URLS.LEARN_NOTIFICATION_DETAIL(courseID, id, courseType));
+  const response = await s.fetchWithToken(notification.url);
   const result = $(await response.text());
   let path = '';
   if (courseType === CourseType.STUDENT) {
@@ -99,12 +106,10 @@ async function parseNotificationDetail(
     path = URLS.LEARN_PREFIX + path;
   }
   return {
-    attachment: {
-      name: attachmentName,
-      id: attachmentId,
-      downloadUrl: path,
-      previewUrl: URLS.LEARN_FILE_PREVIEW(ContentType.NOTIFICATION, attachmentId, courseType, s.previewFirstPage),
-      size,
-    },
+    name: notification.attachmentName,
+    id: attachmentId,
+    downloadUrl: path,
+    previewUrl: URLS.LEARN_FILE_PREVIEW(ContentType.NOTIFICATION, attachmentId, courseType, s.previewFirstPage),
+    size,
   };
 }

package/src/request.ts

@@ -4,7 +4,7 @@ import fetchCookie from 'fetch-cookie';
 import { fetch } from 'node-fetch-native';
 import { CookieJar } from 'tough-cookie';
 
-import type { CredentialProvider, Fetch, HelperConfig } from './types.ts';
+import type { ApiError, CredentialProvider, Fetch, HelperConfig } from './types.ts';
 import { FailReason, Language } from './types.ts';
 
 const CHEERIO_CONFIG: cheerio.CheerioOptions = { xml: true };
@@ -17,6 +17,11 @@ export const noLogin = (res: Response) => res.url.includes('login_timeout') || r
 
 export const YES = '是';
 
+/** default for `HelperConfig.maxConcurrentRequests` */
+export const DEFAULT_MAX_CONCURRENT_REQUESTS = 6;
+/** default for `HelperConfig.requestTimeout`, in milliseconds */
+export const DEFAULT_REQUEST_TIMEOUT = 60_000;
+
 /** add CSRF token to any request URL as parameters */
 export const addCSRFTokenToUrl = (url: string, token: string): string => {
   const newUrl = new URL(url);
@@ -28,32 +33,80 @@ function makeFetch(jar?: CookieJar): typeof fetch {
   return fetchCookie(fetch, jar);
 }
 
-function withReAuth(rawFetch: Fetch, login: () => Promise<void>): Fetch {
-  return async function wrappedFetch(...args) {
-    const retryAfterLogin = async () => {
-      await login();
-      return await rawFetch(...args).then((res: Response) => {
-        if (noLogin(res)) {
-          return Promise.reject({
-            reason: FailReason.NOT_LOGGED_IN,
-          });
-        } else if (res.status !== 200) {
-          return Promise.reject({
-            reason: FailReason.UNEXPECTED_STATUS,
-            extra: {
-              code: res.status,
-              text: res.statusText,
-            },
-          });
-        } else {
-          return res;
-        }
-      });
-    };
-    return await rawFetch(...args).then((res: Response) => (noLogin(res) ? retryAfterLogin() : res));
+const urlOf = (input: Parameters<Fetch>[0]): string =>
+  typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+
+/**
+ * Let at most `max` calls of `inner` run at once; the rest wait in FIFO order. A slot is freed when the response
+ * headers arrive (or the call fails), so reading a body does not hold one.
+ */
+export function limitConcurrency(inner: Fetch, max: number): Fetch {
+  if (!(max >= 1) || max === Infinity) return inner;
+  let active = 0;
+  const waiting: (() => void)[] = [];
+  const acquire = (): Promise<void> => {
+    if (active < max) {
+      active += 1;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => waiting.push(resolve));
+  };
+  const release = () => {
+    const next = waiting.shift();
+    // hand the slot straight to the next caller, so `active` stays the same
+    if (next) next();
+    else active -= 1;
+  };
+  return async (...args) => {
+    await acquire();
+    try {
+      return await inner(...args);
+    } finally {
+      release();
+    }
+  };
+}
+
+/**
+ * Abort a call of `inner` that has no response headers after `ms` milliseconds and reject it with
+ * `FailReason.REQUEST_TIMEOUT`. A caller's own `signal` still works. Uses only `AbortController` and `setTimeout`,
+ * which React Native has (unlike `AbortSignal.timeout` / `AbortSignal.any`).
+ */
+export function withTimeout(inner: Fetch, ms: number): Fetch {
+  if (!(ms > 0) || ms === Infinity) return inner;
+  return async (input, init) => {
+    const controller = new AbortController();
+    const outer = init?.signal ?? undefined;
+    const forward = () => controller.abort(outer?.reason);
+    if (outer?.aborted) forward();
+    else outer?.addEventListener('abort', forward, { once: true });
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      controller.abort();
+    }, ms);
+    try {
+      return await inner(input, { ...init, signal: controller.signal });
+    } catch (err) {
+      if (timedOut) {
+        throw {
+          reason: FailReason.REQUEST_TIMEOUT,
+          extra: { url: urlOf(input), timeout: ms },
+        } as ApiError;
+      }
+      throw err;
+    } finally {
+      clearTimeout(timer);
+      outer?.removeEventListener('abort', forward);
+    }
   };
 }
 
+/** free the connection of a response we won't read */
+const discard = (res: Response) => {
+  res.body?.cancel().catch(() => {});
+};
+
 /**
  * State shared by every request a `Learn2018Helper` makes: cookies, CSRF token, language and the fetch wrappers.
  * Internal; the helper keeps it in a private field.
@@ -61,15 +114,16 @@ function withReAuth(rawFetch: Fetch, login: () => Promise<void>): Fetch {
 export class Session {
   readonly provider?: CredentialProvider;
   readonly cookieJar: CookieJar;
-  /** plain fetch with cookies, no login check */
+  /** fetch with cookies, the concurrency cap and the timeout, but no login check */
   readonly rawFetch: Fetch;
-  /** rejects with `NOT_LOGGED_IN`, or logs in again and retries once when a `CredentialProvider` is set */
-  readonly fetch: Fetch;
-  csrfToken = '';
+  #csrfToken = '';
+  /** bumped whenever the session changes (login, `csrfToken` set), so late failures from an old session don't log in again */
+  #generation = 0;
+  #pendingLogin?: Promise<void>;
   lang = Language.ZH;
   previewFirstPage: boolean;
 
-  /** `login` is called with no arguments when a request needs a (new) session */
+  /** `login` signs in through SSO with the `CredentialProvider`; the session calls it at most once at a time */
   constructor(
     config: HelperConfig | undefined,
     readonly login: () => Promise<void>,
@@ -77,25 +131,84 @@ export class Session {
     this.previewFirstPage = config?.generatePreviewUrlForFirstPage ?? true;
     this.provider = config?.provider;
     this.cookieJar = config?.cookieJar ?? new CookieJar();
-    this.rawFetch = config?.fetch ?? makeFetch(this.cookieJar);
-    this.fetch = this.provider
-      ? withReAuth(this.rawFetch, login)
-      : async (...args) => {
-          const result = await this.rawFetch(...args);
-          if (noLogin(result))
-            return Promise.reject({
-              reason: FailReason.NOT_LOGGED_IN,
-            });
-          return result;
-        };
+    this.rawFetch = limitConcurrency(
+      withTimeout(config?.fetch ?? makeFetch(this.cookieJar), config?.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT),
+      config?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
+    );
   }
 
-  /** like `fetch`, but logs in first if there is no CSRF token yet and adds the token to the URL */
-  readonly fetchWithToken: Fetch = async (...args) => {
-    if (this.csrfToken === '') {
-      await this.login();
+  get csrfToken(): string {
+    return this.#csrfToken;
+  }
+
+  set csrfToken(token: string) {
+    this.#csrfToken = token;
+    this.#generation += 1;
+  }
+
+  /**
+   * Log in with the `CredentialProvider`. Concurrent callers share one SSO login instead of starting one each.
+   */
+  relogin(): Promise<void> {
+    this.#pendingLogin ??= this.login().finally(() => {
+      this.#pendingLogin = undefined;
+    });
+    return this.#pendingLogin;
+  }
+
+  /**
+   * Send a request built by `makeUrl` (called again for the retry, so it can pick up a new CSRF token).
+   * When the session has expired, it rejects with `NOT_LOGGED_IN` if there is no `CredentialProvider`; otherwise it
+   * logs in again (sharing a login already in progress, or skipping it if another request already renewed the
+   * session) and retries once.
+   */
+  async #send(makeUrl: () => string, init?: RequestInit): Promise<Response> {
+    const generation = this.#generation;
+    const res = await this.rawFetch(makeUrl(), init);
+    if (!noLogin(res)) return res;
+    discard(res);
+    if (!this.provider) {
+      throw { reason: FailReason.NOT_LOGGED_IN } as ApiError;
+    }
+    if (this.#generation === generation) {
+      await this.relogin();
+    }
+    const retry = await this.rawFetch(makeUrl(), init);
+    if (noLogin(retry)) {
+      discard(retry);
+      throw { reason: FailReason.NOT_LOGGED_IN } as ApiError;
     }
-    const [url, ...remaining] = args;
-    return this.fetch(addCSRFTokenToUrl(url as string, this.csrfToken), ...remaining);
+    if (retry.status !== 200) {
+      discard(retry);
+      throw {
+        reason: FailReason.UNEXPECTED_STATUS,
+        extra: { code: retry.status, text: retry.statusText },
+      } as ApiError;
+    }
+    return retry;
+  }
+
+  /** fetch with a login check (see `#send`) */
+  readonly fetch: Fetch = (input, init) => this.#send(() => urlOf(input), init);
+
+  /** like `fetch`, but logs in first if there is no CSRF token yet and adds the current token to the URL */
+  readonly fetchWithToken: Fetch = async (input, init) => {
+    if (this.#csrfToken === '') {
+      await this.relogin();
+    }
+    const url = urlOf(input);
+    return this.#send(() => addCSRFTokenToUrl(url, this.#csrfToken), init);
   };
+
+  /**
+   * Make sure the session works: if there is a CSRF token, make one cheap authenticated request (`probeUrl`) and log
+   * in only if it fails; otherwise log in.
+   */
+  async ensureSession(probeUrl: string): Promise<void> {
+    if (this.#csrfToken === '') {
+      await this.relogin();
+      return;
+    }
+    discard(await this.fetchWithToken(probeUrl));
+  }
 }

package/src/types.ts

@@ -14,6 +14,10 @@ export type HelperConfig = {
   fetch?: Fetch;
   cookieJar?: CookieJar;
   generatePreviewUrlForFirstPage?: boolean;
+  /** at most this many requests in flight at once, including login (default 6; `Infinity` for no limit) */
+  maxConcurrentRequests?: number;
+  /** milliseconds to wait for a response before rejecting with `FailReason.REQUEST_TIMEOUT` (default 60000; 0 for none) */
+  requestTimeout?: number;
 };
 
 export enum FailReason {
@@ -27,6 +31,7 @@ export enum FailReason {
   UNEXPECTED_STATUS = 'unexpected status',
   OPERATION_FAILED = 'operation failed',
   ERROR_SETTING_COOKIES = 'could not set cookies',
+  REQUEST_TIMEOUT = 'request timed out',
 }
 
 export interface ApiError {
@@ -78,7 +83,8 @@ interface ICourseInfo {
   name: string;
   chineseName: string;
   englishName: string;
-  timeAndLocation: string[];
+  /** only with `getCourseList(..., { timeAndLocation: true })`; see `getCourseTimeAndLocation` */
+  timeAndLocation?: string[];
   url: string;
   teacherName: string;
   teacherNumber: string;
@@ -111,6 +117,8 @@ export interface INotification {
   expireTime?: string;
   isFavorite: boolean;
   comment?: string;
+  /** name of the attachment, if the notice has one; fetch it with `getNotificationAttachment` */
+  attachmentName?: string;
 }
 
 export interface INotificationDetail {
@@ -362,6 +370,47 @@ export interface IQuestionnaire {
 
 export type Questionnaire = IQuestionnaire;
 
+/** which items get their extra per-item requests: none (default), all, or those the predicate accepts */
+export type DetailSelector<T> = 'none' | 'all' | ((item: T) => boolean);
+
+export interface CourseListOptions {
+  /** fetch each course's time and location (one request per course; default false) */
+  timeAndLocation?: boolean;
+}
+
+export interface HomeworkListOptions {
+  /**
+   * Which homework gets its detail (page + description JSON, 2 requests each); default `'none'`.
+   * The same as calling `getHomeworkDetail` for those items.
+   */
+  details?: DetailSelector<IHomework>;
+}
+
+export interface NotificationListOptions {
+  /**
+   * Which notices with an attachment get it (1 request each); default `'none'`.
+   * The same as calling `getNotificationAttachment` for those items.
+   */
+  attachments?: DetailSelector<INotification>;
+}
+
+export interface AllContentsOptions<T extends ContentType> {
+  courseType?: CourseType;
+  /** homework `details` or notification `attachments`, passed to each course's list */
+  details?: T extends ContentType.HOMEWORK
+    ? DetailSelector<IHomework>
+    : T extends ContentType.NOTIFICATION
+      ? DetailSelector<INotification>
+      : never;
+}
+
+export interface AllContentsResult<T extends ContentType> {
+  /** content of each course that succeeded */
+  contents: CourseContent<T>;
+  /** the error of each course that failed */
+  errors: { [courseID: string]: unknown };
+}
+
 export type ContentTypeMap = {
   [ContentType.NOTIFICATION]: Notification;
   [ContentType.FILE]: File;

package/src/utils.ts

@@ -1,5 +1,6 @@
 import { decodeHTML as _decodeHTML } from 'entities';
 
+import type { DetailSelector } from './types.ts';
 import { ContentType, FailReason, HomeworkGradeLevel, QuestionnaireType, SemesterType } from './types.ts';
 
 export function parseSemesterType(n: number): SemesterType {
@@ -118,3 +119,7 @@ export const QNR_TYPE_MAP = new Map([
   ['填表', QuestionnaireType.FORM],
   ['问卷', QuestionnaireType.SURVEY],
 ]);
+
+/** whether `item` is selected by a `DetailSelector` (`undefined` means `'none'`) */
+export const isSelected = <T>(selector: DetailSelector<T> | undefined, item: T): boolean =>
+  selector === 'all' || (typeof selector === 'function' && selector(item));