@bluessu/meal-scraper 0.1.0 → 0.1.1

package/dist/index.cjs

@@ -378,17 +378,14 @@ var SoongguriScraper = class {
   constructor(settings, cafeteriaType) {
     this.settings = settings;
     this.cafeteriaType = cafeteriaType;
+    this.challengeRetryLimit = 2;
+    this.cookieJar = {};
   }
   async scrapeMenu(date) {
     const normalizedDate = normalizeSgDate(date);
-    const url = `${this.settings.soongguriBaseUrl}?rcd=${getRcd(this.cafeteriaType, this.settings)}&sdt=${normalizedDate}`;
+    const url = this.buildMenuUrl(normalizedDate);
     try {
-      const res = await import_axios.default.get(url, {
-        timeout: this.settings.timeoutMs,
-        responseType: "text",
-        validateStatus: (s) => s >= 200 && s < 300
-      });
-      const html = String(res.data);
+      const html = await this.fetchWithRetry(url, 0, normalizedDate);
       const hasHoliday = html.includes("\uC624\uB298\uC740 \uC27D\uB2C8\uB2E4.") || html.includes("\uD734\uBB34");
       if (hasHoliday) {
         throw new HolidayException(
@@ -450,11 +447,109 @@ var SoongguriScraper = class {
       );
     }
   }
+  async fetchWithRetry(url, attempt = 0, targetDate) {
+    const response = await import_axios.default.get(url, {
+      timeout: this.settings.timeoutMs,
+      responseType: "text",
+      validateStatus: (s) => s >= 200 && s < 300,
+      headers: this.buildBrowserLikeHeaders(attempt)
+    });
+    this.applySetCookies(response.headers);
+    const html = String(response.data);
+    if (this.isChallengeResponse(html) && attempt < this.challengeRetryLimit) {
+      const nextAttempt = attempt + 1;
+      return this.fetchWithRetry(
+        this.buildMenuUrl(url, nextAttempt),
+        nextAttempt,
+        targetDate
+      );
+    }
+    if (this.isChallengeResponse(html) && attempt >= this.challengeRetryLimit) {
+      throw new MenuFetchException(
+        targetDate,
+        this.cafeteriaType,
+        "\uC790\uB3D9\uB4F1\uB85D\uBC29\uC9C0 \uC6B0\uD68C \uC2E4\uD328",
+        html,
+        {
+          endpoint: url,
+          operation: "scrape",
+          cafeteria: this.cafeteriaType,
+          challengeBypass: true,
+          attempts: attempt,
+          ckattempt: nextChallengeAttempt(attempt)
+        }
+      );
+    }
+    return html;
+  }
+  buildMenuUrl(url, ckattempt) {
+    const parsed = new URL(url);
+    const nextUrl = new URL(`${parsed.origin}${parsed.pathname}`);
+    const params = new URLSearchParams(parsed.search);
+    const rcd = params.get("rcd") ?? String(getRcd(this.cafeteriaType, this.settings));
+    const sdt = params.get("sdt") ?? "";
+    nextUrl.searchParams.set("rcd", rcd);
+    nextUrl.searchParams.set("sdt", sdt);
+    if (typeof ckattempt === "number") {
+      nextUrl.searchParams.set("ckattempt", String(ckattempt));
+    }
+    return nextUrl.toString();
+  }
+  buildBrowserLikeHeaders(attempt) {
+    const headers = {
+      accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+      "accept-language": "ko-KR,ko;q=0.9,en-US;q=0.7,en;q=0.6",
+      "cache-control": "no-cache",
+      pragma: "no-cache",
+      referer: this.settings.soongguriBaseUrl,
+      "sec-fetch-dest": "document",
+      "sec-fetch-mode": "navigate",
+      "sec-fetch-site": "same-origin",
+      "user-agent": "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36"
+    };
+    if (attempt > 0) {
+      headers["upgrade-insecure-requests"] = "1";
+    }
+    const cookie = this.getCookieHeader();
+    if (cookie) {
+      headers.cookie = cookie;
+    }
+    return headers;
+  }
+  getCookieHeader() {
+    return Object.entries(this.cookieJar).filter(([, value]) => value.length > 0).map(([name, value]) => `${name}=${value}`).join("; ");
+  }
+  applySetCookies(headers) {
+    const setCookie = headers["set-cookie"];
+    if (!setCookie) {
+      return;
+    }
+    const rawCookies = Array.isArray(setCookie) ? setCookie : [setCookie];
+    for (const raw of rawCookies) {
+      const tuple = raw.split(";")[0];
+      const separatorIdx = tuple.indexOf("=");
+      if (separatorIdx < 1) {
+        continue;
+      }
+      const name = tuple.slice(0, separatorIdx).trim();
+      const value = tuple.slice(separatorIdx + 1).trim();
+      if (!name) {
+        continue;
+      }
+      this.cookieJar[name] = value;
+    }
+  }
+  isChallengeResponse(html) {
+    return html.includes("\uC790\uB3D9\uB4F1\uB85D\uBC29\uC9C0\uB97C \uC704\uD574 \uBCF4\uC548\uC808\uCC28\uB97C \uAC70\uCE58\uACE0 \uC788\uC2B5\uB2C8\uB2E4.") || html.includes("/___verify") || html.includes("Please prove that you are human.");
+  }
 };
 var normalizeSgDate = (date) => {
   const digits = date.replace(/\D/g, "").slice(0, 8);
   return digits.length === 8 ? digits : date;
 };
+var nextChallengeAttempt = (attempt) => {
+  return attempt + 1;
+};
 
 // src/repositories/scrapers/haksikScraper.ts
 var HaksikScraper = class extends SoongguriScraper {

package/dist/index.mjs

@@ -333,17 +333,14 @@ var SoongguriScraper = class {
   constructor(settings, cafeteriaType) {
     this.settings = settings;
     this.cafeteriaType = cafeteriaType;
+    this.challengeRetryLimit = 2;
+    this.cookieJar = {};
   }
   async scrapeMenu(date) {
     const normalizedDate = normalizeSgDate(date);
-    const url = `${this.settings.soongguriBaseUrl}?rcd=${getRcd(this.cafeteriaType, this.settings)}&sdt=${normalizedDate}`;
+    const url = this.buildMenuUrl(normalizedDate);
     try {
-      const res = await axios.get(url, {
-        timeout: this.settings.timeoutMs,
-        responseType: "text",
-        validateStatus: (s) => s >= 200 && s < 300
-      });
-      const html = String(res.data);
+      const html = await this.fetchWithRetry(url, 0, normalizedDate);
       const hasHoliday = html.includes("\uC624\uB298\uC740 \uC27D\uB2C8\uB2E4.") || html.includes("\uD734\uBB34");
       if (hasHoliday) {
         throw new HolidayException(
@@ -405,11 +402,109 @@ var SoongguriScraper = class {
       );
     }
   }
+  async fetchWithRetry(url, attempt = 0, targetDate) {
+    const response = await axios.get(url, {
+      timeout: this.settings.timeoutMs,
+      responseType: "text",
+      validateStatus: (s) => s >= 200 && s < 300,
+      headers: this.buildBrowserLikeHeaders(attempt)
+    });
+    this.applySetCookies(response.headers);
+    const html = String(response.data);
+    if (this.isChallengeResponse(html) && attempt < this.challengeRetryLimit) {
+      const nextAttempt = attempt + 1;
+      return this.fetchWithRetry(
+        this.buildMenuUrl(url, nextAttempt),
+        nextAttempt,
+        targetDate
+      );
+    }
+    if (this.isChallengeResponse(html) && attempt >= this.challengeRetryLimit) {
+      throw new MenuFetchException(
+        targetDate,
+        this.cafeteriaType,
+        "\uC790\uB3D9\uB4F1\uB85D\uBC29\uC9C0 \uC6B0\uD68C \uC2E4\uD328",
+        html,
+        {
+          endpoint: url,
+          operation: "scrape",
+          cafeteria: this.cafeteriaType,
+          challengeBypass: true,
+          attempts: attempt,
+          ckattempt: nextChallengeAttempt(attempt)
+        }
+      );
+    }
+    return html;
+  }
+  buildMenuUrl(url, ckattempt) {
+    const parsed = new URL(url);
+    const nextUrl = new URL(`${parsed.origin}${parsed.pathname}`);
+    const params = new URLSearchParams(parsed.search);
+    const rcd = params.get("rcd") ?? String(getRcd(this.cafeteriaType, this.settings));
+    const sdt = params.get("sdt") ?? "";
+    nextUrl.searchParams.set("rcd", rcd);
+    nextUrl.searchParams.set("sdt", sdt);
+    if (typeof ckattempt === "number") {
+      nextUrl.searchParams.set("ckattempt", String(ckattempt));
+    }
+    return nextUrl.toString();
+  }
+  buildBrowserLikeHeaders(attempt) {
+    const headers = {
+      accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+      "accept-language": "ko-KR,ko;q=0.9,en-US;q=0.7,en;q=0.6",
+      "cache-control": "no-cache",
+      pragma: "no-cache",
+      referer: this.settings.soongguriBaseUrl,
+      "sec-fetch-dest": "document",
+      "sec-fetch-mode": "navigate",
+      "sec-fetch-site": "same-origin",
+      "user-agent": "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36"
+    };
+    if (attempt > 0) {
+      headers["upgrade-insecure-requests"] = "1";
+    }
+    const cookie = this.getCookieHeader();
+    if (cookie) {
+      headers.cookie = cookie;
+    }
+    return headers;
+  }
+  getCookieHeader() {
+    return Object.entries(this.cookieJar).filter(([, value]) => value.length > 0).map(([name, value]) => `${name}=${value}`).join("; ");
+  }
+  applySetCookies(headers) {
+    const setCookie = headers["set-cookie"];
+    if (!setCookie) {
+      return;
+    }
+    const rawCookies = Array.isArray(setCookie) ? setCookie : [setCookie];
+    for (const raw of rawCookies) {
+      const tuple = raw.split(";")[0];
+      const separatorIdx = tuple.indexOf("=");
+      if (separatorIdx < 1) {
+        continue;
+      }
+      const name = tuple.slice(0, separatorIdx).trim();
+      const value = tuple.slice(separatorIdx + 1).trim();
+      if (!name) {
+        continue;
+      }
+      this.cookieJar[name] = value;
+    }
+  }
+  isChallengeResponse(html) {
+    return html.includes("\uC790\uB3D9\uB4F1\uB85D\uBC29\uC9C0\uB97C \uC704\uD574 \uBCF4\uC548\uC808\uCC28\uB97C \uAC70\uCE58\uACE0 \uC788\uC2B5\uB2C8\uB2E4.") || html.includes("/___verify") || html.includes("Please prove that you are human.");
+  }
 };
 var normalizeSgDate = (date) => {
   const digits = date.replace(/\D/g, "").slice(0, 8);
   return digits.length === 8 ? digits : date;
 };
+var nextChallengeAttempt = (attempt) => {
+  return attempt + 1;
+};
 
 // src/repositories/scrapers/haksikScraper.ts
 var HaksikScraper = class extends SoongguriScraper {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bluessu/meal-scraper",
   "private": false,
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "A meal menu scraper for dormitory meal info.",
   "type": "commonjs",
   "author": "bluessu",
@@ -36,6 +36,18 @@
     }
   },
   "funding": "https://github.com/blue-ssu/meal-scraper",
+  "scripts": {
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "build": "tsup",
+    "clean": "rimraf dist",
+    "prepare": "pnpm run build",
+    "quality": "pnpm run clean && pnpm run typecheck && pnpm run build",
+    "prepublishOnly": "pnpm run quality",
+    "prepack": "pnpm run quality",
+    "release:patch": "pnpm version patch && pnpm publish --access public",
+    "release:minor": "pnpm version minor && pnpm publish --access public",
+    "release:major": "pnpm version major && pnpm publish --access public"
+  },
   "files": [
     "dist",
     "README.md",
@@ -56,14 +68,5 @@
     "rimraf": "^5.0.9",
     "typescript": "^5.5.4",
     "@types/node": "^22.7.1"
-  },
-  "scripts": {
-    "typecheck": "tsc --noEmit -p tsconfig.json",
-    "build": "tsup",
-    "clean": "rimraf dist",
-    "quality": "pnpm run clean && pnpm run typecheck && pnpm run build",
-    "release:patch": "pnpm version patch && pnpm publish --access public",
-    "release:minor": "pnpm version minor && pnpm publish --access public",
-    "release:major": "pnpm version major && pnpm publish --access public"
   }
-}
+}