@socialgouv/fiches-travail-data-types 4.625.0 → 4.627.0

package/build/fetch-data/index.js

@@ -8,15 +8,12 @@ const fs_1 = __importDefault(require("fs"));
 const got_1 = __importDefault(require("got"));
 const p_limit_1 = __importDefault(require("p-limit"));
 const path_1 = __importDefault(require("path"));
-const generateHeaders_1 = require("./generateHeaders");
+const injectToken_1 = require("./injectToken");
 const scrapUrl_1 = require("./scrapUrl");
 const FEED_URL = "https://travail-emploi.gouv.fr/?page=oseo_json";
 const limit = (0, p_limit_1.default)(10);
 async function fetchFeed(url) {
-    const response = await got_1.default.post(url, {
-        headers: (0, generateHeaders_1.generateHeaders)({
-            "Content-Type": "application/json",
-        }),
+    const response = await got_1.default.post((0, injectToken_1.injectToken)(url), {
         http2: true,
         retry: 3,
     });

package/build/fetch-data/injectToken.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Inject  a specific token to bypass bot protection
+ */
+export function injectToken(url: any): string;

package/build/fetch-data/injectToken.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.injectToken = void 0;
+/**
+ * Inject  a specific token to bypass bot protection
+ */
+function injectToken(url) {
+    if (!process.env.TOKEN_MT) {
+        throw Error("Token (cgtoken) is required to fetch the data. This token is provided by the travail-emploi.gouv.fr team.");
+    }
+    if (url.includes("?")) {
+        return `${url}&cgtoken=${process.env.TOKEN_MT}`;
+    }
+    else {
+        return `${url}?cgtoken=${process.env.TOKEN_MT}`;
+    }
+}
+exports.injectToken = injectToken;

package/build/fetch-data/scrapUrl.js

@@ -6,23 +6,20 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.scrapUrl = void 0;
 const got_1 = __importDefault(require("got"));
 const jsdom_1 = require("jsdom");
-const generateHeaders_1 = require("./generateHeaders");
+const injectToken_1 = require("./injectToken");
 const parseDom_1 = require("./parseDom");
 async function scrapUrl(id, url) {
-    const headers = (0, generateHeaders_1.generateHeaders)();
     try {
-        let response = await (0, got_1.default)(url, {
+        let response = await (0, got_1.default)((0, injectToken_1.injectToken)(url), {
             followRedirect: true,
-            headers,
             http2: true,
             retry: 3,
         });
         if (/HTTP 30\d/.test(response.body)) {
             const [, redirectUrl] = response.body.match(/href="(.*)"/);
             try {
-                response = await (0, got_1.default)(redirectUrl, {
+                response = await (0, got_1.default)((0, injectToken_1.injectToken)(redirectUrl), {
                     followRedirect: true,
-                    headers,
                     http2: true,
                     retry: 3,
                 });
@@ -41,7 +38,7 @@ async function scrapUrl(id, url) {
             err = new Error(`Parsing Error: ${error.message}`);
         }
         else if (error instanceof got_1.default.HTTPError) {
-            err = new Error(`HTTP Error: ${error.response.statusCode} - ${error.options.url.href} - ${error.message}`);
+            err = new Error(`HTTP Error: ${error.response.statusCode} - ${url} - ${error.message}`);
         }
         else {
             err = new Error(error.message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@socialgouv/fiches-travail-data-types",
-  "version": "4.625.0",
+  "version": "4.627.0",
   "main": "build/index.js",
   "module": "build/index.js",
   "files": [

package/build/fetch-data/generateHeaders.d.ts

@@ -1,4 +0,0 @@
-/**
- * Build the header for request with a specific token to bypass bot protection
- */
-export function generateHeaders(extras: any): any;

package/build/fetch-data/generateHeaders.js

@@ -1,16 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateHeaders = void 0;
-/**
- * Build the header for request with a specific token to bypass bot protection
- */
-function generateHeaders(extras) {
-    if (!process.env.TOKEN_MT) {
-        throw Error("Token (cgtoken) is required to fetch the data. This token is provided by the travail-emploi.gouv.fr team.");
-    }
-    return {
-        ...extras,
-        Cookie: `cgtoken=${process.env.TOKEN_MT};`,
-    };
-}
-exports.generateHeaders = generateHeaders;