@xcrap/got-scraping-client 0.0.1 → 0.0.4

package/README.md

@@ -73,7 +73,7 @@ const client = new GotScraingClient({ userAgent: "Mozilla/5.0 (Windows NT 10.0;
 ```ts
 function randomUserAgent() {
     const userAgents = [
-    "Mozilla/5.0 (iPhone; CPU iPhone OS 9_8_4; like Mac OS X) AppleWebKit/603.37 (KHTML, like Gecko) Chrome/54.0.1244.188 Mobile Safari/601.5", "Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/537.35 (KHTML, like Gecko) Chrome/47.0.1707.185 Safari/601"
+        "Mozilla/5.0 (iPhone; CPU iPhone OS 9_8_4; like Mac OS X) AppleWebKit/603.37 (KHTML, like Gecko) Chrome/54.0.1244.188 Mobile Safari/601.5", "Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/537.35 (KHTML, like Gecko) Chrome/47.0.1707.185 Safari/601"
     ]
 
     const randomIndex = Math.floor(Math.random() * userAgents.length)

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+import { ClientInterface, ClientFetchManyOptions, ClientRequestOptions, BaseClient, BaseClientOptions, HttpResponse } from "@xcrap/core";
+import { ExtendedExtendOptions, GotScraping } from "got-scraping";
+export type GotScrapingProxy = string;
+export type GotScrapingRequestOptions = ClientRequestOptions & ExtendedExtendOptions;
+export type GotScrapingFetchManyOptions = ClientFetchManyOptions<GotScrapingRequestOptions>;
+export type GotScrapingClientOptions = BaseClientOptions<GotScrapingProxy> & ExtendedExtendOptions;
+export declare class GotScrapingClient extends BaseClient<GotScrapingProxy> implements ClientInterface {
+    readonly options: GotScrapingClientOptions;
+    protected gotScrapingInstance: GotScraping | undefined;
+    constructor(options?: GotScrapingClientOptions);
+    protected initGotScraping(): Promise<void>;
+    protected ensureGotScraping(): Promise<void>;
+    fetch({ url, maxRetries, retries, retryDelay, method, ...gotOptions }: GotScrapingRequestOptions): Promise<HttpResponse>;
+    fetchMany({ requests, concurrency, requestDelay }: GotScrapingFetchManyOptions): Promise<HttpResponse[]>;
+}

package/dist/index.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GotScrapingClient = void 0;
+const core_1 = require("@xcrap/core");
+const load_esm_1 = require("load-esm");
+class GotScrapingClient extends core_1.BaseClient {
+    constructor(options = {}) {
+        super(options);
+        this.options = options;
+    }
+    async initGotScraping() {
+        var _a;
+        const module = await (0, load_esm_1.loadEsm)("got-scraping");
+        const gotScraping = module.gotScraping;
+        this.gotScrapingInstance = gotScraping.extend({
+            ...this.options,
+            headers: {
+                "User-Agent": (_a = this.currentUserAgent) !== null && _a !== void 0 ? _a : core_1.defaultUserAgent
+            },
+            proxyUrl: this.currentProxy,
+        });
+    }
+    async ensureGotScraping() {
+        if (!this.gotScrapingInstance) {
+            await this.initGotScraping();
+        }
+    }
+    async fetch({ url, maxRetries = 0, retries = 0, retryDelay, method = "GET", ...gotOptions }) {
+        await this.ensureGotScraping();
+        const failedAttempts = [];
+        const attemptRequest = async (currentRetry) => {
+            var _a, _b, _c, _d, _e;
+            try {
+                const fullUrl = this.currentProxyUrl ? `${this.currentProxyUrl}${url}` : url;
+                const response = await this.gotScrapingInstance({
+                    url: fullUrl,
+                    method: method,
+                    headers: {
+                        "User-Agent": (_a = this.currentUserAgent) !== null && _a !== void 0 ? _a : core_1.defaultUserAgent,
+                        ...gotOptions.headers
+                    },
+                    proxyUrl: this.currentProxy,
+                    ...gotOptions
+                });
+                if (!this.isSuccess(response.statusCode)) {
+                    throw new core_1.InvalidStatusCodeError(response.statusCode);
+                }
+                return new core_1.HttpResponse({
+                    status: response.statusCode,
+                    statusText: response.statusMessage || "OK",
+                    headers: response.headers,
+                    body: response.body,
+                    attempts: currentRetry + 1,
+                    failedAttempts,
+                });
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : "Unknown error";
+                failedAttempts.push({ error: errorMessage, timestamp: new Date() });
+                if (error.response && currentRetry < maxRetries) {
+                    if (retryDelay !== undefined && retryDelay > 0) {
+                        await (0, core_1.delay)(retryDelay);
+                    }
+                    return await attemptRequest(currentRetry + 1);
+                }
+                return new core_1.HttpResponse({
+                    status: ((_b = error.response) === null || _b === void 0 ? void 0 : _b.statusCode) || 500,
+                    statusText: ((_c = error.response) === null || _c === void 0 ? void 0 : _c.statusMessage) || "Request Failed",
+                    body: ((_d = error.response) === null || _d === void 0 ? void 0 : _d.body) || errorMessage,
+                    headers: ((_e = error.response) === null || _e === void 0 ? void 0 : _e.headers) || {},
+                    attempts: currentRetry + 1,
+                    failedAttempts,
+                });
+            }
+        };
+        return await attemptRequest(retries);
+    }
+    async fetchMany({ requests, concurrency, requestDelay }) {
+        const results = [];
+        const executing = [];
+        for (let i = 0; i < requests.length; i++) {
+            const promise = this.executeRequest({
+                request: requests[i],
+                index: i,
+                requestDelay: requestDelay,
+                results: results
+            }).then(() => undefined);
+            executing.push(promise);
+            if (this.shouldThrottle(executing, concurrency)) {
+                await this.handleConcurrency(executing);
+            }
+        }
+        await Promise.all(executing);
+        return results;
+    }
+}
+exports.GotScrapingClient = GotScrapingClient;

package/package.json

@@ -1,24 +1,31 @@
 {
   "name": "@xcrap/got-scraping-client",
-  "version": "0.0.1",
-  "description": "",
-  "main": "index.js",
+  "version": "0.0.4",
+  "description": "Xcrap Got Scraping Client is a package of the Xcrap framework that implements an HTTP client using the Got Scraping library.",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist/*",
+    "!/**/__tests__"
+  ],
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "jest",
+    "build": "tsc"
   },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
+  "keywords": [
+    "web scraping",
+    "xcrap",
+    "http client",
+    "got scraping"
+  ],
+  "author": "Marcuth",
+  "license": "MIT",
   "type": "commonjs",
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^28.0.3",
-    "@rollup/plugin-node-resolve": "^16.0.1",
-    "@rollup/plugin-typescript": "^12.1.2",
     "@types/node": "^22.13.17",
     "header-generator": "^2.1.63",
-    "rollup": "^4.39.0",
     "ts-node": "^10.9.2",
-    "tslib": "^2.8.1",
     "typescript": "^5.8.2"
   },
   "dependencies": {
@@ -28,5 +35,13 @@
   "peerDependencies": {
     "@xcrap/core": "^0.0.3",
     "@xcrap/parser": "^0.0.2"
-  }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Xcrap-Cloud/got-scraping-client.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Xcrap-Cloud/got-scraping-client/issues"
+  },
+  "homepage": "https://github.com/Xcrap-Cloud/got-scraping-client#readme"
 }

package/jest.config.ts

@@ -1,5 +0,0 @@
-export default {
-    preset: "ts-jest",
-    testEnvironment: "node",
-    cache: true
-}

package/rollup.config.js

@@ -1,46 +0,0 @@
-const typescript = require("@rollup/plugin-typescript")
-const resolve = require("@rollup/plugin-node-resolve")
-const commonjs =  require("@rollup/plugin-commonjs")
-const path = require("node:path")
-const fs = require("node:fs")
-
-function getAllTypeScriptFiles(dir) {
-    let files = []
-
-    fs.readdirSync(dir).forEach(file => {
-        const fullPath = path.join(dir, file)
-
-        if (fs.statSync(fullPath).isDirectory()) {
-            files = files.concat(getAllTypeScriptFiles(fullPath))
-        } else if (file.endsWith(".ts")) {
-            files.push(fullPath)
-        }
-    })
-
-    return files
-}
-
-const inputFiles = getAllTypeScriptFiles("src")
-
-module.exports = [
-    {
-        input: inputFiles,
-        output: {
-            dir: "dist",
-            format: "cjs",
-            entryFileNames: "[name].cjs",
-            preserveModules: true,
-        },
-        plugins: [resolve(), commonjs(), typescript()]
-    },
-    {
-        input: inputFiles,
-        output: {
-            dir: "dist",
-            format: "esm",
-            entryFileNames: "[name].mjs",
-            preserveModules: true,
-        },
-        plugins: [resolve(), commonjs(), typescript()]
-    }
-]

package/tsconfig.json

@@ -1,15 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "es2019",                                  
-    "module": "NodeNext",
-    "declaration": true,                              
-    "esModuleInterop": true,                             
-    "forceConsistentCasingInFileNames": true,            
-    "strict": true,                                      
-    "skipLibCheck": true,
-    "outDir": "./dist",
-    "rootDir": "./src"                                 
-  },
-  "include": ["./src"],
-  "exclude": ["./node_modules", "./dist"]
-}