@wargas/crawler 0.0.1 → 0.0.2

package/index.test.ts

@@ -0,0 +1,12 @@
+import { expect, test } from 'bun:test'
+import { Crawler } from "./index";
+
+test(`get page title`, async function() {
+        
+    const c = Crawler.factory();
+    
+    await c.client.get("https://google.com")
+
+    expect(c.document.title).toBe(`Google`)
+    
+})

package/index.ts

@@ -2,10 +2,13 @@ import type { Got } from "got";
 import got from "got";
 import { CookieJar } from "tough-cookie";
 import FileCookieStore from "tough-cookie-file-store";
+import { parseHTML } from "linkedom"
 
 export class Crawler {
     client!: Got
     cookieJar!: CookieJar
+    html = ``
+    document = parseHTML(``).document
 
     static factory() {
         const instance = new Crawler()
@@ -13,7 +16,21 @@ export class Crawler {
         instance.cookieJar = new CookieJar(new FileCookieStore(`cookies.json`))
 
         instance.client = got.extend({
-            cookieJar: instance.cookieJar
+            hooks: {
+                afterResponse: [
+                    res => {
+                        if(typeof res.body == `string`) {
+                            instance.html =  res.body
+                            instance.document = parseHTML(res.body).document
+                        }
+                        
+                        return res
+                    }
+                ]
+            }
+            // followRedirect:false,
+            // cookieJar: instance.cookieJar,
+            
         })
 
         return instance;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wargas/crawler",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "module": "index.ts",
   "main": "dist/index.js",
   "type": "module",
@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "got": "^15.0.5",
+    "linkedom": "^0.18.12",
     "tough-cookie": "^6.0.1",
     "tough-cookie-file-store": "^3.3.0"
   }

package/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     // Environment setup & latest features
-    "lib": ["ESNext"],
+    "lib": ["ESNext", "DOM"],
     "target": "ESNext",
     "module": "Preserve",
     "moduleDetection": "force",