@wargas/crawler 0.0.3 → 0.0.5

package/README.md

@@ -1,15 +1,109 @@
-# crawler
+# Crawler
 
-To install dependencies:
+Classe utilitária para realizar requisições HTTP com:
+
+* Persistência automática de cookies
+* Parsing automático de HTML
+* Manipulação do DOM usando `linkedom`
+* Cliente HTTP baseado em got
+
+---
+
+# Instalação
 
 ```bash
-bun install
+bun add @wargas/crawler
 ```
 
-To run:
+ou usando npm:
 
 ```bash
-bun run index.ts
+npm install @wargas/crawler
+```
+
+---
+
+# Uso básico
+
+```ts
+import { Crawler } from "@wargas/crawler";
+
+const crawler = Crawler.factory();
+
+await crawler.client.get("https://example.com");
+
+console.log(crawler.html);
+
+console.log(
+    crawler.document.querySelector("title")?.textContent
+);
+```
+
+---
+
+# Cookies persistentes
+
+Os cookies são armazenados automaticamente no arquivo:
+
+```txt
+cookies.json
 ```
 
-This project was created using `bun init` in bun v1.3.14. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
+Isso permite manter sessão entre execuções.
+
+---
+
+# Limpar cookies
+
+```ts
+await crawler.removeAllCookies();
+```
+
+---
+
+# Acessando o DOM
+
+Como o HTML é convertido automaticamente usando `linkedom`, é possível utilizar APIs similares ao navegador:
+
+```ts
+const links = crawler.document.querySelectorAll("a");
+
+for (const link of links) {
+    console.log(link.getAttribute("href"));
+}
+```
+
+---
+
+# Configurações atuais
+
+A instância do `got` é criada com:
+
+```ts
+followRedirect: false
+```
+
+Ou seja:
+
+* redirects não são seguidos automaticamente
+* cookies são persistidos
+* HTML é parseado automaticamente após cada resposta
+
+---
+
+# Possíveis melhorias
+
+* Suporte a proxy
+* Retry automático
+* Timeout configurável
+* User-Agent customizado
+* Suporte a certificados digitais
+* Suporte a HTTP2
+* Métodos helper (`get`, `post`, `login`, etc.)
+* Cache de páginas
+
+---
+
+# Licença
+
+MIT

package/bun.lock

@@ -12,9 +12,10 @@
       },
       "devDependencies": {
         "@types/bun": "latest",
+        "tsc": "^2.0.4",
       },
       "peerDependencies": {
-        "typescript": "^5",
+        "typescript": "^6.0.3",
       },
     },
   },
@@ -103,9 +104,11 @@
 
     "tough-cookie-file-store": ["tough-cookie-file-store@3.3.0", "", { "dependencies": { "tough-cookie": "^6.0.0" } }, "sha512-FbO/cOi/jp4wweo8soVNG/ZjDsgpBZWqaxWwu7gRKvsjg/Qt44kStp87VLfJnin749DlTbZDYvV1wuSr5jly2g=="],
 
+    "tsc": ["tsc@2.0.4", "", { "bin": { "tsc": "bin/tsc" } }, "sha512-fzoSieZI5KKJVBYGvwbVZs/J5za84f2lSTLPYf6AGiIf43tZ3GNrI1QzTLcjtyDDP4aLxd46RTZq1nQxe7+k5Q=="],
+
     "type-fest": ["type-fest@5.6.0", "", { "dependencies": { "tagged-tag": "^1.0.0" } }, "sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA=="],
 
-    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
+    "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="],
 
     "uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
 

package/dist/index.js

@@ -23300,7 +23300,9 @@ class Crawler {
             return res;
           }
         ]
-      }
+      },
+      followRedirect: false,
+      cookieJar: instance.cookieJar
     });
     return instance;
   }

package/index.d.ts

@@ -0,0 +1,15 @@
+// crawler.d.ts
+
+import type { Got } from "got";
+import type { CookieJar } from "tough-cookie";
+
+export declare class Crawler {
+    client: Got;
+    cookieJar: CookieJar;
+    html: string;
+    document: Document;
+
+    static factory(): Crawler;
+
+    removeAllCookies(): Promise<void>;
+}

package/index.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from 'bun:test'
-import { Crawler } from "./index";
+import { Crawler } from "./";
 
 test(`get page title`, async function() {
         

package/index.ts

@@ -8,7 +8,7 @@ export class Crawler {
     client!: Got
     cookieJar!: CookieJar
     html = ``
-    document = parseHTML(``).document
+    document: Document = parseHTML(``).document
 
     static factory() {
         const instance = new Crawler()
@@ -27,9 +27,9 @@ export class Crawler {
                         return res
                     }
                 ]
-            }
-            // followRedirect:false,
-            // cookieJar: instance.cookieJar,
+            },
+            followRedirect:false,
+            cookieJar: instance.cookieJar,
             
         })
 

package/package.json

@@ -1,19 +1,24 @@
 {
   "name": "@wargas/crawler",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "module": "index.ts",
   "main": "dist/index.js",
   "type": "module",
   "private": false,
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wargas/crawler.git"
+  },
   "scripts": {
     "build": "bun build index.ts --target node --outdir dist",
     "publish": "npm publish --tag latest --access public"
   },
   "devDependencies": {
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "tsc": "^2.0.4"
   },
   "peerDependencies": {
-    "typescript": "^5"
+    "typescript": "^6.0.3"
   },
   "dependencies": {
     "got": "^15.0.5",