@hej-ai/crawler 0.1.0

package/README.md

@@ -0,0 +1,51 @@
+# @hej-ai/crawler
+
+Scrape any webpage into clean markdown.
+
+## Install
+
+```bash
+npm install @hej-ai/crawler
+```
+
+## Usage
+
+```typescript
+import {crawler} from "@hej-ai/crawler"
+
+const {scrape} = crawler("your-api-key")
+
+const page = await scrape("https://example.com")
+console.log(page.title)    // "Example Domain"
+console.log(page.markdown) // clean markdown content
+console.log(page.links)    // internal links found on the page
+```
+
+### Scraping modes
+
+```typescript
+// Fast HTTP fetch + Readability extraction
+await scrape("https://example.com", "static")
+
+// Full headless browser render (default)
+await scrape("https://example.com", "playwright")
+
+// Tries static first, uses browser if content differs significantly
+await scrape("https://example.com", "scout")
+```
+
+### Response
+
+```typescript
+{
+  url: string      // final URL after redirects
+  title: string    // page title
+  markdown: string // page content as markdown
+  links: string[]  // internal links found
+  mode: string     // mode used to produce the result
+}
+```
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export type ScrapeMode = "static" | "playwright" | "scout";
+export type ScrapedPage = {
+    url: string;
+    title: string;
+    markdown: string;
+    links: string[];
+    mode: ScrapeMode;
+};
+export declare function crawler(apiKey: string, apiUrl?: string): {
+    scrape: (url: string, mode?: ScrapeMode) => Promise<ScrapedPage>;
+};

package/dist/index.js

@@ -0,0 +1,50 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __moduleCache = /* @__PURE__ */ new WeakMap;
+var __toCommonJS = (from) => {
+  var entry = __moduleCache.get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function")
+    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    }));
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+
+// src/index.ts
+var exports_src = {};
+__export(exports_src, {
+  crawler: () => crawler
+});
+module.exports = __toCommonJS(exports_src);
+var DEFAULT_API = "https://api.hej.chat/api/v1";
+function crawler(apiKey, apiUrl = DEFAULT_API) {
+  async function scrape(url, mode) {
+    const res = await fetch(`${apiUrl}/scrape`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ url, mode })
+    });
+    if (!res.ok) {
+      const body = await res.json().catch(() => null);
+      throw new Error(body?.error ?? `Request failed with status ${res.status}`);
+    }
+    return res.json();
+  }
+  return { scrape };
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@hej-ai/crawler",
+  "version": "0.1.0",
+  "description": "Scrape any webpage into clean markdown",
+  "type": "module",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": ["dist"],
+  "scripts": {
+    "build": "bun build src/index.ts --outdir dist --format esm --target node && bun build src/index.ts --outdir dist --format cjs --target node --outfile dist/index.cjs && tsc --emitDeclarationOnly",
+    "prepublishOnly": "bun run build"
+  },
+  "devDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hej-ai/crawler"
+  },
+  "keywords": ["scraper", "crawler", "markdown", "web-scraping"]
+}