@agentsean/crawler 1.0.0

package/dist/persist.js

@@ -0,0 +1,186 @@
+import { randomUUID } from "node:crypto";
+import { crawls, entitySightings, findings, pages, pageSnapshots, sites, } from "@agentsean/db";
+import { eq } from "drizzle-orm";
+import { urlHash } from "./hash.js";
+export async function persistCrawl(db, result, siteName, opts) {
+    const now = new Date().toISOString();
+    const existing = db.select().from(sites).where(eq(sites.origin, result.origin)).all();
+    let siteId = existing[0]?.id;
+    if (!siteId) {
+        siteId = randomUUID();
+        const observeUntil = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString();
+        db.insert(sites)
+            .values({
+            id: siteId,
+            origin: result.origin,
+            name: siteName ?? result.origin,
+            observeUntil,
+            createdAt: now,
+            updatedAt: now,
+        })
+            .run();
+    }
+    recordCrawlEntity(db, siteId, result.origin + "/", now);
+    const status = opts?.status ??
+        (result.aborted ? "running" : result.truncated ? "running" : "complete");
+    let crawlId = opts?.crawlId;
+    if (crawlId) {
+        const row = db.select().from(crawls).where(eq(crawls.id, crawlId)).get();
+        if (row) {
+            db.update(crawls)
+                .set({
+                finishedAt: status === "complete" ? result.finishedAt : null,
+                status,
+                pagesSeen: (row.pagesSeen ?? 0) + result.pagesSeen,
+                pagesChanged: (row.pagesChanged ?? 0) + result.pagesChanged,
+                error: result.aborted ? "aborted" : null,
+            })
+                .where(eq(crawls.id, crawlId))
+                .run();
+        }
+        else {
+            crawlId = undefined;
+        }
+    }
+    if (!crawlId) {
+        crawlId = randomUUID();
+        db.insert(crawls)
+            .values({
+            id: crawlId,
+            siteId,
+            startedAt: result.startedAt,
+            finishedAt: status === "complete" ? result.finishedAt : null,
+            status,
+            pagesSeen: result.pagesSeen,
+            pagesChanged: result.pagesChanged,
+            error: result.aborted ? "aborted" : null,
+        })
+            .run();
+    }
+    for (const page of result.pages) {
+        persistPage(db, siteId, crawlId, page, now);
+    }
+    return { siteId, crawlId };
+}
+function persistPage(db, siteId, crawlId, page, now) {
+    const hash = urlHash(page.url);
+    const found = db
+        .select()
+        .from(pages)
+        .where(eq(pages.siteId, siteId))
+        .all()
+        .find((row) => row.urlHash === hash);
+    const extract = page.extract;
+    const values = {
+        url: page.url,
+        urlHash: hash,
+        statusCode: page.statusCode,
+        contentHash: page.contentHash,
+        title: extract?.title ?? null,
+        metaDescription: extract?.metaDescription ?? null,
+        canonical: extract?.canonicalHtml[0] ?? null,
+        h1: extract?.h1[0] ?? null,
+        wordCount: extract?.mainWordCount ?? extract?.wordCount ?? null,
+        lang: extract?.lang ?? null,
+        jsonld: extract?.jsonLd.length
+            ? JSON.stringify(extract.jsonLd.map((j) => j.parsed))
+            : null,
+        lastCrawledAt: now,
+        lastChangedAt: page.notModified ? (found?.lastChangedAt ?? now) : now,
+        inlinkCount: page.inlinkCount,
+        outlinkCount: page.outlinkCount,
+    };
+    let pageId;
+    if (found) {
+        pageId = found.id;
+        db.update(pages).set(values).where(eq(pages.id, pageId)).run();
+    }
+    else {
+        pageId = randomUUID();
+        db.insert(pages)
+            .values({
+            id: pageId,
+            siteId,
+            firstSeenAt: now,
+            lastAuditedAt: null,
+            ...values,
+        })
+            .run();
+    }
+    db.insert(pageSnapshots)
+        .values({
+        id: randomUUID(),
+        pageId,
+        crawlId,
+        fetchedAt: page.fetchedAt,
+        statusCode: page.statusCode,
+        contentHash: page.contentHash,
+        body: page.html ? page.html.slice(0, 2_000_000) : null,
+        headers: JSON.stringify(page.headers),
+    })
+        .run();
+    recordCrawlEntity(db, siteId, page.url, now);
+}
+function recordCrawlEntity(db, siteId, url, now) {
+    const existing = db
+        .select()
+        .from(entitySightings)
+        .where(eq(entitySightings.siteId, siteId))
+        .all()
+        .find((row) => row.entity === url);
+    if (existing)
+        return;
+    db.insert(entitySightings)
+        .values({
+        id: randomUUID(),
+        siteId,
+        entity: url,
+        entityKind: "url",
+        source: "crawl",
+        firstSeenAt: now,
+    })
+        .run();
+}
+export function persistFindings(db, siteId, rows) {
+    const now = new Date().toISOString();
+    const pageRows = db.select().from(pages).where(eq(pages.siteId, siteId)).all();
+    const byUrl = new Map(pageRows.map((p) => [p.url, p.id]));
+    for (const row of rows) {
+        const existing = db
+            .select()
+            .from(findings)
+            .where(eq(findings.fingerprint, row.fingerprint))
+            .all()[0];
+        if (existing) {
+            db.update(findings)
+                .set({
+                severity: row.severity,
+                title: row.title,
+                explanation: row.explanation,
+                evidence: JSON.stringify(row.evidence),
+                status: "open",
+            })
+                .where(eq(findings.id, existing.id))
+                .run();
+            continue;
+        }
+        db.insert(findings)
+            .values({
+            id: randomUUID(),
+            siteId,
+            pageId: row.pageUrl ? (byUrl.get(row.pageUrl) ?? null) : null,
+            ruleId: row.ruleId,
+            severity: row.severity,
+            autonomyTier: row.autonomyTier,
+            title: row.title,
+            explanation: row.explanation,
+            evidence: JSON.stringify(row.evidence),
+            status: "open",
+            fingerprint: row.fingerprint,
+            firstDetectedAt: now,
+            resolvedAt: null,
+        })
+            .run();
+    }
+}
+//# sourceMappingURL=persist.js.map

package/dist/persist.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"persist.js","sourceRoot":"","sources":["../src/persist.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EACL,MAAM,EACN,eAAe,EACf,QAAQ,EACR,KAAK,EACL,aAAa,EACb,KAAK,GACN,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAkB,EAClB,MAAmB,EACnB,QAAiB,EACjB,IAAoE;IAEpE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACtF,IAAI,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,UAAU,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAClF,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;aACb,MAAM,CAAC;YACN,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM;YAC/B,YAAY;YACZ,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG;SACf,CAAC;aACD,GAAG,EAAE,CAAC;IACX,CAAC;IACD,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAExD,MAAM,MAAM,GACV,IAAI,EAAE,MAAM;QACZ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC;IAC5B,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACzE,IAAI,GAAG,EAAE,CAAC;YACR,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACd,GAAG,CAAC;gBACH,UAAU,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;gBAC5D,MAAM;gBACN,SAAS,EAAE,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS;gBAClD,YAAY,EAAE,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,YAAY;gBAC3D,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;aACzC,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;iBAC7B,GAAG,EAAE,CAAC;QACX,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,UAAU,EAAE,CAAC;QACvB,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;aACd,MAAM,CAAC;YACN,EAAE,EAAE,OAAO;YACX,MAAM;YACN,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;YAC5D,MAAM;YACN,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;SACzC,CAAC;aACD,GAAG,EAAE,CAAC;IACX,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAClB,EAAkB,EAClB,MAAc,EACd,OAAe,EACf,IAAiB,EACjB,GAAW;IAEX,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,EAAE;SACb,MAAM,EAAE;SACR,IAAI,CAAC,KAAK,CAAC;SACX,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SAC/B,GAAG,EAAE;SACL,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG;QACb,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,OAAO,EAAE,IAAI;QACb,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI;QAC7B,eAAe,EAAE,OAAO,EAAE,eAAe,IAAI,IAAI;QACjD,SAAS,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI;QAC5C,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI;QAC1B,SAAS,EAAE,OAAO,EAAE,aAAa,IAAI,OAAO,EAAE,SAAS,IAAI,IAAI;QAC/D,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QAC3B,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM;YAC5B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACrD,CAAC,CAAC,IAAI;QACR,aAAa,EAAE,GAAG;QAClB,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,aAAa,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;QACrE,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;KAChC,CAAC;IAEF,IAAI,MAAc,CAAC;IACnB,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACjE,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,UAAU,EAAE,CAAC;QACtB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;aACb,MAAM,CAAC;YACN,EAAE,EAAE,MAAM;YACV,MAAM;YACN,WAAW,EAAE,GAAG;YAChB,aAAa,EAAE,IAAI;YACnB,GAAG,MAAM;SACV,CAAC;aACD,GAAG,EAAE,CAAC;IACX,CAAC;IAED,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC;SACrB,MAAM,CAAC;QACN,EAAE,EAAE,UAAU,EAAE;QAChB,MAAM;QACN,OAAO;QACP,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;QACtD,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;KACtC,CAAC;SACD,GAAG,EAAE,CAAC;IAET,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,iBAAiB,CACxB,EAAkB,EAClB,MAAc,EACd,GAAW,EACX,GAAW;IAEX,MAAM,QAAQ,GAAG,EAAE;SAChB,MAAM,EAAE;SACR,IAAI,CAAC,eAAe,CAAC;SACrB,KAAK,CAAC,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SACzC,GAAG,EAAE;SACL,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC;IACrC,IAAI,QAAQ;QAAE,OAAO;IACrB,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;SACvB,MAAM,CAAC;QACN,EAAE,EAAE,UAAU,EAAE;QAChB,MAAM;QACN,MAAM,EAAE,GAAG;QACX,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,OAAO;QACf,WAAW,EAAE,GAAG;KACjB,CAAC;SACD,GAAG,EAAE,CAAC;AACX,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,EAAkB,EAClB,MAAc,EACd,IASG;IAEH,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,EAAE;aAChB,MAAM,EAAE;aACR,IAAI,CAAC,QAAQ,CAAC;aACd,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;aAChD,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACZ,IAAI,QAAQ,EAAE,CAAC;YACb,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;iBAChB,GAAG,CAAC;gBACH,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACtC,MAAM,EAAE,MAAM;aACf,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;iBACnC,GAAG,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QACD,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;aAChB,MAAM,CAAC;YACN,EAAE,EAAE,UAAU,EAAE;YAChB,MAAM;YACN,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YAC7D,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;YACtC,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,eAAe,EAAE,GAAG;YACpB,UAAU,EAAE,IAAI;SACjB,CAAC;aACD,GAAG,EAAE,CAAC;IACX,CAAC;AACH,CAAC"}

package/dist/probe.d.ts

@@ -0,0 +1,4 @@
+import type { Agent } from "undici";
+import type { OriginProbe } from "./types.js";
+export declare function probeOrigin(startUrl: string, agent: Agent): Promise<OriginProbe>;
+//# sourceMappingURL=probe.d.ts.map

package/dist/probe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAKpC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,KAAK,GACX,OAAO,CAAC,WAAW,CAAC,CA2FtB"}

package/dist/probe.js

@@ -0,0 +1,132 @@
+import tls from "node:tls";
+import { fetchUrl } from "./http.js";
+import { contentHash } from "./hash.js";
+import { originOf } from "./url.js";
+import { randomBytes } from "node:crypto";
+export async function probeOrigin(startUrl, agent) {
+    const origin = originOf(startUrl);
+    const u = new URL(origin);
+    const https = u.protocol === "https:";
+    let httpRedirectsToHttps = null;
+    let wwwSplit = null;
+    let wwwPreferred = null;
+    let trailingSlashSplit = null;
+    let randomSoft404 = false;
+    let randomSoft404Url = null;
+    let randomSoft404Hash = null;
+    let hsts = null;
+    if (https) {
+        try {
+            const httpUrl = `http://${u.host}${u.pathname}`;
+            const res = await fetchUrl(httpUrl, { agent, timeoutMs: 8_000, maxRedirects: 5 });
+            httpRedirectsToHttps = res.finalUrl.startsWith("https:");
+        }
+        catch {
+            httpRedirectsToHttps = false;
+        }
+    }
+    try {
+        const host = u.hostname;
+        const alt = host.startsWith("www.") ? host.slice(4) : `www.${host}`;
+        const altUrl = `${u.protocol}//${alt}/`;
+        const a = await fetchUrl(`${origin}/`, {
+            agent,
+            timeoutMs: 8_000,
+            maxRedirects: 0,
+        });
+        const b = await fetchUrl(altUrl, { agent, timeoutMs: 8_000, maxRedirects: 0 });
+        const a200 = a.statusCode === 200;
+        const b200 = b.statusCode === 200;
+        wwwSplit = a200 && b200;
+        if (!wwwSplit) {
+            wwwPreferred = a200 ? origin : b200 ? `${u.protocol}//${alt}` : origin;
+        }
+        hsts = a.headers["strict-transport-security"] ?? null;
+    }
+    catch {
+        wwwSplit = null;
+    }
+    try {
+        const withSlash = `${origin}/probe-slash-test/`;
+        const noSlash = `${origin}/probe-slash-test`;
+        const a = await fetchUrl(withSlash, { agent, timeoutMs: 8_000, maxRedirects: 0 });
+        const b = await fetchUrl(noSlash, { agent, timeoutMs: 8_000, maxRedirects: 0 });
+        trailingSlashSplit = a.statusCode === 200 && b.statusCode === 200;
+    }
+    catch {
+        trailingSlashSplit = null;
+    }
+    try {
+        const slug = randomBytes(16).toString("hex");
+        randomSoft404Url = `${origin}/${slug}`;
+        const res = await fetchUrl(randomSoft404Url, {
+            agent,
+            timeoutMs: 8_000,
+            maxRedirects: 0,
+        });
+        randomSoft404 = res.statusCode === 200;
+        if (randomSoft404 && res.decoded.length) {
+            randomSoft404Hash = contentHash(res.decoded);
+        }
+    }
+    catch {
+        randomSoft404 = false;
+    }
+    const cert = https
+        ? await readCert(u.hostname, Number(u.port || 443))
+        : {
+            certValidTo: null,
+            certDaysRemaining: null,
+            certError: null,
+            alpn: null,
+        };
+    return {
+        https,
+        httpRedirectsToHttps,
+        wwwSplit,
+        wwwPreferred,
+        trailingSlashSplit,
+        randomSoft404,
+        randomSoft404Url,
+        randomSoft404Hash,
+        hsts,
+        ...cert,
+    };
+}
+function readCert(host, port) {
+    return new Promise((resolve) => {
+        const sock = tls.connect({ host, port, servername: host, rejectUnauthorized: true }, () => {
+            const cert = sock.getPeerCertificate();
+            const validTo = cert.valid_to ? new Date(cert.valid_to) : null;
+            const days = validTo
+                ? Math.floor((validTo.getTime() - Date.now()) / 86_400_000)
+                : null;
+            const alpn = sock.alpnProtocol || sock.getProtocol() || null;
+            sock.end();
+            resolve({
+                certValidTo: validTo?.toISOString() ?? null,
+                certDaysRemaining: days,
+                certError: null,
+                alpn: typeof alpn === "string" ? alpn : null,
+            });
+        });
+        sock.setTimeout(5000, () => {
+            sock.destroy();
+            resolve({
+                certValidTo: null,
+                certDaysRemaining: null,
+                certError: "timeout",
+                alpn: null,
+            });
+        });
+        sock.on("error", (e) => {
+            resolve({
+                certValidTo: null,
+                certDaysRemaining: null,
+                certError: e.message,
+                alpn: null,
+            });
+        });
+    });
+}
+//# sourceMappingURL=probe.js.map

package/dist/probe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"probe.js","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,UAAU,CAAC;AAE3B,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,KAAY;IAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IACtC,IAAI,oBAAoB,GAAmB,IAAI,CAAC;IAChD,IAAI,QAAQ,GAAmB,IAAI,CAAC;IACpC,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,kBAAkB,GAAmB,IAAI,CAAC;IAC9C,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAC3C,IAAI,iBAAiB,GAAkB,IAAI,CAAC;IAC5C,IAAI,IAAI,GAAkB,IAAI,CAAC;IAE/B,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;YAClF,oBAAoB,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB,GAAG,KAAK,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,GAAG,GAAG,CAAC;QACxC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,GAAG,MAAM,GAAG,EAAE;YACrC,KAAK;YACL,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC;QAClC,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACzE,CAAC;QACD,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,2BAA2B,CAAC,IAAI,IAAI,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,GAAG,MAAM,oBAAoB,CAAC;QAChD,MAAM,OAAO,GAAG,GAAG,MAAM,mBAAmB,CAAC;QAC7C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAClF,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAChF,kBAAkB,GAAG,CAAC,CAAC,UAAU,KAAK,GAAG,IAAI,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7C,gBAAgB,GAAG,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,gBAAgB,EAAE;YAC3C,KAAK;YACL,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;QACH,aAAa,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC;QACvC,IAAI,aAAa,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,iBAAiB,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,aAAa,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,MAAM,IAAI,GAAG,KAAK;QAChB,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;QACnD,CAAC,CAAC;YACE,WAAW,EAAE,IAAI;YACjB,iBAAiB,EAAE,IAAI;YACvB,SAAS,EAAE,IAAI;YACf,IAAI,EAAE,IAAI;SACX,CAAC;IAEN,OAAO;QACL,KAAK;QACL,oBAAoB;QACpB,QAAQ;QACR,YAAY;QACZ,kBAAkB;QAClB,aAAa;QACb,gBAAgB;QAChB,iBAAiB;QACjB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,IAAY,EACZ,IAAY;IAOZ,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CACtB,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,EAC1D,GAAG,EAAE;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/D,MAAM,IAAI,GAAG,OAAO;gBAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC;gBAC3D,CAAC,CAAC,IAAI,CAAC;YACT,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;YAC7D,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,OAAO,CAAC;gBACN,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,IAAI;gBAC3C,iBAAiB,EAAE,IAAI;gBACvB,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;aAC7C,CAAC,CAAC;QACL,CAAC,CACF,CAAC;QACF,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE;YACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,CAAC;gBACN,WAAW,EAAE,IAAI;gBACjB,iBAAiB,EAAE,IAAI;gBACvB,SAAS,EAAE,SAAS;gBACpB,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;YACrB,OAAO,CAAC;gBACN,WAAW,EAAE,IAAI;gBACjB,iBAAiB,EAAE,IAAI;gBACvB,SAAS,EAAE,CAAC,CAAC,OAAO;gBACpB,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/readability.d.ts

@@ -0,0 +1,9 @@
+export type ReadabilityResult = {
+    title: string;
+    text: string;
+    wordCount: number;
+};
+export declare function extractMainContent(html: string, documentTitle: string): ReadabilityResult;
+export declare function words(text: string): number;
+export declare function loremIpsumIn(text: string): boolean;
+//# sourceMappingURL=readability.d.ts.map

package/dist/readability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability.d.ts","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AA0BA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,GACpB,iBAAiB,CA0BnB;AAED,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG1C;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAIlD"}

package/dist/readability.js

@@ -0,0 +1,60 @@
+/**
+ * Vendored main-content extractor.
+ *
+ * @mozilla/readability 0.6.0 last published 2025-03-03 (stale). We vendor a
+ * focused extractor rather than taking the unmaintained runtime dep.
+ *
+ * IMPORTANT: Mozilla Readability sources `title` from attacker-controlled
+ * JSON-LD `headline` and `excerpt` from `og:description`. Both MUST be
+ * treated as untrusted. This implementation never reads JSON-LD or Open
+ * Graph for title/excerpt.
+ */
+import { parseHTML } from "linkedom";
+const STRIP = new Set([
+    "SCRIPT",
+    "STYLE",
+    "NOSCRIPT",
+    "NAV",
+    "FOOTER",
+    "ASIDE",
+    "FORM",
+    "IFRAME",
+    "SVG",
+    "HEADER",
+]);
+export function extractMainContent(html, documentTitle) {
+    let text = "";
+    try {
+        const { document } = parseHTML(html);
+        for (const el of document.querySelectorAll("*")) {
+            if (STRIP.has(el.tagName))
+                el.remove();
+        }
+        const main = document.querySelector("article") ??
+            document.querySelector("main") ??
+            document.querySelector("[role='main']") ??
+            document.querySelector("body");
+        text = (main?.textContent ?? "").replace(/\s+/g, " ").trim();
+    }
+    catch {
+        text = html
+            .replace(/<script[\s\S]*?<\/script>/gi, " ")
+            .replace(/<style[\s\S]*?<\/style>/gi, " ")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+    return {
+        title: documentTitle,
+        text,
+        wordCount: words(text),
+    };
+}
+export function words(text) {
+    const m = text.trim().match(/\S+/g);
+    return m ? m.length : 0;
+}
+export function loremIpsumIn(text) {
+    return /lorem ipsum|dolor sit amet|placeholder text|\bTODO\b|\bFIXME\b|\[insert/i.test(text);
+}
+//# sourceMappingURL=readability.js.map

package/dist/readability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability.js","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC;IACpB,QAAQ;IACR,OAAO;IACP,UAAU;IACV,KAAK;IACL,QAAQ;IACR,OAAO;IACP,MAAM;IACN,QAAQ;IACR,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAQH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,aAAqB;IAErB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC;gBAAE,EAAE,CAAC,MAAM,EAAE,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GACR,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC;YACjC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC;YAC9B,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC;YACvC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,IAAI;aACR,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;aAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;aACzC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;IACZ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,aAAa;QACpB,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC;KACvB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,IAAY;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,0EAA0E,CAAC,IAAI,CACpF,IAAI,CACL,CAAC;AACJ,CAAC"}

package/dist/render.d.ts

@@ -0,0 +1,15 @@
+import type { PageExtract } from "./types.js";
+export type RenderResult = {
+    html: string;
+    extract: PageExtract;
+    finalUrl: string;
+    errors: string[];
+    timedOut: boolean;
+};
+/**
+ * Adaptive Playwright renderer. Playwright is optional — if Chromium is not
+ * installed we skip JS rendering rather than failing the audit.
+ */
+export declare function renderPage(url: string, rawHtml: string): Promise<RenderResult | null>;
+export declare function jsDependencyScore(html: string, extract: PageExtract): number;
+//# sourceMappingURL=render.d.ts.map

package/dist/render.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../src/render.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA6D9B;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,MAAM,CAQ5E"}

package/dist/render.js

@@ -0,0 +1,87 @@
+import { extractPage } from "./extract.js";
+/**
+ * Adaptive Playwright renderer. Playwright is optional — if Chromium is not
+ * installed we skip JS rendering rather than failing the audit.
+ */
+export async function renderPage(url, rawHtml) {
+    let playwright;
+    try {
+        playwright = await import("playwright");
+    }
+    catch {
+        return null;
+    }
+    const browserType = playwright.chromium;
+    let browser;
+    try {
+        browser = await browserType.launch({
+            headless: true,
+            args: [
+                "--disable-dev-shm-usage",
+                "--disable-gpu",
+                "--disable-extensions",
+                "--mute-audio",
+            ],
+        });
+        const context = await browser.newContext({
+            viewport: { width: 412, height: 823 },
+            deviceScaleFactor: 2.625,
+            isMobile: true,
+        });
+        const page = await context.newPage();
+        await page.route("**/*", (route) => {
+            const type = route.request().resourceType();
+            if (type === "image" || type === "media" || type === "font") {
+                return route.abort();
+            }
+            return route.continue();
+        });
+        const errors = [];
+        page.on("pageerror", (e) => errors.push(e.message));
+        let timedOut = false;
+        try {
+            await page.goto(url, { waitUntil: "domcontentloaded", timeout: 15_000 });
+        }
+        catch {
+            timedOut = true;
+        }
+        const html = await page.content();
+        const finalUrl = page.url();
+        await context.close();
+        return {
+            html,
+            extract: extractPage(html, finalUrl),
+            finalUrl,
+            errors,
+            timedOut,
+        };
+    }
+    catch {
+        return {
+            html: rawHtml,
+            extract: extractPage(rawHtml, url),
+            finalUrl: url,
+            errors: ["playwright_unavailable"],
+            timedOut: false,
+        };
+    }
+    finally {
+        if (browser)
+            await browser.close().catch(() => undefined);
+    }
+}
+export function jsDependencyScore(html, extract) {
+    let score = 0;
+    if (extract.spaRootEmpty)
+        score += 0.5;
+    if (extract.hasNoscriptJsWarning)
+        score += 0.3;
+    if (extract.mainWordCount < 50)
+        score += 0.2;
+    if (extract.scripts.filter((s) => s.src).length >= 8)
+        score += 0.1;
+    if (/#\/|#!/.test(html))
+        score += 0.2;
+    return Math.min(1, score);
+}
+//# sourceMappingURL=render.js.map

package/dist/render.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"render.js","sourceRoot":"","sources":["../src/render.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAU3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAAW,EACX,OAAe;IAEf,IAAI,UAAuC,CAAC;IAC5C,IAAI,CAAC;QACH,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC;IACxC,IAAI,OAAmE,CAAC;IACxE,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC;YACjC,QAAQ,EAAE,IAAI;YACd,IAAI,EAAE;gBACJ,yBAAyB;gBACzB,eAAe;gBACf,sBAAsB;gBACtB,cAAc;aACf;SACF,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;YACrC,iBAAiB,EAAE,KAAK;YACxB,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;YAC5C,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5D,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;YACD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACpD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC;YACpC,QAAQ;YACR,MAAM;YACN,QAAQ;SACT,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC;YAClC,QAAQ,EAAE,GAAG;YACb,MAAM,EAAE,CAAC,wBAAwB,CAAC;YAClC,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAoB;IAClE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,CAAC,YAAY;QAAE,KAAK,IAAI,GAAG,CAAC;IACvC,IAAI,OAAO,CAAC,oBAAoB;QAAE,KAAK,IAAI,GAAG,CAAC;IAC/C,IAAI,OAAO,CAAC,aAAa,GAAG,EAAE;QAAE,KAAK,IAAI,GAAG,CAAC;IAC7C,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC;QAAE,KAAK,IAAI,GAAG,CAAC;IACnE,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,KAAK,IAAI,GAAG,CAAC;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5B,CAAC"}

package/dist/robots.d.ts

@@ -0,0 +1,13 @@
+import type { RobotsOutcome } from "./types.js";
+export declare function parseRobotsTxt(raw: string): Omit<RobotsOutcome, "statusCode" | "contentType" | "bytes" | "redirectHops" | "error" | "mode">;
+export declare function robotsFromFetch(opts: {
+    statusCode: number | null;
+    raw: string;
+    contentType: string;
+    bytes: number;
+    redirectHops: number;
+    error: string | null;
+}): RobotsOutcome;
+export declare function isAllowedByRobots(robots: RobotsOutcome, url: string, userAgent?: string): boolean;
+export declare function matchRule(pattern: string, path: string): boolean;
+//# sourceMappingURL=robots.d.ts.map

package/dist/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,YAAY,CAAC;AAI7D,wBAAgB,cAAc,CAC5B,GAAG,EAAE,MAAM,GACV,IAAI,CACL,aAAa,EACb,YAAY,GAAG,aAAa,GAAG,OAAO,GAAG,cAAc,GAAG,OAAO,GAAG,MAAM,CAC3E,CA8FA;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE;IACpC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,GAAG,aAAa,CAoBhB;AAED,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,EACrB,GAAG,EAAE,MAAM,EACX,SAAS,SAAY,GACpB,OAAO,CAOT;AAoCD,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAchE"}