@agentsean/crawler 1.0.0

package/dist/types.d.ts

@@ -0,0 +1,251 @@
+export type RedirectHop = {
+    url: string;
+    status: number;
+    location: string | null;
+};
+export type ExtractedLink = {
+    href: string;
+    absUrl: string;
+    anchorText: string;
+    rel: string[];
+    target: string | null;
+    isNofollow: boolean;
+    isUgc: boolean;
+    isSponsored: boolean;
+    isInternal: boolean;
+    position: "nav" | "header" | "footer" | "aside" | "main" | "unknown";
+};
+export type ExtractedImage = {
+    src: string;
+    absUrl: string;
+    alt: string | null;
+    hasAltAttr: boolean;
+    width: string | null;
+    height: string | null;
+    loading: string | null;
+    srcset: string | null;
+    decoding: string | null;
+};
+export type HreflangAlt = {
+    lang: string;
+    href: string;
+    absUrl: string;
+    inHead: boolean;
+    source: "head" | "header" | "body";
+};
+export type JsonLdBlock = {
+    raw: string;
+    parsed: unknown;
+    error: string | null;
+    inHead: boolean;
+};
+export type PageExtract = {
+    title: string | null;
+    titles: string[];
+    titleInHead: boolean;
+    metaDescription: string | null;
+    metaDescriptions: string[];
+    metaDescInHead: boolean;
+    h1: string[];
+    h2: string[];
+    headings: {
+        level: number;
+        text: string;
+    }[];
+    canonicalHtml: string[];
+    canonicalHeader: string[];
+    canonicalInHead: boolean;
+    canonicalOutsideHead: boolean;
+    headBrokenEarly: boolean;
+    robotsMeta: string[];
+    robotsMetaInHead: boolean;
+    xRobotsTag: string | null;
+    viewport: string | null;
+    lang: string | null;
+    charset: string | null;
+    charsetInFirst1024: boolean;
+    hreflang: HreflangAlt[];
+    jsonLd: JsonLdBlock[];
+    openGraph: Record<string, string>;
+    twitterCard: Record<string, string>;
+    links: ExtractedLink[];
+    images: ExtractedImage[];
+    scripts: {
+        src: string | null;
+        absUrl: string | null;
+        async: boolean;
+        defer: boolean;
+    }[];
+    stylesheets: {
+        href: string | null;
+        absUrl: string | null;
+    }[];
+    wordCount: number;
+    mainWordCount: number;
+    mainText: string;
+    allText: string;
+    boilerplateRatio: number;
+    metaRefresh: string | null;
+    hasNoscriptJsWarning: boolean;
+    spaRootEmpty: boolean;
+    hashFragmentMeta: boolean;
+    forms: {
+        action: string;
+        absAction: string;
+    }[];
+    mixedActive: string[];
+    mixedPassive: string[];
+    unsafeTargetBlanks: number;
+    plugins: boolean;
+    loremIpsum: boolean;
+    paginationRel: {
+        next: string | null;
+        prev: string | null;
+    };
+    htmlLang: string | null;
+};
+export type CrawledPage = {
+    url: string;
+    finalUrl: string;
+    statusCode: number | null;
+    error: string | null;
+    contentType: string;
+    headers: Record<string, string>;
+    ttfbMs: number;
+    totalMs: number;
+    wireBytes: number;
+    decodedBytes: number;
+    httpVersion: string | null;
+    redirectChain: RedirectHop[];
+    redirectLoop: boolean;
+    exceedsGoogleRedirectLimit: boolean;
+    html: string | null;
+    renderedHtml: string | null;
+    extract: PageExtract | null;
+    renderedExtract: PageExtract | null;
+    contentHash: string | null;
+    simhash: string | null;
+    etag: string | null;
+    lastModified: string | null;
+    notModified: boolean;
+    supports304: boolean | null;
+    inSitemap: boolean;
+    robotsAllowed: boolean;
+    depth: number;
+    inlinkCount: number;
+    outlinkCount: number;
+    followInlinkCount: number;
+    isInternal: boolean;
+    fetchedAt: string;
+    renderPolicy: "always" | "never" | "sample" | "skipped";
+    rendered: boolean;
+    jsDependencyScore: number;
+    templateKey: string;
+};
+export type RobotsOutcome = {
+    statusCode: number | null;
+    raw: string;
+    contentType: string;
+    bytes: number;
+    redirectHops: number;
+    error: string | null;
+    groups: RobotsGroup[];
+    sitemaps: string[];
+    unknownDirectives: string[];
+    hasNoindexDirective: boolean;
+    hasCrawlDelay: boolean;
+    crawlDelay: number | null;
+    disallowAll: boolean;
+    hash: string;
+    mode: "allow-all" | "disallow-all" | "parsed";
+};
+export type RobotsGroup = {
+    agents: string[];
+    rules: {
+        type: "allow" | "disallow";
+        pattern: string;
+    }[];
+    crawlDelay: number | null;
+};
+export type SitemapUrl = {
+    loc: string;
+    lastmod: string | null;
+    changefreq: string | null;
+    priority: string | null;
+    images: string[];
+    videoValid: boolean | null;
+    hreflang: {
+        lang: string;
+        href: string;
+    }[];
+};
+export type ParsedSitemap = {
+    url: string;
+    statusCode: number | null;
+    error: string | null;
+    contentType: string;
+    bytes: number;
+    gzipBroken: boolean;
+    malformed: boolean;
+    namespace: string | null;
+    isIndex: boolean;
+    nestedIndex: boolean;
+    urls: SitemapUrl[];
+    childSitemaps: string[];
+    relativeLocs: string[];
+};
+export type OriginProbe = {
+    https: boolean;
+    httpRedirectsToHttps: boolean | null;
+    wwwSplit: boolean | null;
+    wwwPreferred: string | null;
+    trailingSlashSplit: boolean | null;
+    randomSoft404: boolean;
+    randomSoft404Url: string | null;
+    randomSoft404Hash: string | null;
+    certValidTo: string | null;
+    certDaysRemaining: number | null;
+    certError: string | null;
+    alpn: string | null;
+    hsts: string | null;
+};
+export type CrawlCheckpoint = {
+    seen: string[];
+    queue: {
+        url: string;
+        depth: number;
+    }[];
+    pagesSeen: number;
+};
+export type CrawlResult = {
+    origin: string;
+    startUrl: string;
+    startedAt: string;
+    finishedAt: string;
+    pages: CrawledPage[];
+    robots: RobotsOutcome | null;
+    sitemaps: ParsedSitemap[];
+    originProbe: OriginProbe;
+    pagesSeen: number;
+    pagesChanged: number;
+    maxPages: number;
+    truncated: boolean;
+    aborted: boolean;
+    checkpoint: CrawlCheckpoint;
+};
+export type CrawlOptions = {
+    startUrl: string;
+    maxPages?: number | undefined;
+    concurrency?: number | undefined;
+    rps?: number | undefined;
+    timeoutMs?: number | undefined;
+    render?: boolean | undefined;
+    userAgent?: string | undefined;
+    followExternal?: boolean | undefined;
+    check304?: boolean | undefined;
+    signal?: AbortSignal | undefined;
+    resume?: CrawlCheckpoint | undefined;
+    onCheckpoint?: ((checkpoint: CrawlCheckpoint) => void) | undefined;
+    checkpointEvery?: number | undefined;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,cAAc,EAAE,OAAO,CAAC;IACxB,EAAE,EAAE,MAAM,EAAE,CAAC;IACb,EAAE,EAAE,MAAM,EAAE,CAAC;IACb,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,KAAK,EAAE,OAAO,CAAC;QACf,KAAK,EAAE,OAAO,CAAC;KAChB,EAAE,CAAC;IACJ,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EAAE,CAAC;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,YAAY,EAAE,OAAO,CAAC;IACtB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAC5D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,WAAW,EAAE,CAAC;IAC7B,YAAY,EAAE,OAAO,CAAC;IACtB,0BAA0B,EAAE,OAAO,CAAC;IACpC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,eAAe,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;IACxD,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,cAAc,GAAG,QAAQ,CAAC;CAC/C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE;QAAE,IAAI,EAAE,OAAO,GAAG,UAAU,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACzD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,OAAO,CAAC;IACf,oBAAoB,EAAE,OAAO,GAAG,IAAI,CAAC;IACrC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,kBAAkB,EAAE,OAAO,GAAG,IAAI,CAAC;IACnC,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,eAAe,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;IACrC,YAAY,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,eAAe,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACnE,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,CAAC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/ua.d.ts

@@ -0,0 +1,3 @@
+export declare const SEAN_UA = "Mozilla/5.0 (compatible; SeanBot/0.0; +https://github.com/seziro-team/agentsean) undici/8";
+export declare const GOOGLEBOT_SMARTPHONE_UA = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+//# sourceMappingURL=ua.d.ts.map

package/dist/ua.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ua.d.ts","sourceRoot":"","sources":["../src/ua.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,8FACyE,CAAC;AAE9F,eAAO,MAAM,uBAAuB,wMACmK,CAAC"}

package/dist/ua.js

@@ -0,0 +1,3 @@
+export const SEAN_UA = "Mozilla/5.0 (compatible; SeanBot/0.0; +https://github.com/seziro-team/agentsean) undici/8";
+export const GOOGLEBOT_SMARTPHONE_UA = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+//# sourceMappingURL=ua.js.map

package/dist/ua.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ua.js","sourceRoot":"","sources":["../src/ua.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAClB,2FAA2F,CAAC;AAE9F,MAAM,CAAC,MAAM,uBAAuB,GAClC,qMAAqM,CAAC"}

package/dist/url.d.ts

@@ -0,0 +1,11 @@
+export declare function originOf(url: string): string;
+export declare function registrableHost(hostname: string): string;
+export declare function sameOrigin(a: string, b: string): boolean;
+export declare function absolutize(href: string, base: string): string | null;
+export declare function normalizeUrl(url: string): string;
+export declare function stripTrackingParams(url: string): string;
+export declare function hasTrackingParams(url: string): boolean;
+export declare function isInternalUrl(url: string, origin: string): boolean;
+export declare function pathOf(url: string): string;
+export declare function templateKey(url: string): string;
+//# sourceMappingURL=url.d.ts.map

package/dist/url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../src/url.ts"],"names":[],"mappings":"AAyBA,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAG5C;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAMxD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMpE;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOvD;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAUtD;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAYlE;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO1C;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAW/C"}

package/dist/url.js

@@ -0,0 +1,114 @@
+const TRACKING_PARAMS = new Set([
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_term",
+    "utm_content",
+    "utm_id",
+    "gclid",
+    "gbraid",
+    "wbraid",
+    "fbclid",
+    "msclkid",
+    "ttclid",
+    "twclid",
+    "li_fat_id",
+    "igshid",
+    "mc_cid",
+    "mc_eid",
+    "_ga",
+    "_gl",
+    "yclid",
+    "ref",
+    "referrer",
+]);
+export function originOf(url) {
+    const u = new URL(url);
+    return `${u.protocol}//${u.host}`;
+}
+export function registrableHost(hostname) {
+    const parts = hostname.replace(/\.$/, "").toLowerCase().split(".");
+    if (parts.length <= 2)
+        return parts.join(".");
+    return parts.slice(-2).join(".");
+}
+export function sameOrigin(a, b) {
+    try {
+        return new URL(a).origin === new URL(b).origin;
+    }
+    catch {
+        return false;
+    }
+}
+export function absolutize(href, base) {
+    try {
+        return new URL(href, base).href;
+    }
+    catch {
+        return null;
+    }
+}
+export function normalizeUrl(url) {
+    const u = new URL(url);
+    u.hash = "";
+    if (u.pathname === "")
+        u.pathname = "/";
+    return u.href;
+}
+export function stripTrackingParams(url) {
+    const u = new URL(url);
+    const keys = Array.from(u.searchParams.keys());
+    for (const key of keys) {
+        if (TRACKING_PARAMS.has(key.toLowerCase()))
+            u.searchParams.delete(key);
+    }
+    return u.href;
+}
+export function hasTrackingParams(url) {
+    try {
+        const u = new URL(url);
+        for (const key of u.searchParams.keys()) {
+            if (TRACKING_PARAMS.has(key.toLowerCase()))
+                return true;
+        }
+    }
+    catch {
+        return false;
+    }
+    return false;
+}
+export function isInternalUrl(url, origin) {
+    try {
+        const u = new URL(url);
+        const o = new URL(origin);
+        return (u.hostname === o.hostname ||
+            u.hostname === `www.${o.hostname}` ||
+            o.hostname === `www.${u.hostname}`);
+    }
+    catch {
+        return false;
+    }
+}
+export function pathOf(url) {
+    try {
+        const u = new URL(url);
+        return `${u.pathname}${u.search}`;
+    }
+    catch {
+        return url;
+    }
+}
+export function templateKey(url) {
+    try {
+        const u = new URL(url);
+        const segs = u.pathname
+            .split("/")
+            .filter(Boolean)
+            .map((s) => (/^\d+$/.test(s) || /^[0-9a-f-]{8,}$/i.test(s) ? ":id" : s));
+        return `${segs.length}:${segs.join("/")}`;
+    }
+    catch {
+        return "0:";
+    }
+}
+//# sourceMappingURL=url.js.map

package/dist/url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../src/url.ts"],"names":[],"mappings":"AAAA,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,UAAU;IACV,aAAa;IACb,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,KAAK;IACL,OAAO;IACP,KAAK;IACL,UAAU;CACX,CAAC,CAAC;AAEH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,CAAS,EAAE,CAAS;IAC7C,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,IAAY;IACnD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;IACZ,IAAI,CAAC,CAAC,QAAQ,KAAK,EAAE;QAAE,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;IACxC,OAAO,CAAC,CAAC,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,CAAC,CAAC,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;YACxC,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC1D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,CACL,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;YACzB,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,QAAQ,EAAE;YAClC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,QAAQ,EAAE,CACnC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IAChC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ;aACpB,KAAK,CAAC,GAAG,CAAC;aACV,MAAM,CAAC,OAAO,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@agentsean/crawler",
+  "version": "1.0.0",
+  "license": "AGPL-3.0-only",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.19.0"
+  },
+  "dependencies": {
+    "drizzle-orm": "0.45.2",
+    "cheerio": "1.2.0",
+    "htmlparser2": "12.0.0",
+    "linkedom": "0.18.13",
+    "parse5": "8.0.1",
+    "saxes": "6.0.0",
+    "undici": "8.10.1",
+    "@agentsean/db": "1.0.0"
+  },
+  "optionalDependencies": {
+    "playwright": "1.62.1"
+  },
+  "devDependencies": {
+    "typescript": "~5.9.3",
+    "vitest": "^4.1.11"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/seziro-team/agentsean.git",
+    "directory": "packages/crawler"
+  },
+  "homepage": "https://github.com/seziro-team/agentsean#readme",
+  "bugs": {
+    "url": "https://github.com/seziro-team/agentsean/issues"
+  },
+  "author": "Seziro (https://seziro.com)",
+  "description": "Agent Sean — crawler module",
+  "keywords": [
+    "seo",
+    "agent",
+    "agentsean",
+    "seo-automation"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run --config ../../vitest.config.ts"
+  }
+}