@dromney/mapthis 0.1.0

package/dist/schemas-Dy5coqXo.d.ts

@@ -0,0 +1,484 @@
+import { z } from 'zod';
+
+/**
+ * Runtime validation schemas and their inferred types.
+ *
+ * These schemas define the public contract between producers and consumers of
+ * mapthis data. They are intentionally framework-agnostic — no Prisma, no
+ * Next.js, no auth assumptions.
+ */
+declare const llmProvider: z.ZodLiteral<"openai">;
+type LlmProvider = z.infer<typeof llmProvider>;
+declare const placeProvider: z.ZodUnion<[z.ZodLiteral<"google">, z.ZodLiteral<"locationiq">]>;
+type PlaceProvider = z.infer<typeof placeProvider>;
+declare const generationSourceType: z.ZodUnion<[z.ZodLiteral<"blank">, z.ZodLiteral<"autocomplete">, z.ZodLiteral<"url">, z.ZodLiteral<"text">, z.ZodLiteral<"list">]>;
+type GenerationSourceType = z.infer<typeof generationSourceType>;
+declare const latitude: z.ZodNumber;
+declare const longitude: z.ZodNumber;
+declare const coordinates: z.ZodObject<{
+    lat: z.ZodNumber;
+    lng: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    lat: number;
+    lng: number;
+}, {
+    lat: number;
+    lng: number;
+}>;
+type Coordinates = z.infer<typeof coordinates>;
+declare const placesFromUrl: z.ZodString;
+declare const placesFromList: z.ZodEffects<z.ZodEffects<z.ZodString, string, unknown>, string, unknown>;
+declare const placesFromText: z.ZodString;
+declare const mapExternalId: z.ZodString;
+declare const mapTitle: z.ZodString;
+declare const mapDescription: z.ZodString;
+declare const mapCreationQuery: z.ZodString;
+declare const groupName: z.ZodString;
+declare const groupDescription: z.ZodString;
+declare const groupColor: z.ZodString;
+declare const placeName: z.ZodString;
+declare const placeDescription: z.ZodString;
+declare const createMapBase: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const createMapSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+} & {
+    sourceType: z.ZodUnion<[z.ZodLiteral<"blank">, z.ZodLiteral<"autocomplete">, z.ZodLiteral<"url">, z.ZodLiteral<"text">, z.ZodLiteral<"list">]>;
+    source: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+    source?: string | undefined;
+}, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+    source?: string | undefined;
+}>;
+declare const createBlankMapSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const createMapFromUrlSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+} & {
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    url: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const createMapFromListSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+} & {
+    list: z.ZodEffects<z.ZodEffects<z.ZodString, string, unknown>, string, unknown>;
+}, "strip", z.ZodTypeAny, {
+    list: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    list?: unknown;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const createMapFromTextSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+} & {
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    text: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    text: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const createMapFromQueryOrUrlSchema: z.ZodObject<{
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    externalId: z.ZodOptional<z.ZodString>;
+} & {
+    queryOrUrl: z.ZodUnion<[z.ZodString, z.ZodString]>;
+}, "strip", z.ZodTypeAny, {
+    queryOrUrl: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}, {
+    queryOrUrl: string;
+    title?: string | undefined;
+    description?: string | undefined;
+    externalId?: string | undefined;
+}>;
+declare const addPlacesBaseSchema: z.ZodObject<{
+    sourceType: z.ZodUnion<[z.ZodLiteral<"blank">, z.ZodLiteral<"autocomplete">, z.ZodLiteral<"url">, z.ZodLiteral<"text">, z.ZodLiteral<"list">]>;
+    source: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    source: string;
+}, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    source: string;
+}>;
+declare const addPlacesFromSourceSchema: z.ZodObject<{
+    sourceType: z.ZodUnion<[z.ZodLiteral<"blank">, z.ZodLiteral<"autocomplete">, z.ZodLiteral<"url">, z.ZodLiteral<"text">, z.ZodLiteral<"list">]>;
+    source: z.ZodString;
+} & {
+    mapId: z.ZodString;
+    groupId: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    source: string;
+    mapId: string;
+    groupId?: string | undefined;
+}, {
+    sourceType: "blank" | "autocomplete" | "url" | "text" | "list";
+    source: string;
+    mapId: string;
+    groupId?: string | undefined;
+}>;
+declare const addPlacesFromUrlSchema: z.ZodObject<{
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+}, {
+    url: string;
+}>;
+declare const addPlacesFromListSchema: z.ZodObject<{
+    list: z.ZodEffects<z.ZodEffects<z.ZodString, string, unknown>, string, unknown>;
+}, "strip", z.ZodTypeAny, {
+    list: string;
+}, {
+    list?: unknown;
+}>;
+declare const addPlacesFromTextSchema: z.ZodObject<{
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    text: string;
+}, {
+    text: string;
+}>;
+declare const providerAutocompletePlace: z.ZodObject<{
+    query: z.ZodString;
+    provider: z.ZodLiteral<"google">;
+    providerId: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    address: z.ZodString;
+    lat: z.ZodNumber;
+    lng: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    lat: number;
+    lng: number;
+    query: string;
+    provider: "google";
+    providerId: string;
+    name: string;
+    address: string;
+    description?: string | undefined;
+}, {
+    lat: number;
+    lng: number;
+    query: string;
+    provider: "google";
+    providerId: string;
+    name: string;
+    address: string;
+    description?: string | undefined;
+}>;
+type ProviderAutocompletePlace = z.infer<typeof providerAutocompletePlace>;
+declare const addPlacesFromClientAutocompleteSchema: z.ZodObject<{
+    query: z.ZodString;
+    provider: z.ZodLiteral<"google">;
+    providerId: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    address: z.ZodString;
+    lat: z.ZodNumber;
+    lng: z.ZodNumber;
+} & {
+    mapId: z.ZodString;
+    groupId: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    lat: number;
+    lng: number;
+    mapId: string;
+    query: string;
+    provider: "google";
+    providerId: string;
+    name: string;
+    address: string;
+    description?: string | undefined;
+    groupId?: string | undefined;
+}, {
+    lat: number;
+    lng: number;
+    mapId: string;
+    query: string;
+    provider: "google";
+    providerId: string;
+    name: string;
+    address: string;
+    description?: string | undefined;
+    groupId?: string | undefined;
+}>;
+declare const updateMapSchema: z.ZodObject<{
+    title: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    userPublic: z.ZodOptional<z.ZodBoolean>;
+    partnerPublic: z.ZodOptional<z.ZodBoolean>;
+    url: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+}, "strip", z.ZodTypeAny, {
+    title: string;
+    url?: string | null | undefined;
+    description?: string | undefined;
+    userPublic?: boolean | undefined;
+    partnerPublic?: boolean | undefined;
+}, {
+    title: string;
+    url?: string | null | undefined;
+    description?: string | undefined;
+    userPublic?: boolean | undefined;
+    partnerPublic?: boolean | undefined;
+}>;
+declare const reorderMapSchema: z.ZodObject<{
+    groups: z.ZodArray<z.ZodObject<{
+        id: z.ZodNullable<z.ZodString>;
+        position: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        id: string | null;
+        position: number;
+    }, {
+        id: string | null;
+        position: number;
+    }>, "many">;
+    places: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        groupId: z.ZodNullable<z.ZodString>;
+        position: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        groupId: string | null;
+        id: string;
+        position: number;
+    }, {
+        groupId: string | null;
+        id: string;
+        position: number;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    groups: {
+        id: string | null;
+        position: number;
+    }[];
+    places: {
+        groupId: string | null;
+        id: string;
+        position: number;
+    }[];
+}, {
+    groups: {
+        id: string | null;
+        position: number;
+    }[];
+    places: {
+        groupId: string | null;
+        id: string;
+        position: number;
+    }[];
+}>;
+declare const updatePlaceFormSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    hidden: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    id: string;
+    hidden: boolean;
+    description?: string | undefined;
+}, {
+    name: string;
+    id: string;
+    hidden: boolean;
+    description?: string | undefined;
+}>;
+declare const searchSchema: z.ZodObject<{
+    query: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    query: string;
+}, {
+    query: string;
+}>;
+declare const createGroupSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    parentGroupId: z.ZodOptional<z.ZodString>;
+    color: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    description?: string | undefined;
+    parentGroupId?: string | undefined;
+    color?: string | undefined;
+}, {
+    name: string;
+    description?: string | undefined;
+    parentGroupId?: string | undefined;
+    color?: string | undefined;
+}>;
+declare const editGroupSchema: z.ZodObject<{
+    id: z.ZodString;
+    color: z.ZodNullable<z.ZodString>;
+    hidden: z.ZodBoolean;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    id: string;
+    hidden: boolean;
+    color: string | null;
+    description?: string | undefined;
+}, {
+    name: string;
+    id: string;
+    hidden: boolean;
+    color: string | null;
+    description?: string | undefined;
+}>;
+declare const placeMeta: z.ZodObject<{
+    id: z.ZodString;
+    updatedAt: z.ZodDate;
+    groupId: z.ZodNullable<z.ZodString>;
+    position: z.ZodNullable<z.ZodNumber>;
+    error: z.ZodNullable<z.ZodString>;
+    lat: z.ZodNullable<z.ZodNumber>;
+    lon: z.ZodNullable<z.ZodNumber>;
+    name: z.ZodNullable<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    lat: number | null;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number | null;
+    updatedAt: Date;
+    error: string | null;
+    lon: number | null;
+}, {
+    lat: number | null;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number | null;
+    updatedAt: Date;
+    error: string | null;
+    lon: number | null;
+}>;
+type PlaceMeta = z.infer<typeof placeMeta>;
+declare const positionedPlaceMeta: z.ZodObject<{
+    id: z.ZodString;
+    updatedAt: z.ZodDate;
+    groupId: z.ZodNullable<z.ZodString>;
+    error: z.ZodNullable<z.ZodString>;
+    lat: z.ZodNullable<z.ZodNumber>;
+    lon: z.ZodNullable<z.ZodNumber>;
+    name: z.ZodNullable<z.ZodString>;
+} & {
+    position: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    lat: number | null;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number;
+    updatedAt: Date;
+    error: string | null;
+    lon: number | null;
+}, {
+    lat: number | null;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number;
+    updatedAt: Date;
+    error: string | null;
+    lon: number | null;
+}>;
+type PositionedPlaceMeta = z.infer<typeof positionedPlaceMeta>;
+declare const mappablePlaceMeta: z.ZodObject<{
+    id: z.ZodString;
+    updatedAt: z.ZodDate;
+    groupId: z.ZodNullable<z.ZodString>;
+    position: z.ZodNullable<z.ZodNumber>;
+    name: z.ZodNullable<z.ZodString>;
+} & {
+    lat: z.ZodNumber;
+    lon: z.ZodNumber;
+    error: z.ZodNull;
+}, "strip", z.ZodTypeAny, {
+    lat: number;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number | null;
+    updatedAt: Date;
+    error: null;
+    lon: number;
+}, {
+    lat: number;
+    groupId: string | null;
+    name: string | null;
+    id: string;
+    position: number | null;
+    updatedAt: Date;
+    error: null;
+    lon: number;
+}>;
+type MappablePlaceMeta = z.infer<typeof mappablePlaceMeta>;
+/**
+ * Filter a list of place projections down to those that are mappable
+ * (have coordinates and no error).
+ */
+declare function getMappablePlaceMetas(places: (PlaceMeta | null | undefined)[]): MappablePlaceMeta[];
+
+export { longitude as A, mapCreationQuery as B, type Coordinates as C, mapDescription as D, mapExternalId as E, mapTitle as F, type GenerationSourceType as G, mappablePlaceMeta as H, placeDescription as I, placeMeta as J, placeName as K, type LlmProvider as L, type MappablePlaceMeta as M, placeProvider as N, placesFromList as O, type PlaceMeta as P, placesFromText as Q, placesFromUrl as R, positionedPlaceMeta as S, providerAutocompletePlace as T, reorderMapSchema as U, searchSchema as V, updateMapSchema as W, updatePlaceFormSchema as X, type PlaceProvider as a, type PositionedPlaceMeta as b, type ProviderAutocompletePlace as c, addPlacesBaseSchema as d, addPlacesFromClientAutocompleteSchema as e, addPlacesFromListSchema as f, addPlacesFromSourceSchema as g, addPlacesFromTextSchema as h, addPlacesFromUrlSchema as i, coordinates as j, createBlankMapSchema as k, createGroupSchema as l, createMapBase as m, createMapFromListSchema as n, createMapFromQueryOrUrlSchema as o, createMapFromTextSchema as p, createMapFromUrlSchema as q, createMapSchema as r, editGroupSchema as s, generationSourceType as t, getMappablePlaceMetas as u, groupColor as v, groupDescription as w, groupName as x, latitude as y, llmProvider as z };

package/dist/scrape/index.cjs

@@ -0,0 +1,133 @@
+'use strict';
+
+var zod = require('zod');
+var htmlToText$1 = require('html-to-text');
+
+// src/types/errors.ts
+var MapthisError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MapthisError";
+  }
+};
+
+// src/scrape/errors.ts
+var ScrapeError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "ScrapeError";
+  }
+};
+var InvalidUrlError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "InvalidUrlError";
+  }
+};
+var HtmlUnauthorizedError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlUnauthorizedError";
+  }
+};
+var HtmlToTextError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlToTextError";
+  }
+};
+var withHttps = (url) => {
+  const hasHttp = /^http?:\/\//i.test(url);
+  const hasHttps = /^https?:\/\//i.test(url);
+  if (!hasHttp && !hasHttps) return `https://${url}`;
+  return url;
+};
+async function getHtmlFromUrl(url) {
+  const urlFixed = withHttps(url.toLowerCase());
+  const isUrl = zod.z.string().url().safeParse(urlFixed);
+  if (!isUrl.success) throw new InvalidUrlError(`Invalid URL: ${url}`);
+  try {
+    const response = await fetch(urlFixed);
+    const html = await response.text();
+    if (response.status === 403 || response.status === 401) {
+      throw new HtmlUnauthorizedError(
+        `${response.status} Not authorized to scrape this website`
+      );
+    }
+    if (!response.ok) {
+      throw new ScrapeError(
+        `Bad response when getting HTML. Status: ${response.status} ${response.statusText}`
+      );
+    }
+    return html;
+  } catch (error) {
+    if (error instanceof TypeError) {
+      throw new InvalidUrlError(`Invalid URL. Original error: ${error.message}`);
+    }
+    throw error;
+  }
+}
+var SKIP_SELECTORS = [
+  "img",
+  "header",
+  "footer",
+  "audio",
+  "button",
+  "canvas",
+  "code",
+  "nav",
+  "#nav",
+  "figure",
+  "figcaption",
+  ".comment",
+  ".comments",
+  "#comments",
+  "#related-posts",
+  "#related",
+  ".related",
+  ".related-posts"
+];
+function htmlToText(html) {
+  try {
+    let out = htmlToText$1.convert(html, {
+      wordwrap: false,
+      selectors: [
+        {
+          selector: "a",
+          options: {
+            ignoreHref: true,
+            hideLinkHrefIfSameAsText: true
+          }
+        },
+        ...SKIP_SELECTORS.map((tag) => ({ selector: tag, format: "skip" }))
+      ]
+    });
+    out = out.replaceAll("\r\n", "\n");
+    for (let i = 0; i < 20; i++) {
+      out = out.replaceAll("\n\n\n", "\n\n");
+      out = out.replaceAll("  ", " ");
+    }
+    return out;
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new HtmlToTextError(`HTML to text conversion failed: ${error.message}`, {
+        cause: error
+      });
+    }
+    throw new HtmlToTextError("HTML to text conversion failed");
+  }
+}
+async function getTextFromUrl(url) {
+  const html = await getHtmlFromUrl(url);
+  return htmlToText(html);
+}
+
+exports.HtmlToTextError = HtmlToTextError;
+exports.HtmlUnauthorizedError = HtmlUnauthorizedError;
+exports.InvalidUrlError = InvalidUrlError;
+exports.ScrapeError = ScrapeError;
+exports.getHtmlFromUrl = getHtmlFromUrl;
+exports.getTextFromUrl = getTextFromUrl;
+exports.htmlToText = htmlToText;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/scrape/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/types/errors.ts","../../src/scrape/errors.ts","../../src/scrape/html.ts","../../src/scrape/text.ts"],"names":["z","convert"],"mappings":";;;;;;AAOO,IAAM,YAAA,GAAN,cAA2B,KAAA,CAAM;AAAA,EACpC,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,cAAA;AAAA,EAChB;AACJ,CAAA;;;ACVO,IAAM,WAAA,GAAN,cAA0B,YAAA,CAAa;AAAA,EAC1C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,aAAA;AAAA,EAChB;AACJ;AAKO,IAAM,eAAA,GAAN,cAA8B,WAAA,CAAY;AAAA,EAC7C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,iBAAA;AAAA,EAChB;AACJ;AAMO,IAAM,qBAAA,GAAN,cAAoC,WAAA,CAAY;AAAA,EACnD,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,uBAAA;AAAA,EAChB;AACJ;AAKO,IAAM,eAAA,GAAN,cAA8B,WAAA,CAAY;AAAA,EAC7C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,iBAAA;AAAA,EAChB;AACJ;ACnCA,IAAM,SAAA,GAAY,CAAC,GAAA,KAAwB;AACvC,EAAA,MAAM,OAAA,GAAU,cAAA,CAAe,IAAA,CAAK,GAAG,CAAA;AACvC,EAAA,MAAM,QAAA,GAAW,eAAA,CAAgB,IAAA,CAAK,GAAG,CAAA;AACzC,EAAA,IAAI,CAAC,OAAA,IAAW,CAAC,QAAA,EAAU,OAAO,WAAW,GAAG,CAAA,CAAA;AAChD,EAAA,OAAO,GAAA;AACX,CAAA;AAgBA,eAAsB,eAAe,GAAA,EAA8B;AAC/D,EAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,WAAA,EAAa,CAAA;AAC5C,EAAA,MAAM,QAAQA,KAAA,CAAE,MAAA,GAAS,GAAA,EAAI,CAAE,UAAU,QAAQ,CAAA;AACjD,EAAA,IAAI,CAAC,MAAM,OAAA,EAAS,MAAM,IAAI,eAAA,CAAgB,CAAA,aAAA,EAAgB,GAAG,CAAA,CAAE,CAAA;AAEnE,EAAA,IAAI;AACA,IAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,QAAQ,CAAA;AACrC,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AACjC,IAAA,IAAI,QAAA,CAAS,MAAA,KAAW,GAAA,IAAO,QAAA,CAAS,WAAW,GAAA,EAAK;AACpD,MAAA,MAAM,IAAI,qBAAA;AAAA,QACN,CAAA,EAAG,SAAS,MAAM,CAAA,sCAAA;AAAA,OACtB;AAAA,IACJ;AACA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AACd,MAAA,MAAM,IAAI,WAAA;AAAA,QACN,CAAA,wCAAA,EAA2C,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,OACrF;AAAA,IACJ;AACA,IAAA,OAAO,IAAA;AAAA,EACX,SAAS,KAAA,EAAO;AACZ,IAAA,IAAI,iBAAiB,SAAA,EAAW;AAC5B,MAAA,MAAM,IAAI,eAAA,CAAgB,CAAA,6BAAA,EAAgC,KAAA,CAAM,OAAO,CAAA,CAAE,CAAA;AAAA,IAC7E;AACA,IAAA,MAAM,KAAA;AAAA,EACV;AACJ;AC7CA,IAAM,cAAA,GAAiB;AAAA,EACnB,KAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,OAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EACA,YAAA;AAAA,EACA,UAAA;AAAA,EACA,WAAA;AAAA,EACA,WAAA;AAAA,EACA,gBAAA;AAAA,EACA,UAAA;AAAA,EACA,UAAA;AAAA,EACA;AACJ,CAAA;AAYO,SAAS,WAAW,IAAA,EAAsB;AAC7C,EAAA,IAAI;AACA,IAAA,IAAI,GAAA,GAAMC,qBAAQ,IAAA,EAAM;AAAA,MACpB,QAAA,EAAU,KAAA;AAAA,MACV,SAAA,EAAW;AAAA,QACP;AAAA,UACI,QAAA,EAAU,GAAA;AAAA,UACV,OAAA,EAAS;AAAA,YACL,UAAA,EAAY,IAAA;AAAA,YACZ,wBAAA,EAA0B;AAAA;AAC9B,SACJ;AAAA,QACA,GAAG,cAAA,CAAe,GAAA,CAAI,CAAC,GAAA,MAAS,EAAE,QAAA,EAAU,GAAA,EAAK,MAAA,EAAQ,MAAA,EAAO,CAAE;AAAA;AACtE,KACH,CAAA;AACD,IAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,MAAA,EAAQ,IAAI,CAAA;AAGjC,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,EAAA,EAAI,CAAA,EAAA,EAAK;AACzB,MAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,QAAA,EAAU,MAAM,CAAA;AACrC,MAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,IAAA,EAAM,GAAG,CAAA;AAAA,IAClC;AACA,IAAA,OAAO,GAAA;AAAA,EACX,SAAS,KAAA,EAAO;AACZ,IAAA,IAAI,iBAAiB,KAAA,EAAO;AACxB,MAAA,MAAM,IAAI,eAAA,CAAgB,CAAA,gCAAA,EAAmC,KAAA,CAAM,OAAO,CAAA,CAAA,EAAI;AAAA,QAC1E,KAAA,EAAO;AAAA,OACV,CAAA;AAAA,IACL;AACA,IAAA,MAAM,IAAI,gBAAgB,gCAAgC,CAAA;AAAA,EAC9D;AACJ;AAOA,eAAsB,eAAe,GAAA,EAA8B;AAC/D,EAAA,MAAM,IAAA,GAAO,MAAM,cAAA,CAAe,GAAG,CAAA;AACrC,EAAA,OAAO,WAAW,IAAI,CAAA;AAC1B","file":"index.cjs","sourcesContent":["/**\n * Base error class for every error thrown by the mapthis package.\n *\n * All feature-specific error classes (scrape, search, ai, geocoding) extend\n * this, so consumers can do `catch (e) { if (e instanceof MapthisError) ... }`\n * to distinguish library errors from other exceptions.\n */\nexport class MapthisError extends Error {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"MapthisError\"\n    }\n}\n","import { MapthisError } from \"../types/errors\"\n\nexport class ScrapeError extends MapthisError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"ScrapeError\"\n    }\n}\n\n/**\n * Thrown when a provided URL fails validation or cannot be parsed by `fetch`.\n */\nexport class InvalidUrlError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"InvalidUrlError\"\n    }\n}\n\n/**\n * Thrown when a remote server returns 401 or 403 for an HTML fetch, meaning\n * the page does not permit anonymous scraping.\n */\nexport class HtmlUnauthorizedError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"HtmlUnauthorizedError\"\n    }\n}\n\n/**\n * Thrown when the underlying `html-to-text` conversion fails.\n */\nexport class HtmlToTextError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"HtmlToTextError\"\n    }\n}\n","import { z } from \"zod\"\nimport { HtmlUnauthorizedError, InvalidUrlError, ScrapeError } from \"./errors\"\n\nconst withHttps = (url: string): string => {\n    const hasHttp = /^http?:\\/\\//i.test(url)\n    const hasHttps = /^https?:\\/\\//i.test(url)\n    if (!hasHttp && !hasHttps) return `https://${url}`\n    return url\n}\n\n/**\n * Fetch the raw HTML of a URL, performing light validation first.\n *\n * - URLs without a scheme are prefixed with `https://`.\n * - URL shape is validated via Zod before the network call.\n * - 401/403 responses throw {@link HtmlUnauthorizedError}.\n * - Other non-OK responses throw {@link ScrapeError} with the status text.\n * - `TypeError` from `fetch` (typically malformed URL) is rethrown as\n *   {@link InvalidUrlError}.\n *\n * This function does not set any special headers, user agent, or cookies.\n * Sites with aggressive bot-detection may return empty bodies or errors; the\n * caller is responsible for retry / user-agent rotation strategies if needed.\n */\nexport async function getHtmlFromUrl(url: string): Promise<string> {\n    const urlFixed = withHttps(url.toLowerCase())\n    const isUrl = z.string().url().safeParse(urlFixed)\n    if (!isUrl.success) throw new InvalidUrlError(`Invalid URL: ${url}`)\n\n    try {\n        const response = await fetch(urlFixed)\n        const html = await response.text()\n        if (response.status === 403 || response.status === 401) {\n            throw new HtmlUnauthorizedError(\n                `${response.status} Not authorized to scrape this website`,\n            )\n        }\n        if (!response.ok) {\n            throw new ScrapeError(\n                `Bad response when getting HTML. Status: ${response.status} ${response.statusText}`,\n            )\n        }\n        return html\n    } catch (error) {\n        if (error instanceof TypeError) {\n            throw new InvalidUrlError(`Invalid URL. Original error: ${error.message}`)\n        }\n        throw error\n    }\n}\n","import { convert } from \"html-to-text\"\nimport { HtmlToTextError } from \"./errors\"\nimport { getHtmlFromUrl } from \"./html\"\n\nconst SKIP_SELECTORS = [\n    \"img\",\n    \"header\",\n    \"footer\",\n    \"audio\",\n    \"button\",\n    \"canvas\",\n    \"code\",\n    \"nav\",\n    \"#nav\",\n    \"figure\",\n    \"figcaption\",\n    \".comment\",\n    \".comments\",\n    \"#comments\",\n    \"#related-posts\",\n    \"#related\",\n    \".related\",\n    \".related-posts\",\n] as const\n\n/**\n * Convert raw HTML to plain text suitable for LLM consumption.\n *\n * The converter is configured to:\n * - strip anchor hrefs when they duplicate the link text;\n * - skip navigation, footer, comments, related-post widgets, images, and\n *   other non-content elements;\n * - collapse repeated blank lines and double spaces so token count doesn't\n *   balloon from whitespace.\n */\nexport function htmlToText(html: string): string {\n    try {\n        let out = convert(html, {\n            wordwrap: false,\n            selectors: [\n                {\n                    selector: \"a\",\n                    options: {\n                        ignoreHref: true,\n                        hideLinkHrefIfSameAsText: true,\n                    },\n                },\n                ...SKIP_SELECTORS.map((tag) => ({ selector: tag, format: \"skip\" })),\n            ],\n        })\n        out = out.replaceAll(\"\\r\\n\", \"\\n\")\n        // Iteratively collapse blank lines and double spaces. 20 passes is\n        // overkill for any realistic input but cheap and bounds worst case.\n        for (let i = 0; i < 20; i++) {\n            out = out.replaceAll(\"\\n\\n\\n\", \"\\n\\n\")\n            out = out.replaceAll(\"  \", \" \")\n        }\n        return out\n    } catch (error) {\n        if (error instanceof Error) {\n            throw new HtmlToTextError(`HTML to text conversion failed: ${error.message}`, {\n                cause: error,\n            })\n        }\n        throw new HtmlToTextError(\"HTML to text conversion failed\")\n    }\n}\n\n/**\n * Convenience: fetch a URL and convert its HTML to plain text in one call.\n *\n * Equivalent to `htmlToText(await getHtmlFromUrl(url))`.\n */\nexport async function getTextFromUrl(url: string): Promise<string> {\n    const html = await getHtmlFromUrl(url)\n    return htmlToText(html)\n}\n"]}

package/dist/scrape/index.d.cts

@@ -0,0 +1,60 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.cjs';
+
+declare class ScrapeError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a provided URL fails validation or cannot be parsed by `fetch`.
+ */
+declare class InvalidUrlError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a remote server returns 401 or 403 for an HTML fetch, meaning
+ * the page does not permit anonymous scraping.
+ */
+declare class HtmlUnauthorizedError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when the underlying `html-to-text` conversion fails.
+ */
+declare class HtmlToTextError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+/**
+ * Fetch the raw HTML of a URL, performing light validation first.
+ *
+ * - URLs without a scheme are prefixed with `https://`.
+ * - URL shape is validated via Zod before the network call.
+ * - 401/403 responses throw {@link HtmlUnauthorizedError}.
+ * - Other non-OK responses throw {@link ScrapeError} with the status text.
+ * - `TypeError` from `fetch` (typically malformed URL) is rethrown as
+ *   {@link InvalidUrlError}.
+ *
+ * This function does not set any special headers, user agent, or cookies.
+ * Sites with aggressive bot-detection may return empty bodies or errors; the
+ * caller is responsible for retry / user-agent rotation strategies if needed.
+ */
+declare function getHtmlFromUrl(url: string): Promise<string>;
+
+/**
+ * Convert raw HTML to plain text suitable for LLM consumption.
+ *
+ * The converter is configured to:
+ * - strip anchor hrefs when they duplicate the link text;
+ * - skip navigation, footer, comments, related-post widgets, images, and
+ *   other non-content elements;
+ * - collapse repeated blank lines and double spaces so token count doesn't
+ *   balloon from whitespace.
+ */
+declare function htmlToText(html: string): string;
+/**
+ * Convenience: fetch a URL and convert its HTML to plain text in one call.
+ *
+ * Equivalent to `htmlToText(await getHtmlFromUrl(url))`.
+ */
+declare function getTextFromUrl(url: string): Promise<string>;
+
+export { HtmlToTextError, HtmlUnauthorizedError, InvalidUrlError, ScrapeError, getHtmlFromUrl, getTextFromUrl, htmlToText };