@dromney/mapthis 0.1.0

package/dist/scrape/index.d.ts

@@ -0,0 +1,60 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.js';
+
+declare class ScrapeError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a provided URL fails validation or cannot be parsed by `fetch`.
+ */
+declare class InvalidUrlError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a remote server returns 401 or 403 for an HTML fetch, meaning
+ * the page does not permit anonymous scraping.
+ */
+declare class HtmlUnauthorizedError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when the underlying `html-to-text` conversion fails.
+ */
+declare class HtmlToTextError extends ScrapeError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+/**
+ * Fetch the raw HTML of a URL, performing light validation first.
+ *
+ * - URLs without a scheme are prefixed with `https://`.
+ * - URL shape is validated via Zod before the network call.
+ * - 401/403 responses throw {@link HtmlUnauthorizedError}.
+ * - Other non-OK responses throw {@link ScrapeError} with the status text.
+ * - `TypeError` from `fetch` (typically malformed URL) is rethrown as
+ *   {@link InvalidUrlError}.
+ *
+ * This function does not set any special headers, user agent, or cookies.
+ * Sites with aggressive bot-detection may return empty bodies or errors; the
+ * caller is responsible for retry / user-agent rotation strategies if needed.
+ */
+declare function getHtmlFromUrl(url: string): Promise<string>;
+
+/**
+ * Convert raw HTML to plain text suitable for LLM consumption.
+ *
+ * The converter is configured to:
+ * - strip anchor hrefs when they duplicate the link text;
+ * - skip navigation, footer, comments, related-post widgets, images, and
+ *   other non-content elements;
+ * - collapse repeated blank lines and double spaces so token count doesn't
+ *   balloon from whitespace.
+ */
+declare function htmlToText(html: string): string;
+/**
+ * Convenience: fetch a URL and convert its HTML to plain text in one call.
+ *
+ * Equivalent to `htmlToText(await getHtmlFromUrl(url))`.
+ */
+declare function getTextFromUrl(url: string): Promise<string>;
+
+export { HtmlToTextError, HtmlUnauthorizedError, InvalidUrlError, ScrapeError, getHtmlFromUrl, getTextFromUrl, htmlToText };

package/dist/scrape/index.js

@@ -0,0 +1,125 @@
+import { z } from 'zod';
+import { convert } from 'html-to-text';
+
+// src/types/errors.ts
+var MapthisError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MapthisError";
+  }
+};
+
+// src/scrape/errors.ts
+var ScrapeError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "ScrapeError";
+  }
+};
+var InvalidUrlError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "InvalidUrlError";
+  }
+};
+var HtmlUnauthorizedError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlUnauthorizedError";
+  }
+};
+var HtmlToTextError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlToTextError";
+  }
+};
+var withHttps = (url) => {
+  const hasHttp = /^http?:\/\//i.test(url);
+  const hasHttps = /^https?:\/\//i.test(url);
+  if (!hasHttp && !hasHttps) return `https://${url}`;
+  return url;
+};
+async function getHtmlFromUrl(url) {
+  const urlFixed = withHttps(url.toLowerCase());
+  const isUrl = z.string().url().safeParse(urlFixed);
+  if (!isUrl.success) throw new InvalidUrlError(`Invalid URL: ${url}`);
+  try {
+    const response = await fetch(urlFixed);
+    const html = await response.text();
+    if (response.status === 403 || response.status === 401) {
+      throw new HtmlUnauthorizedError(
+        `${response.status} Not authorized to scrape this website`
+      );
+    }
+    if (!response.ok) {
+      throw new ScrapeError(
+        `Bad response when getting HTML. Status: ${response.status} ${response.statusText}`
+      );
+    }
+    return html;
+  } catch (error) {
+    if (error instanceof TypeError) {
+      throw new InvalidUrlError(`Invalid URL. Original error: ${error.message}`);
+    }
+    throw error;
+  }
+}
+var SKIP_SELECTORS = [
+  "img",
+  "header",
+  "footer",
+  "audio",
+  "button",
+  "canvas",
+  "code",
+  "nav",
+  "#nav",
+  "figure",
+  "figcaption",
+  ".comment",
+  ".comments",
+  "#comments",
+  "#related-posts",
+  "#related",
+  ".related",
+  ".related-posts"
+];
+function htmlToText(html) {
+  try {
+    let out = convert(html, {
+      wordwrap: false,
+      selectors: [
+        {
+          selector: "a",
+          options: {
+            ignoreHref: true,
+            hideLinkHrefIfSameAsText: true
+          }
+        },
+        ...SKIP_SELECTORS.map((tag) => ({ selector: tag, format: "skip" }))
+      ]
+    });
+    out = out.replaceAll("\r\n", "\n");
+    for (let i = 0; i < 20; i++) {
+      out = out.replaceAll("\n\n\n", "\n\n");
+      out = out.replaceAll("  ", " ");
+    }
+    return out;
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new HtmlToTextError(`HTML to text conversion failed: ${error.message}`, {
+        cause: error
+      });
+    }
+    throw new HtmlToTextError("HTML to text conversion failed");
+  }
+}
+async function getTextFromUrl(url) {
+  const html = await getHtmlFromUrl(url);
+  return htmlToText(html);
+}
+
+export { HtmlToTextError, HtmlUnauthorizedError, InvalidUrlError, ScrapeError, getHtmlFromUrl, getTextFromUrl, htmlToText };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/scrape/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/types/errors.ts","../../src/scrape/errors.ts","../../src/scrape/html.ts","../../src/scrape/text.ts"],"names":[],"mappings":";;;;AAOO,IAAM,YAAA,GAAN,cAA2B,KAAA,CAAM;AAAA,EACpC,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,cAAA;AAAA,EAChB;AACJ,CAAA;;;ACVO,IAAM,WAAA,GAAN,cAA0B,YAAA,CAAa;AAAA,EAC1C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,aAAA;AAAA,EAChB;AACJ;AAKO,IAAM,eAAA,GAAN,cAA8B,WAAA,CAAY;AAAA,EAC7C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,iBAAA;AAAA,EAChB;AACJ;AAMO,IAAM,qBAAA,GAAN,cAAoC,WAAA,CAAY;AAAA,EACnD,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,uBAAA;AAAA,EAChB;AACJ;AAKO,IAAM,eAAA,GAAN,cAA8B,WAAA,CAAY;AAAA,EAC7C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,iBAAA;AAAA,EAChB;AACJ;ACnCA,IAAM,SAAA,GAAY,CAAC,GAAA,KAAwB;AACvC,EAAA,MAAM,OAAA,GAAU,cAAA,CAAe,IAAA,CAAK,GAAG,CAAA;AACvC,EAAA,MAAM,QAAA,GAAW,eAAA,CAAgB,IAAA,CAAK,GAAG,CAAA;AACzC,EAAA,IAAI,CAAC,OAAA,IAAW,CAAC,QAAA,EAAU,OAAO,WAAW,GAAG,CAAA,CAAA;AAChD,EAAA,OAAO,GAAA;AACX,CAAA;AAgBA,eAAsB,eAAe,GAAA,EAA8B;AAC/D,EAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,WAAA,EAAa,CAAA;AAC5C,EAAA,MAAM,QAAQ,CAAA,CAAE,MAAA,GAAS,GAAA,EAAI,CAAE,UAAU,QAAQ,CAAA;AACjD,EAAA,IAAI,CAAC,MAAM,OAAA,EAAS,MAAM,IAAI,eAAA,CAAgB,CAAA,aAAA,EAAgB,GAAG,CAAA,CAAE,CAAA;AAEnE,EAAA,IAAI;AACA,IAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,QAAQ,CAAA;AACrC,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AACjC,IAAA,IAAI,QAAA,CAAS,MAAA,KAAW,GAAA,IAAO,QAAA,CAAS,WAAW,GAAA,EAAK;AACpD,MAAA,MAAM,IAAI,qBAAA;AAAA,QACN,CAAA,EAAG,SAAS,MAAM,CAAA,sCAAA;AAAA,OACtB;AAAA,IACJ;AACA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AACd,MAAA,MAAM,IAAI,WAAA;AAAA,QACN,CAAA,wCAAA,EAA2C,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,OACrF;AAAA,IACJ;AACA,IAAA,OAAO,IAAA;AAAA,EACX,SAAS,KAAA,EAAO;AACZ,IAAA,IAAI,iBAAiB,SAAA,EAAW;AAC5B,MAAA,MAAM,IAAI,eAAA,CAAgB,CAAA,6BAAA,EAAgC,KAAA,CAAM,OAAO,CAAA,CAAE,CAAA;AAAA,IAC7E;AACA,IAAA,MAAM,KAAA;AAAA,EACV;AACJ;AC7CA,IAAM,cAAA,GAAiB;AAAA,EACnB,KAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,OAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EACA,YAAA;AAAA,EACA,UAAA;AAAA,EACA,WAAA;AAAA,EACA,WAAA;AAAA,EACA,gBAAA;AAAA,EACA,UAAA;AAAA,EACA,UAAA;AAAA,EACA;AACJ,CAAA;AAYO,SAAS,WAAW,IAAA,EAAsB;AAC7C,EAAA,IAAI;AACA,IAAA,IAAI,GAAA,GAAM,QAAQ,IAAA,EAAM;AAAA,MACpB,QAAA,EAAU,KAAA;AAAA,MACV,SAAA,EAAW;AAAA,QACP;AAAA,UACI,QAAA,EAAU,GAAA;AAAA,UACV,OAAA,EAAS;AAAA,YACL,UAAA,EAAY,IAAA;AAAA,YACZ,wBAAA,EAA0B;AAAA;AAC9B,SACJ;AAAA,QACA,GAAG,cAAA,CAAe,GAAA,CAAI,CAAC,GAAA,MAAS,EAAE,QAAA,EAAU,GAAA,EAAK,MAAA,EAAQ,MAAA,EAAO,CAAE;AAAA;AACtE,KACH,CAAA;AACD,IAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,MAAA,EAAQ,IAAI,CAAA;AAGjC,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,EAAA,EAAI,CAAA,EAAA,EAAK;AACzB,MAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,QAAA,EAAU,MAAM,CAAA;AACrC,MAAA,GAAA,GAAM,GAAA,CAAI,UAAA,CAAW,IAAA,EAAM,GAAG,CAAA;AAAA,IAClC;AACA,IAAA,OAAO,GAAA;AAAA,EACX,SAAS,KAAA,EAAO;AACZ,IAAA,IAAI,iBAAiB,KAAA,EAAO;AACxB,MAAA,MAAM,IAAI,eAAA,CAAgB,CAAA,gCAAA,EAAmC,KAAA,CAAM,OAAO,CAAA,CAAA,EAAI;AAAA,QAC1E,KAAA,EAAO;AAAA,OACV,CAAA;AAAA,IACL;AACA,IAAA,MAAM,IAAI,gBAAgB,gCAAgC,CAAA;AAAA,EAC9D;AACJ;AAOA,eAAsB,eAAe,GAAA,EAA8B;AAC/D,EAAA,MAAM,IAAA,GAAO,MAAM,cAAA,CAAe,GAAG,CAAA;AACrC,EAAA,OAAO,WAAW,IAAI,CAAA;AAC1B","file":"index.js","sourcesContent":["/**\n * Base error class for every error thrown by the mapthis package.\n *\n * All feature-specific error classes (scrape, search, ai, geocoding) extend\n * this, so consumers can do `catch (e) { if (e instanceof MapthisError) ... }`\n * to distinguish library errors from other exceptions.\n */\nexport class MapthisError extends Error {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"MapthisError\"\n    }\n}\n","import { MapthisError } from \"../types/errors\"\n\nexport class ScrapeError extends MapthisError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"ScrapeError\"\n    }\n}\n\n/**\n * Thrown when a provided URL fails validation or cannot be parsed by `fetch`.\n */\nexport class InvalidUrlError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"InvalidUrlError\"\n    }\n}\n\n/**\n * Thrown when a remote server returns 401 or 403 for an HTML fetch, meaning\n * the page does not permit anonymous scraping.\n */\nexport class HtmlUnauthorizedError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"HtmlUnauthorizedError\"\n    }\n}\n\n/**\n * Thrown when the underlying `html-to-text` conversion fails.\n */\nexport class HtmlToTextError extends ScrapeError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"HtmlToTextError\"\n    }\n}\n","import { z } from \"zod\"\nimport { HtmlUnauthorizedError, InvalidUrlError, ScrapeError } from \"./errors\"\n\nconst withHttps = (url: string): string => {\n    const hasHttp = /^http?:\\/\\//i.test(url)\n    const hasHttps = /^https?:\\/\\//i.test(url)\n    if (!hasHttp && !hasHttps) return `https://${url}`\n    return url\n}\n\n/**\n * Fetch the raw HTML of a URL, performing light validation first.\n *\n * - URLs without a scheme are prefixed with `https://`.\n * - URL shape is validated via Zod before the network call.\n * - 401/403 responses throw {@link HtmlUnauthorizedError}.\n * - Other non-OK responses throw {@link ScrapeError} with the status text.\n * - `TypeError` from `fetch` (typically malformed URL) is rethrown as\n *   {@link InvalidUrlError}.\n *\n * This function does not set any special headers, user agent, or cookies.\n * Sites with aggressive bot-detection may return empty bodies or errors; the\n * caller is responsible for retry / user-agent rotation strategies if needed.\n */\nexport async function getHtmlFromUrl(url: string): Promise<string> {\n    const urlFixed = withHttps(url.toLowerCase())\n    const isUrl = z.string().url().safeParse(urlFixed)\n    if (!isUrl.success) throw new InvalidUrlError(`Invalid URL: ${url}`)\n\n    try {\n        const response = await fetch(urlFixed)\n        const html = await response.text()\n        if (response.status === 403 || response.status === 401) {\n            throw new HtmlUnauthorizedError(\n                `${response.status} Not authorized to scrape this website`,\n            )\n        }\n        if (!response.ok) {\n            throw new ScrapeError(\n                `Bad response when getting HTML. Status: ${response.status} ${response.statusText}`,\n            )\n        }\n        return html\n    } catch (error) {\n        if (error instanceof TypeError) {\n            throw new InvalidUrlError(`Invalid URL. Original error: ${error.message}`)\n        }\n        throw error\n    }\n}\n","import { convert } from \"html-to-text\"\nimport { HtmlToTextError } from \"./errors\"\nimport { getHtmlFromUrl } from \"./html\"\n\nconst SKIP_SELECTORS = [\n    \"img\",\n    \"header\",\n    \"footer\",\n    \"audio\",\n    \"button\",\n    \"canvas\",\n    \"code\",\n    \"nav\",\n    \"#nav\",\n    \"figure\",\n    \"figcaption\",\n    \".comment\",\n    \".comments\",\n    \"#comments\",\n    \"#related-posts\",\n    \"#related\",\n    \".related\",\n    \".related-posts\",\n] as const\n\n/**\n * Convert raw HTML to plain text suitable for LLM consumption.\n *\n * The converter is configured to:\n * - strip anchor hrefs when they duplicate the link text;\n * - skip navigation, footer, comments, related-post widgets, images, and\n *   other non-content elements;\n * - collapse repeated blank lines and double spaces so token count doesn't\n *   balloon from whitespace.\n */\nexport function htmlToText(html: string): string {\n    try {\n        let out = convert(html, {\n            wordwrap: false,\n            selectors: [\n                {\n                    selector: \"a\",\n                    options: {\n                        ignoreHref: true,\n                        hideLinkHrefIfSameAsText: true,\n                    },\n                },\n                ...SKIP_SELECTORS.map((tag) => ({ selector: tag, format: \"skip\" })),\n            ],\n        })\n        out = out.replaceAll(\"\\r\\n\", \"\\n\")\n        // Iteratively collapse blank lines and double spaces. 20 passes is\n        // overkill for any realistic input but cheap and bounds worst case.\n        for (let i = 0; i < 20; i++) {\n            out = out.replaceAll(\"\\n\\n\\n\", \"\\n\\n\")\n            out = out.replaceAll(\"  \", \" \")\n        }\n        return out\n    } catch (error) {\n        if (error instanceof Error) {\n            throw new HtmlToTextError(`HTML to text conversion failed: ${error.message}`, {\n                cause: error,\n            })\n        }\n        throw new HtmlToTextError(\"HTML to text conversion failed\")\n    }\n}\n\n/**\n * Convenience: fetch a URL and convert its HTML to plain text in one call.\n *\n * Equivalent to `htmlToText(await getHtmlFromUrl(url))`.\n */\nexport async function getTextFromUrl(url: string): Promise<string> {\n    const html = await getHtmlFromUrl(url)\n    return htmlToText(html)\n}\n"]}

package/dist/search/index.cjs

@@ -0,0 +1,76 @@
+'use strict';
+
+// src/types/errors.ts
+var MapthisError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MapthisError";
+  }
+};
+
+// src/search/errors.ts
+var SearchError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "SearchError";
+  }
+};
+var NoSearchResultsError = class extends SearchError {
+  constructor(message, options) {
+    super(message ?? "No search results found", options);
+    this.name = "NoSearchResultsError";
+  }
+};
+
+// src/search/blacklist.ts
+var DEFAULT_SEARCH_BLACKLIST = [
+  "reddit.com",
+  "quora.com",
+  "tripadvisor.com",
+  "youtube.com"
+];
+
+// src/search/client.ts
+function createSearchClient(config) {
+  const blacklist = config.blacklist ?? DEFAULT_SEARCH_BLACKLIST;
+  const exclusions = blacklist.map((site) => `-site:${site}`).join(" ");
+  return {
+    async getUrlsFromQuery(req) {
+      const { query } = req;
+      const q = encodeURIComponent(`${query} ${exclusions}`.trim());
+      const url = `https://www.googleapis.com/customsearch/v1?key=${config.apiKey}&cx=${config.engineId}&q=${q}`;
+      let response;
+      try {
+        response = await fetch(url);
+      } catch (error) {
+        throw new SearchError(
+          `Search request failed: ${error instanceof Error ? error.message : "unknown"}`,
+          { cause: error instanceof Error ? error : void 0 }
+        );
+      }
+      if (!response.ok) {
+        throw new SearchError(
+          `Search API returned ${response.status} ${response.statusText}`
+        );
+      }
+      const search = await response.json();
+      if (!search?.items?.length) throw new NoSearchResultsError();
+      return {
+        query,
+        items: search.items.map((item) => ({
+          url: item.link,
+          title: item.title,
+          snippet: item.snippet,
+          image: item.pagemap?.cse_image?.[0]?.src
+        }))
+      };
+    }
+  };
+}
+
+exports.DEFAULT_SEARCH_BLACKLIST = DEFAULT_SEARCH_BLACKLIST;
+exports.NoSearchResultsError = NoSearchResultsError;
+exports.SearchError = SearchError;
+exports.createSearchClient = createSearchClient;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/search/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/types/errors.ts","../../src/search/errors.ts","../../src/search/blacklist.ts","../../src/search/client.ts"],"names":[],"mappings":";;;AAOO,IAAM,YAAA,GAAN,cAA2B,KAAA,CAAM;AAAA,EACpC,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,cAAA;AAAA,EAChB;AACJ,CAAA;;;ACVO,IAAM,WAAA,GAAN,cAA0B,YAAA,CAAa;AAAA,EAC1C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,aAAA;AAAA,EAChB;AACJ;AAKO,IAAM,oBAAA,GAAN,cAAmC,WAAA,CAAY;AAAA,EAClD,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,OAAA,IAAW,2BAA2B,OAAO,CAAA;AACnD,IAAA,IAAA,CAAK,IAAA,GAAO,sBAAA;AAAA,EAChB;AACJ;;;ACTO,IAAM,wBAAA,GAA8C;AAAA,EACvD,YAAA;AAAA,EACA,WAAA;AAAA,EACA,iBAAA;AAAA,EACA;AACJ;;;ACyBO,SAAS,mBAAmB,MAAA,EAA0C;AACzE,EAAA,MAAM,SAAA,GAAY,OAAO,SAAA,IAAa,wBAAA;AACtC,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,GAAA,CAAI,CAAC,IAAA,KAAS,SAAS,IAAI,CAAA,CAAE,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA;AAEpE,EAAA,OAAO;AAAA,IACH,MAAM,iBAAiB,GAAA,EAA6C;AAChE,MAAA,MAAM,EAAE,OAAM,GAAI,GAAA;AAClB,MAAA,MAAM,CAAA,GAAI,mBAAmB,CAAA,EAAG,KAAK,IAAI,UAAU,CAAA,CAAA,CAAG,MAAM,CAAA;AAC5D,MAAA,MAAM,GAAA,GAAM,kDAAkD,MAAA,CAAO,MAAM,OAAO,MAAA,CAAO,QAAQ,MAAM,CAAC,CAAA,CAAA;AAExG,MAAA,IAAI,QAAA;AACJ,MAAA,IAAI;AACA,QAAA,QAAA,GAAW,MAAM,MAAM,GAAG,CAAA;AAAA,MAC9B,SAAS,KAAA,EAAO;AACZ,QAAA,MAAM,IAAI,WAAA;AAAA,UACN,CAAA,uBAAA,EAA0B,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,UAAU,SAAS,CAAA,CAAA;AAAA,UAC5E,EAAE,KAAA,EAAO,KAAA,YAAiB,KAAA,GAAQ,QAAQ,MAAA;AAAU,SACxD;AAAA,MACJ;AAEA,MAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AACd,QAAA,MAAM,IAAI,WAAA;AAAA,UACN,CAAA,oBAAA,EAAuB,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,SACjE;AAAA,MACJ;AAEA,MAAA,MAAM,MAAA,GAAU,MAAM,QAAA,CAAS,IAAA,EAAK;AACpC,MAAA,IAAI,CAAC,MAAA,EAAQ,KAAA,EAAO,MAAA,EAAQ,MAAM,IAAI,oBAAA,EAAqB;AAE3D,MAAA,OAAO;AAAA,QACH,KAAA;AAAA,QACA,KAAA,EAAO,MAAA,CAAO,KAAA,CAAM,GAAA,CAAI,CAAC,IAAA,MAAU;AAAA,UAC/B,KAAK,IAAA,CAAK,IAAA;AAAA,UACV,OAAO,IAAA,CAAK,KAAA;AAAA,UACZ,SAAS,IAAA,CAAK,OAAA;AAAA,UACd,KAAA,EAAO,IAAA,CAAK,OAAA,EAAS,SAAA,GAAY,CAAC,CAAA,EAAG;AAAA,SACzC,CAAE;AAAA,OACN;AAAA,IACJ;AAAA,GACJ;AACJ","file":"index.cjs","sourcesContent":["/**\n * Base error class for every error thrown by the mapthis package.\n *\n * All feature-specific error classes (scrape, search, ai, geocoding) extend\n * this, so consumers can do `catch (e) { if (e instanceof MapthisError) ... }`\n * to distinguish library errors from other exceptions.\n */\nexport class MapthisError extends Error {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"MapthisError\"\n    }\n}\n","import { MapthisError } from \"../types/errors\"\n\nexport class SearchError extends MapthisError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"SearchError\"\n    }\n}\n\n/**\n * Thrown when a search query returned no items.\n */\nexport class NoSearchResultsError extends SearchError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message ?? \"No search results found\", options)\n        this.name = \"NoSearchResultsError\"\n    }\n}\n","/**\n * Sites that do not allow anonymous scraping (by robots policy or aggressive\n * bot-detection) and are excluded from search results by default.\n *\n * Consumers can override this by passing their own `blacklist` option when\n * creating a search client, or extend it by spreading\n * `DEFAULT_SEARCH_BLACKLIST` into a custom list.\n */\nexport const DEFAULT_SEARCH_BLACKLIST: readonly string[] = [\n    \"reddit.com\",\n    \"quora.com\",\n    \"tripadvisor.com\",\n    \"youtube.com\",\n]\n","import { DEFAULT_SEARCH_BLACKLIST } from \"./blacklist\"\nimport type { CustomSearchResponse } from \"./cse-types\"\nimport { NoSearchResultsError, SearchError } from \"./errors\"\nimport type { SearchRequest, SearchResponse } from \"./types\"\n\nexport interface SearchClientConfig {\n    /** Google API key with Custom Search API enabled. */\n    apiKey: string\n    /** Google Custom Search Engine ID (the `cx` parameter). */\n    engineId: string\n    /**\n     * Domains to exclude from results (added as `-site:` operators to the\n     * query). Defaults to {@link DEFAULT_SEARCH_BLACKLIST}.\n     */\n    blacklist?: readonly string[]\n}\n\nexport interface SearchClient {\n    /**\n     * Execute a search and return the top results, filtered by the client's\n     * blacklist. Throws {@link NoSearchResultsError} if the response is empty\n     * and {@link SearchError} on network / API failures.\n     */\n    getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse>\n}\n\n/**\n * Create a Google Custom Search client.\n *\n * @example\n * ```ts\n * const search = createSearchClient({\n *   apiKey: process.env.GOOGLE_CUSTOM_SEARCH_KEY!,\n *   engineId: process.env.GOOGLE_CUSTOM_SEARCH_ENGINE!,\n * })\n * const { items } = await search.getUrlsFromQuery({ query: \"best ramen in tokyo\" })\n * ```\n */\nexport function createSearchClient(config: SearchClientConfig): SearchClient {\n    const blacklist = config.blacklist ?? DEFAULT_SEARCH_BLACKLIST\n    const exclusions = blacklist.map((site) => `-site:${site}`).join(\" \")\n\n    return {\n        async getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse> {\n            const { query } = req\n            const q = encodeURIComponent(`${query} ${exclusions}`.trim())\n            const url = `https://www.googleapis.com/customsearch/v1?key=${config.apiKey}&cx=${config.engineId}&q=${q}`\n\n            let response: Response\n            try {\n                response = await fetch(url)\n            } catch (error) {\n                throw new SearchError(\n                    `Search request failed: ${error instanceof Error ? error.message : \"unknown\"}`,\n                    { cause: error instanceof Error ? error : undefined },\n                )\n            }\n\n            if (!response.ok) {\n                throw new SearchError(\n                    `Search API returned ${response.status} ${response.statusText}`,\n                )\n            }\n\n            const search = (await response.json()) as CustomSearchResponse\n            if (!search?.items?.length) throw new NoSearchResultsError()\n\n            return {\n                query,\n                items: search.items.map((item) => ({\n                    url: item.link,\n                    title: item.title,\n                    snippet: item.snippet,\n                    image: item.pagemap?.cse_image?.[0]?.src,\n                })),\n            }\n        },\n    }\n}\n"]}

package/dist/search/index.d.cts

@@ -0,0 +1,75 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.cjs';
+
+declare class SearchError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a search query returned no items.
+ */
+declare class NoSearchResultsError extends SearchError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+interface SearchRequest {
+    query: string;
+    /**
+     * Maximum number of items to return. The underlying Google CSE cap is 10
+     * per call; consumers wanting more must paginate themselves.
+     */
+    num?: number;
+}
+interface SearchResultItem {
+    url: string;
+    title: string;
+    snippet: string;
+    image?: string;
+}
+interface SearchResponse {
+    query: string;
+    items: SearchResultItem[];
+}
+
+/**
+ * Sites that do not allow anonymous scraping (by robots policy or aggressive
+ * bot-detection) and are excluded from search results by default.
+ *
+ * Consumers can override this by passing their own `blacklist` option when
+ * creating a search client, or extend it by spreading
+ * `DEFAULT_SEARCH_BLACKLIST` into a custom list.
+ */
+declare const DEFAULT_SEARCH_BLACKLIST: readonly string[];
+
+interface SearchClientConfig {
+    /** Google API key with Custom Search API enabled. */
+    apiKey: string;
+    /** Google Custom Search Engine ID (the `cx` parameter). */
+    engineId: string;
+    /**
+     * Domains to exclude from results (added as `-site:` operators to the
+     * query). Defaults to {@link DEFAULT_SEARCH_BLACKLIST}.
+     */
+    blacklist?: readonly string[];
+}
+interface SearchClient {
+    /**
+     * Execute a search and return the top results, filtered by the client's
+     * blacklist. Throws {@link NoSearchResultsError} if the response is empty
+     * and {@link SearchError} on network / API failures.
+     */
+    getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse>;
+}
+/**
+ * Create a Google Custom Search client.
+ *
+ * @example
+ * ```ts
+ * const search = createSearchClient({
+ *   apiKey: process.env.GOOGLE_CUSTOM_SEARCH_KEY!,
+ *   engineId: process.env.GOOGLE_CUSTOM_SEARCH_ENGINE!,
+ * })
+ * const { items } = await search.getUrlsFromQuery({ query: "best ramen in tokyo" })
+ * ```
+ */
+declare function createSearchClient(config: SearchClientConfig): SearchClient;
+
+export { DEFAULT_SEARCH_BLACKLIST, NoSearchResultsError, type SearchClient, type SearchClientConfig, SearchError, type SearchRequest, type SearchResponse, type SearchResultItem, createSearchClient };

package/dist/search/index.d.ts

@@ -0,0 +1,75 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.js';
+
+declare class SearchError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a search query returned no items.
+ */
+declare class NoSearchResultsError extends SearchError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+interface SearchRequest {
+    query: string;
+    /**
+     * Maximum number of items to return. The underlying Google CSE cap is 10
+     * per call; consumers wanting more must paginate themselves.
+     */
+    num?: number;
+}
+interface SearchResultItem {
+    url: string;
+    title: string;
+    snippet: string;
+    image?: string;
+}
+interface SearchResponse {
+    query: string;
+    items: SearchResultItem[];
+}
+
+/**
+ * Sites that do not allow anonymous scraping (by robots policy or aggressive
+ * bot-detection) and are excluded from search results by default.
+ *
+ * Consumers can override this by passing their own `blacklist` option when
+ * creating a search client, or extend it by spreading
+ * `DEFAULT_SEARCH_BLACKLIST` into a custom list.
+ */
+declare const DEFAULT_SEARCH_BLACKLIST: readonly string[];
+
+interface SearchClientConfig {
+    /** Google API key with Custom Search API enabled. */
+    apiKey: string;
+    /** Google Custom Search Engine ID (the `cx` parameter). */
+    engineId: string;
+    /**
+     * Domains to exclude from results (added as `-site:` operators to the
+     * query). Defaults to {@link DEFAULT_SEARCH_BLACKLIST}.
+     */
+    blacklist?: readonly string[];
+}
+interface SearchClient {
+    /**
+     * Execute a search and return the top results, filtered by the client's
+     * blacklist. Throws {@link NoSearchResultsError} if the response is empty
+     * and {@link SearchError} on network / API failures.
+     */
+    getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse>;
+}
+/**
+ * Create a Google Custom Search client.
+ *
+ * @example
+ * ```ts
+ * const search = createSearchClient({
+ *   apiKey: process.env.GOOGLE_CUSTOM_SEARCH_KEY!,
+ *   engineId: process.env.GOOGLE_CUSTOM_SEARCH_ENGINE!,
+ * })
+ * const { items } = await search.getUrlsFromQuery({ query: "best ramen in tokyo" })
+ * ```
+ */
+declare function createSearchClient(config: SearchClientConfig): SearchClient;
+
+export { DEFAULT_SEARCH_BLACKLIST, NoSearchResultsError, type SearchClient, type SearchClientConfig, SearchError, type SearchRequest, type SearchResponse, type SearchResultItem, createSearchClient };

package/dist/search/index.js

@@ -0,0 +1,71 @@
+// src/types/errors.ts
+var MapthisError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MapthisError";
+  }
+};
+
+// src/search/errors.ts
+var SearchError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "SearchError";
+  }
+};
+var NoSearchResultsError = class extends SearchError {
+  constructor(message, options) {
+    super(message ?? "No search results found", options);
+    this.name = "NoSearchResultsError";
+  }
+};
+
+// src/search/blacklist.ts
+var DEFAULT_SEARCH_BLACKLIST = [
+  "reddit.com",
+  "quora.com",
+  "tripadvisor.com",
+  "youtube.com"
+];
+
+// src/search/client.ts
+function createSearchClient(config) {
+  const blacklist = config.blacklist ?? DEFAULT_SEARCH_BLACKLIST;
+  const exclusions = blacklist.map((site) => `-site:${site}`).join(" ");
+  return {
+    async getUrlsFromQuery(req) {
+      const { query } = req;
+      const q = encodeURIComponent(`${query} ${exclusions}`.trim());
+      const url = `https://www.googleapis.com/customsearch/v1?key=${config.apiKey}&cx=${config.engineId}&q=${q}`;
+      let response;
+      try {
+        response = await fetch(url);
+      } catch (error) {
+        throw new SearchError(
+          `Search request failed: ${error instanceof Error ? error.message : "unknown"}`,
+          { cause: error instanceof Error ? error : void 0 }
+        );
+      }
+      if (!response.ok) {
+        throw new SearchError(
+          `Search API returned ${response.status} ${response.statusText}`
+        );
+      }
+      const search = await response.json();
+      if (!search?.items?.length) throw new NoSearchResultsError();
+      return {
+        query,
+        items: search.items.map((item) => ({
+          url: item.link,
+          title: item.title,
+          snippet: item.snippet,
+          image: item.pagemap?.cse_image?.[0]?.src
+        }))
+      };
+    }
+  };
+}
+
+export { DEFAULT_SEARCH_BLACKLIST, NoSearchResultsError, SearchError, createSearchClient };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/search/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/types/errors.ts","../../src/search/errors.ts","../../src/search/blacklist.ts","../../src/search/client.ts"],"names":[],"mappings":";AAOO,IAAM,YAAA,GAAN,cAA2B,KAAA,CAAM;AAAA,EACpC,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,cAAA;AAAA,EAChB;AACJ,CAAA;;;ACVO,IAAM,WAAA,GAAN,cAA0B,YAAA,CAAa;AAAA,EAC1C,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,aAAA;AAAA,EAChB;AACJ;AAKO,IAAM,oBAAA,GAAN,cAAmC,WAAA,CAAY;AAAA,EAClD,WAAA,CAAY,SAAkB,OAAA,EAAwB;AAClD,IAAA,KAAA,CAAM,OAAA,IAAW,2BAA2B,OAAO,CAAA;AACnD,IAAA,IAAA,CAAK,IAAA,GAAO,sBAAA;AAAA,EAChB;AACJ;;;ACTO,IAAM,wBAAA,GAA8C;AAAA,EACvD,YAAA;AAAA,EACA,WAAA;AAAA,EACA,iBAAA;AAAA,EACA;AACJ;;;ACyBO,SAAS,mBAAmB,MAAA,EAA0C;AACzE,EAAA,MAAM,SAAA,GAAY,OAAO,SAAA,IAAa,wBAAA;AACtC,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,GAAA,CAAI,CAAC,IAAA,KAAS,SAAS,IAAI,CAAA,CAAE,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA;AAEpE,EAAA,OAAO;AAAA,IACH,MAAM,iBAAiB,GAAA,EAA6C;AAChE,MAAA,MAAM,EAAE,OAAM,GAAI,GAAA;AAClB,MAAA,MAAM,CAAA,GAAI,mBAAmB,CAAA,EAAG,KAAK,IAAI,UAAU,CAAA,CAAA,CAAG,MAAM,CAAA;AAC5D,MAAA,MAAM,GAAA,GAAM,kDAAkD,MAAA,CAAO,MAAM,OAAO,MAAA,CAAO,QAAQ,MAAM,CAAC,CAAA,CAAA;AAExG,MAAA,IAAI,QAAA;AACJ,MAAA,IAAI;AACA,QAAA,QAAA,GAAW,MAAM,MAAM,GAAG,CAAA;AAAA,MAC9B,SAAS,KAAA,EAAO;AACZ,QAAA,MAAM,IAAI,WAAA;AAAA,UACN,CAAA,uBAAA,EAA0B,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,UAAU,SAAS,CAAA,CAAA;AAAA,UAC5E,EAAE,KAAA,EAAO,KAAA,YAAiB,KAAA,GAAQ,QAAQ,MAAA;AAAU,SACxD;AAAA,MACJ;AAEA,MAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AACd,QAAA,MAAM,IAAI,WAAA;AAAA,UACN,CAAA,oBAAA,EAAuB,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,SACjE;AAAA,MACJ;AAEA,MAAA,MAAM,MAAA,GAAU,MAAM,QAAA,CAAS,IAAA,EAAK;AACpC,MAAA,IAAI,CAAC,MAAA,EAAQ,KAAA,EAAO,MAAA,EAAQ,MAAM,IAAI,oBAAA,EAAqB;AAE3D,MAAA,OAAO;AAAA,QACH,KAAA;AAAA,QACA,KAAA,EAAO,MAAA,CAAO,KAAA,CAAM,GAAA,CAAI,CAAC,IAAA,MAAU;AAAA,UAC/B,KAAK,IAAA,CAAK,IAAA;AAAA,UACV,OAAO,IAAA,CAAK,KAAA;AAAA,UACZ,SAAS,IAAA,CAAK,OAAA;AAAA,UACd,KAAA,EAAO,IAAA,CAAK,OAAA,EAAS,SAAA,GAAY,CAAC,CAAA,EAAG;AAAA,SACzC,CAAE;AAAA,OACN;AAAA,IACJ;AAAA,GACJ;AACJ","file":"index.js","sourcesContent":["/**\n * Base error class for every error thrown by the mapthis package.\n *\n * All feature-specific error classes (scrape, search, ai, geocoding) extend\n * this, so consumers can do `catch (e) { if (e instanceof MapthisError) ... }`\n * to distinguish library errors from other exceptions.\n */\nexport class MapthisError extends Error {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"MapthisError\"\n    }\n}\n","import { MapthisError } from \"../types/errors\"\n\nexport class SearchError extends MapthisError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message, options)\n        this.name = \"SearchError\"\n    }\n}\n\n/**\n * Thrown when a search query returned no items.\n */\nexport class NoSearchResultsError extends SearchError {\n    constructor(message?: string, options?: ErrorOptions) {\n        super(message ?? \"No search results found\", options)\n        this.name = \"NoSearchResultsError\"\n    }\n}\n","/**\n * Sites that do not allow anonymous scraping (by robots policy or aggressive\n * bot-detection) and are excluded from search results by default.\n *\n * Consumers can override this by passing their own `blacklist` option when\n * creating a search client, or extend it by spreading\n * `DEFAULT_SEARCH_BLACKLIST` into a custom list.\n */\nexport const DEFAULT_SEARCH_BLACKLIST: readonly string[] = [\n    \"reddit.com\",\n    \"quora.com\",\n    \"tripadvisor.com\",\n    \"youtube.com\",\n]\n","import { DEFAULT_SEARCH_BLACKLIST } from \"./blacklist\"\nimport type { CustomSearchResponse } from \"./cse-types\"\nimport { NoSearchResultsError, SearchError } from \"./errors\"\nimport type { SearchRequest, SearchResponse } from \"./types\"\n\nexport interface SearchClientConfig {\n    /** Google API key with Custom Search API enabled. */\n    apiKey: string\n    /** Google Custom Search Engine ID (the `cx` parameter). */\n    engineId: string\n    /**\n     * Domains to exclude from results (added as `-site:` operators to the\n     * query). Defaults to {@link DEFAULT_SEARCH_BLACKLIST}.\n     */\n    blacklist?: readonly string[]\n}\n\nexport interface SearchClient {\n    /**\n     * Execute a search and return the top results, filtered by the client's\n     * blacklist. Throws {@link NoSearchResultsError} if the response is empty\n     * and {@link SearchError} on network / API failures.\n     */\n    getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse>\n}\n\n/**\n * Create a Google Custom Search client.\n *\n * @example\n * ```ts\n * const search = createSearchClient({\n *   apiKey: process.env.GOOGLE_CUSTOM_SEARCH_KEY!,\n *   engineId: process.env.GOOGLE_CUSTOM_SEARCH_ENGINE!,\n * })\n * const { items } = await search.getUrlsFromQuery({ query: \"best ramen in tokyo\" })\n * ```\n */\nexport function createSearchClient(config: SearchClientConfig): SearchClient {\n    const blacklist = config.blacklist ?? DEFAULT_SEARCH_BLACKLIST\n    const exclusions = blacklist.map((site) => `-site:${site}`).join(\" \")\n\n    return {\n        async getUrlsFromQuery(req: SearchRequest): Promise<SearchResponse> {\n            const { query } = req\n            const q = encodeURIComponent(`${query} ${exclusions}`.trim())\n            const url = `https://www.googleapis.com/customsearch/v1?key=${config.apiKey}&cx=${config.engineId}&q=${q}`\n\n            let response: Response\n            try {\n                response = await fetch(url)\n            } catch (error) {\n                throw new SearchError(\n                    `Search request failed: ${error instanceof Error ? error.message : \"unknown\"}`,\n                    { cause: error instanceof Error ? error : undefined },\n                )\n            }\n\n            if (!response.ok) {\n                throw new SearchError(\n                    `Search API returned ${response.status} ${response.statusText}`,\n                )\n            }\n\n            const search = (await response.json()) as CustomSearchResponse\n            if (!search?.items?.length) throw new NoSearchResultsError()\n\n            return {\n                query,\n                items: search.items.map((item) => ({\n                    url: item.link,\n                    title: item.title,\n                    snippet: item.snippet,\n                    image: item.pagemap?.cse_image?.[0]?.src,\n                })),\n            }\n        },\n    }\n}\n"]}