@dromney/mapthis 0.1.0

package/dist/ai/index.d.cts

@@ -0,0 +1,117 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.cjs';
+import { O as OpenAiBackend, S as SummarizedText, G as GetLocationsResponse } from '../parser-CzXzpmVv.cjs';
+export { D as DEFAULT_MODEL, a as DEFAULT_TOKEN_LIMIT, b as GenericAIJsonRequest, L as LocationParser, c as LocationParserConfig, d as OpenAIOutput, e as OpenAiBackendConfig, f as createLocationParser, g as createOpenAiBackend } from '../parser-CzXzpmVv.cjs';
+import { JSONSchema7Object } from 'json-schema';
+import '../domain-Dc1wSTkf.cjs';
+import '../schemas-Dy5coqXo.cjs';
+import 'zod';
+
+declare class AiError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM returned zero locations after deduplication.
+ */
+declare class NoLocationsFoundError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Summarization pipeline failed (one of the chunk calls rejected).
+ */
+declare class SummarizeTextError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * A JSON schema passed to the LLM was rejected as invalid by the provider.
+ */
+declare class InvalidJsonSchemaError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM response could not be parsed as JSON (typically because the output
+ * was truncated or malformed).
+ */
+declare class AiResponseJsonError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Input text exceeded the model's context window.
+ */
+declare class AiInputLengthError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM hit the output token limit before completing its JSON response.
+ */
+declare class AiOutputLengthError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+/**
+ * Semantic version of the location-extraction system prompt. Stored alongside
+ * each place record in the private app so results can be re-run when the
+ * prompt is updated. Bump when you change {@link LOCATIONS_SYSTEM_MESSAGE} or
+ * {@link LOCATIONS_SCHEMA} in a way that meaningfully changes output.
+ */
+declare const LOCATIONS_PROMPT_VERSION = "0.0.1";
+declare const LOCATIONS_FUNCTIONS_NAME = "get_places";
+declare const LOCATIONS_SYSTEM_MESSAGE = "You parse a list of place addresses, corresponding brief one-sentence descriptions, and a descriptive title (preferably directly quoted) from text. These addresses might be countries, cities, attractions, parks, bars, etc.\nIf the text seems to be listing places that are all within a single parent place, you don't return the parent place, but you do add it to all the childrens' addresses.\nFor example, for an article about places in greece, you would not return \"greece\", but if the Acropolis was mentioned, you would return \"Acropolis, Athens, Greece\"\n\nYou ignore extraneous text, such as recommended articles, advertisments, etc.\n";
+declare const LOCATIONS_SCHEMA: JSONSchema7Object;
+declare const LOCATIONS_MAX_OUTPUT_TOKENS = 500;
+/**
+ * Semantic version of the summarization prompt. Bump when meaningful changes
+ * are made to {@link SUMMARIZATION_PROMPT}.
+ */
+declare const SUMMARIZE_PROMPT_VERSION = "0.0.1";
+declare const SUMMARIZATION_PROMPT = "Summarize the given text. The summary must reduce the number of characters from the original text by at least a factor of {RATIO}.\nFor example, text of 1000 characters must output a summary less than {EXAMPLE} characters.\nThis summary will be used to extract a list of locations and addresses, so keep any text that seems to be a list of locations/places, a location and paragraph describing it, or otherwise clearly a location name or address.\nHere is the text:\n";
+
+/**
+ * Reduce a large text to a summary that will fit inside the model's context
+ * window alongside a known downstream prompt (e.g. the location-extraction
+ * call).
+ *
+ * Algorithm overview
+ * ------------------
+ * We need to fit both the summary and a downstream "final" prompt into the
+ * model's context window. Given:
+ *
+ * - `token_limit` — model context window (with a safety factor)
+ * - `text_tokens` — size of the input text
+ * - `summarize_prompt_tokens` — size of the summarization prompt itself
+ * - `final_prompt_tokens` — size of the downstream prompt
+ * - `final_output_tokens` — expected size of the downstream response
+ *
+ * We split the input into chunks, summarize each in parallel, and concatenate
+ * the results. The per-chunk size is chosen so that the combined summary
+ * leaves room for the downstream prompt:
+ *
+ *     final_input_tokens    = token_limit - final_output_tokens
+ *     summary_output_tokens = final_input_tokens - final_prompt_tokens
+ *     num_chunks            = ceil((text_tokens + summary_output_tokens)
+ *                                  / (token_limit - summarize_prompt_tokens))
+ *     chunk_output_tokens   = summary_output_tokens / num_chunks
+ *     chunk_input_tokens    = token_limit - summarize_prompt_tokens - chunk_output_tokens
+ *
+ * A safety factor is applied to chunk output sizes because LLMs routinely
+ * overshoot explicit length instructions.
+ */
+declare function summarizeText(backend: OpenAiBackend, text: string, finalPrompt?: string, finalOutputTokens?: number, chunkOutputSafetyFactor?: number, tokenLimitSafetyFactor?: number): Promise<SummarizedText>;
+
+/**
+ * Parse locations from freeform text. Two-stage: summarize to fit the context
+ * window, then a function-calling JSON completion for extraction.
+ */
+declare function parseLocationsFromText(backend: OpenAiBackend, text: string): Promise<GetLocationsResponse>;
+/**
+ * Parse locations from HTML. Equivalent to `htmlToText` followed by
+ * {@link parseLocationsFromText}.
+ */
+declare function parseLocationsFromHtml(backend: OpenAiBackend, html: string): Promise<GetLocationsResponse>;
+/**
+ * Parse locations from a URL. Equivalent to fetching the HTML, converting to
+ * text, and calling {@link parseLocationsFromText}. Throws any scrape errors
+ * unchanged.
+ */
+declare function parseLocationsFromUrl(backend: OpenAiBackend, url: string): Promise<GetLocationsResponse>;
+
+export { AiError, AiInputLengthError, AiOutputLengthError, AiResponseJsonError, GetLocationsResponse, InvalidJsonSchemaError, LOCATIONS_FUNCTIONS_NAME, LOCATIONS_MAX_OUTPUT_TOKENS, LOCATIONS_PROMPT_VERSION, LOCATIONS_SCHEMA, LOCATIONS_SYSTEM_MESSAGE, NoLocationsFoundError, OpenAiBackend, SUMMARIZATION_PROMPT, SUMMARIZE_PROMPT_VERSION, SummarizeTextError, SummarizedText, parseLocationsFromHtml, parseLocationsFromText, parseLocationsFromUrl, summarizeText };

package/dist/ai/index.d.ts

@@ -0,0 +1,117 @@
+import { M as MapthisError } from '../errors-Bw97z_4m.js';
+import { O as OpenAiBackend, S as SummarizedText, G as GetLocationsResponse } from '../parser-N7-fNxeu.js';
+export { D as DEFAULT_MODEL, a as DEFAULT_TOKEN_LIMIT, b as GenericAIJsonRequest, L as LocationParser, c as LocationParserConfig, d as OpenAIOutput, e as OpenAiBackendConfig, f as createLocationParser, g as createOpenAiBackend } from '../parser-N7-fNxeu.js';
+import { JSONSchema7Object } from 'json-schema';
+import '../domain-CZ-L-ntu.js';
+import '../schemas-Dy5coqXo.js';
+import 'zod';
+
+declare class AiError extends MapthisError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM returned zero locations after deduplication.
+ */
+declare class NoLocationsFoundError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Summarization pipeline failed (one of the chunk calls rejected).
+ */
+declare class SummarizeTextError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * A JSON schema passed to the LLM was rejected as invalid by the provider.
+ */
+declare class InvalidJsonSchemaError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM response could not be parsed as JSON (typically because the output
+ * was truncated or malformed).
+ */
+declare class AiResponseJsonError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * Input text exceeded the model's context window.
+ */
+declare class AiInputLengthError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+/**
+ * The LLM hit the output token limit before completing its JSON response.
+ */
+declare class AiOutputLengthError extends AiError {
+    constructor(message?: string, options?: ErrorOptions);
+}
+
+/**
+ * Semantic version of the location-extraction system prompt. Stored alongside
+ * each place record in the private app so results can be re-run when the
+ * prompt is updated. Bump when you change {@link LOCATIONS_SYSTEM_MESSAGE} or
+ * {@link LOCATIONS_SCHEMA} in a way that meaningfully changes output.
+ */
+declare const LOCATIONS_PROMPT_VERSION = "0.0.1";
+declare const LOCATIONS_FUNCTIONS_NAME = "get_places";
+declare const LOCATIONS_SYSTEM_MESSAGE = "You parse a list of place addresses, corresponding brief one-sentence descriptions, and a descriptive title (preferably directly quoted) from text. These addresses might be countries, cities, attractions, parks, bars, etc.\nIf the text seems to be listing places that are all within a single parent place, you don't return the parent place, but you do add it to all the childrens' addresses.\nFor example, for an article about places in greece, you would not return \"greece\", but if the Acropolis was mentioned, you would return \"Acropolis, Athens, Greece\"\n\nYou ignore extraneous text, such as recommended articles, advertisments, etc.\n";
+declare const LOCATIONS_SCHEMA: JSONSchema7Object;
+declare const LOCATIONS_MAX_OUTPUT_TOKENS = 500;
+/**
+ * Semantic version of the summarization prompt. Bump when meaningful changes
+ * are made to {@link SUMMARIZATION_PROMPT}.
+ */
+declare const SUMMARIZE_PROMPT_VERSION = "0.0.1";
+declare const SUMMARIZATION_PROMPT = "Summarize the given text. The summary must reduce the number of characters from the original text by at least a factor of {RATIO}.\nFor example, text of 1000 characters must output a summary less than {EXAMPLE} characters.\nThis summary will be used to extract a list of locations and addresses, so keep any text that seems to be a list of locations/places, a location and paragraph describing it, or otherwise clearly a location name or address.\nHere is the text:\n";
+
+/**
+ * Reduce a large text to a summary that will fit inside the model's context
+ * window alongside a known downstream prompt (e.g. the location-extraction
+ * call).
+ *
+ * Algorithm overview
+ * ------------------
+ * We need to fit both the summary and a downstream "final" prompt into the
+ * model's context window. Given:
+ *
+ * - `token_limit` — model context window (with a safety factor)
+ * - `text_tokens` — size of the input text
+ * - `summarize_prompt_tokens` — size of the summarization prompt itself
+ * - `final_prompt_tokens` — size of the downstream prompt
+ * - `final_output_tokens` — expected size of the downstream response
+ *
+ * We split the input into chunks, summarize each in parallel, and concatenate
+ * the results. The per-chunk size is chosen so that the combined summary
+ * leaves room for the downstream prompt:
+ *
+ *     final_input_tokens    = token_limit - final_output_tokens
+ *     summary_output_tokens = final_input_tokens - final_prompt_tokens
+ *     num_chunks            = ceil((text_tokens + summary_output_tokens)
+ *                                  / (token_limit - summarize_prompt_tokens))
+ *     chunk_output_tokens   = summary_output_tokens / num_chunks
+ *     chunk_input_tokens    = token_limit - summarize_prompt_tokens - chunk_output_tokens
+ *
+ * A safety factor is applied to chunk output sizes because LLMs routinely
+ * overshoot explicit length instructions.
+ */
+declare function summarizeText(backend: OpenAiBackend, text: string, finalPrompt?: string, finalOutputTokens?: number, chunkOutputSafetyFactor?: number, tokenLimitSafetyFactor?: number): Promise<SummarizedText>;
+
+/**
+ * Parse locations from freeform text. Two-stage: summarize to fit the context
+ * window, then a function-calling JSON completion for extraction.
+ */
+declare function parseLocationsFromText(backend: OpenAiBackend, text: string): Promise<GetLocationsResponse>;
+/**
+ * Parse locations from HTML. Equivalent to `htmlToText` followed by
+ * {@link parseLocationsFromText}.
+ */
+declare function parseLocationsFromHtml(backend: OpenAiBackend, html: string): Promise<GetLocationsResponse>;
+/**
+ * Parse locations from a URL. Equivalent to fetching the HTML, converting to
+ * text, and calling {@link parseLocationsFromText}. Throws any scrape errors
+ * unchanged.
+ */
+declare function parseLocationsFromUrl(backend: OpenAiBackend, url: string): Promise<GetLocationsResponse>;
+
+export { AiError, AiInputLengthError, AiOutputLengthError, AiResponseJsonError, GetLocationsResponse, InvalidJsonSchemaError, LOCATIONS_FUNCTIONS_NAME, LOCATIONS_MAX_OUTPUT_TOKENS, LOCATIONS_PROMPT_VERSION, LOCATIONS_SCHEMA, LOCATIONS_SYSTEM_MESSAGE, NoLocationsFoundError, OpenAiBackend, SUMMARIZATION_PROMPT, SUMMARIZE_PROMPT_VERSION, SummarizeTextError, SummarizedText, parseLocationsFromHtml, parseLocationsFromText, parseLocationsFromUrl, summarizeText };

package/dist/ai/index.js

@@ -0,0 +1,447 @@
+import { z } from 'zod';
+import { convert } from 'html-to-text';
+import { encodingForModel, getEncoding } from 'js-tiktoken';
+import OpenAI from 'openai';
+
+// src/types/errors.ts
+var MapthisError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MapthisError";
+  }
+};
+
+// src/ai/errors.ts
+var AiError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "AiError";
+  }
+};
+var NoLocationsFoundError = class extends AiError {
+  constructor(message, options) {
+    super(message ?? "No locations found in input", options);
+    this.name = "NoLocationsFoundError";
+  }
+};
+var SummarizeTextError = class extends AiError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "SummarizeTextError";
+  }
+};
+var InvalidJsonSchemaError = class extends AiError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "InvalidJsonSchemaError";
+  }
+};
+var AiResponseJsonError = class extends AiError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "AiResponseJsonError";
+  }
+};
+var AiInputLengthError = class extends AiError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "AiInputLengthError";
+  }
+};
+var AiOutputLengthError = class extends AiError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "AiOutputLengthError";
+  }
+};
+
+// src/ai/prompts.ts
+var LOCATIONS_PROMPT_VERSION = "0.0.1";
+var LOCATIONS_FUNCTIONS_NAME = "get_places";
+var LOCATIONS_SYSTEM_MESSAGE = `You parse a list of place addresses, corresponding brief one-sentence descriptions, and a descriptive title (preferably directly quoted) from text. These addresses might be countries, cities, attractions, parks, bars, etc.
+If the text seems to be listing places that are all within a single parent place, you don't return the parent place, but you do add it to all the childrens' addresses.
+For example, for an article about places in greece, you would not return "greece", but if the Acropolis was mentioned, you would return "Acropolis, Athens, Greece"
+
+You ignore extraneous text, such as recommended articles, advertisments, etc.
+`;
+var LOCATIONS_SCHEMA = {
+  type: "object",
+  required: ["locations", "title"],
+  properties: {
+    locations: {
+      type: "array",
+      items: {
+        type: "object",
+        required: ["address", "description"],
+        properties: {
+          address: { type: "string" },
+          description: { type: "string" }
+        }
+      }
+    },
+    title: { type: "string" }
+  }
+};
+var LOCATIONS_MAX_OUTPUT_TOKENS = 500;
+var SUMMARIZE_PROMPT_VERSION = "0.0.1";
+var SUMMARIZATION_PROMPT = `Summarize the given text. The summary must reduce the number of characters from the original text by at least a factor of {RATIO}.
+For example, text of 1000 characters must output a summary less than {EXAMPLE} characters.
+This summary will be used to extract a list of locations and addresses, so keep any text that seems to be a list of locations/places, a location and paragraph describing it, or otherwise clearly a location name or address.
+Here is the text:
+`;
+
+// src/scrape/errors.ts
+var ScrapeError = class extends MapthisError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "ScrapeError";
+  }
+};
+var InvalidUrlError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "InvalidUrlError";
+  }
+};
+var HtmlUnauthorizedError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlUnauthorizedError";
+  }
+};
+var HtmlToTextError = class extends ScrapeError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "HtmlToTextError";
+  }
+};
+
+// src/scrape/html.ts
+var withHttps = (url) => {
+  const hasHttp = /^http?:\/\//i.test(url);
+  const hasHttps = /^https?:\/\//i.test(url);
+  if (!hasHttp && !hasHttps) return `https://${url}`;
+  return url;
+};
+async function getHtmlFromUrl(url) {
+  const urlFixed = withHttps(url.toLowerCase());
+  const isUrl = z.string().url().safeParse(urlFixed);
+  if (!isUrl.success) throw new InvalidUrlError(`Invalid URL: ${url}`);
+  try {
+    const response = await fetch(urlFixed);
+    const html = await response.text();
+    if (response.status === 403 || response.status === 401) {
+      throw new HtmlUnauthorizedError(
+        `${response.status} Not authorized to scrape this website`
+      );
+    }
+    if (!response.ok) {
+      throw new ScrapeError(
+        `Bad response when getting HTML. Status: ${response.status} ${response.statusText}`
+      );
+    }
+    return html;
+  } catch (error) {
+    if (error instanceof TypeError) {
+      throw new InvalidUrlError(`Invalid URL. Original error: ${error.message}`);
+    }
+    throw error;
+  }
+}
+var SKIP_SELECTORS = [
+  "img",
+  "header",
+  "footer",
+  "audio",
+  "button",
+  "canvas",
+  "code",
+  "nav",
+  "#nav",
+  "figure",
+  "figcaption",
+  ".comment",
+  ".comments",
+  "#comments",
+  "#related-posts",
+  "#related",
+  ".related",
+  ".related-posts"
+];
+function htmlToText(html) {
+  try {
+    let out = convert(html, {
+      wordwrap: false,
+      selectors: [
+        {
+          selector: "a",
+          options: {
+            ignoreHref: true,
+            hideLinkHrefIfSameAsText: true
+          }
+        },
+        ...SKIP_SELECTORS.map((tag) => ({ selector: tag, format: "skip" }))
+      ]
+    });
+    out = out.replaceAll("\r\n", "\n");
+    for (let i = 0; i < 20; i++) {
+      out = out.replaceAll("\n\n\n", "\n\n");
+      out = out.replaceAll("  ", " ");
+    }
+    return out;
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new HtmlToTextError(`HTML to text conversion failed: ${error.message}`, {
+        cause: error
+      });
+    }
+    throw new HtmlToTextError("HTML to text conversion failed");
+  }
+}
+
+// src/ai/summarize.ts
+async function summarizeText(backend, text, finalPrompt = "", finalOutputTokens = 0, chunkOutputSafetyFactor = 0.3, tokenLimitSafetyFactor = 0.02) {
+  const tokenLimit = backend.tokenLimit * (1 - tokenLimitSafetyFactor);
+  const textTokens = backend.countTokens(text);
+  const summarizePromptTokens = backend.countTokens(SUMMARIZATION_PROMPT);
+  const finalPromptTokens = backend.countTokens(finalPrompt);
+  const finalInputTokens = tokenLimit - finalOutputTokens;
+  const summaryOutputTokens = finalInputTokens - finalPromptTokens;
+  const numChunks = Math.max(
+    1,
+    Math.ceil((textTokens + summaryOutputTokens) / (tokenLimit - summarizePromptTokens))
+  );
+  const chunkOutputTokens = Math.floor(
+    summaryOutputTokens / numChunks * (1 - chunkOutputSafetyFactor)
+  );
+  const chunkInputTokens = Math.floor(tokenLimit - summarizePromptTokens - chunkOutputTokens);
+  const summaryRatio = chunkOutputTokens / chunkInputTokens;
+  const textLines = text.split("\n");
+  const chunks = [];
+  let currentChunk = "";
+  let currentChunkTokens = 0;
+  const nextChunk = () => {
+    if (currentChunk) chunks.push(currentChunk);
+    currentChunk = "";
+    currentChunkTokens = 0;
+  };
+  for (const line of textLines) {
+    const numLineTokens = backend.countTokens(line) + 1;
+    if (numLineTokens > chunkInputTokens) {
+      nextChunk();
+      const charsPerToken = line.length / Math.max(1, backend.countTokens(line));
+      const charsPerChunk = Math.max(1, Math.floor(chunkInputTokens * charsPerToken));
+      for (let i = 0; i < line.length; i += charsPerChunk) {
+        chunks.push(line.slice(i, i + charsPerChunk));
+      }
+      continue;
+    }
+    if (currentChunkTokens + numLineTokens > chunkInputTokens) {
+      nextChunk();
+    }
+    currentChunkTokens += numLineTokens;
+    currentChunk += line + "\n";
+  }
+  nextChunk();
+  try {
+    const ratioText = (Math.ceil(summaryRatio * 10) / 10).toFixed(1);
+    const exampleText = Math.ceil(1e3 / Math.max(summaryRatio, 0.01)).toFixed(0);
+    const summaryPrompt = SUMMARIZATION_PROMPT.replace("{RATIO}", ratioText).replace(
+      "{EXAMPLE}",
+      exampleText
+    );
+    let inputTokens = 0;
+    let outputTokens = 0;
+    const summaries = await Promise.all(
+      chunks.map(async (chunk) => {
+        const response = await backend.chatCompletion(summaryPrompt + chunk);
+        inputTokens += response.inputTokens;
+        outputTokens += response.outputTokens;
+        return response.output;
+      })
+    );
+    const summarized = summaries.join("\n");
+    const summarizedTokens = backend.countTokens(summarized);
+    return {
+      text,
+      textTokens,
+      summarized,
+      summarizedTokens,
+      inputTokens,
+      outputTokens,
+      chunks: chunks.length,
+      summarizePromptVersion: SUMMARIZE_PROMPT_VERSION
+    };
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new SummarizeTextError(`Failed to summarize text: ${error.message}`, {
+        cause: error
+      });
+    }
+    throw new SummarizeTextError("Failed to summarize text (unknown error)");
+  }
+}
+
+// src/ai/locations.ts
+var FINAL_PROMPT = LOCATIONS_SYSTEM_MESSAGE + JSON.stringify(LOCATIONS_SCHEMA) + LOCATIONS_FUNCTIONS_NAME;
+async function callLocationExtraction(backend, text) {
+  const out = await backend.chatFunctionJson(
+    LOCATIONS_SYSTEM_MESSAGE,
+    text,
+    LOCATIONS_SCHEMA,
+    LOCATIONS_FUNCTIONS_NAME
+  );
+  const seen = /* @__PURE__ */ new Set();
+  const unique = out.output.locations.filter((loc) => {
+    if (seen.has(loc.address)) return false;
+    seen.add(loc.address);
+    return true;
+  });
+  if (unique.length === 0) throw new NoLocationsFoundError();
+  out.output.locations = unique;
+  return out;
+}
+async function parseLocationsFromText(backend, text) {
+  const {
+    summarized,
+    inputTokens: sumIn,
+    outputTokens: sumOut,
+    textTokens,
+    summarizedTokens,
+    summarizePromptVersion
+  } = await summarizeText(backend, text, FINAL_PROMPT, LOCATIONS_MAX_OUTPUT_TOKENS);
+  const {
+    output,
+    inputTokens: locIn,
+    outputTokens: locOut
+  } = await callLocationExtraction(backend, summarized);
+  return {
+    output,
+    locationInputTokens: locIn,
+    locationOutputTokens: locOut,
+    summaryInputTokens: sumIn,
+    summaryOutputTokens: sumOut,
+    inputTokens: sumIn + locIn,
+    outputTokens: sumOut + locOut,
+    textTokens,
+    summarizedTokens,
+    summaryPromptVersion: summarizePromptVersion,
+    locationPromptVersion: LOCATIONS_PROMPT_VERSION
+  };
+}
+async function parseLocationsFromHtml(backend, html) {
+  return parseLocationsFromText(backend, htmlToText(html));
+}
+async function parseLocationsFromUrl(backend, url) {
+  const html = await getHtmlFromUrl(url);
+  return parseLocationsFromHtml(backend, html);
+}
+var DEFAULT_MODEL = "gpt-4o-mini";
+var DEFAULT_TOKEN_LIMIT = 128e3;
+function resolveEncoding(model) {
+  try {
+    return encodingForModel(model);
+  } catch {
+    return getEncoding("cl100k_base");
+  }
+}
+function createOpenAiBackend(config) {
+  const model = config.model ?? DEFAULT_MODEL;
+  const tokenLimit = config.tokenLimit ?? DEFAULT_TOKEN_LIMIT;
+  const openai = new OpenAI({
+    apiKey: config.apiKey,
+    ...config.baseURL ? { baseURL: config.baseURL } : {}
+  });
+  const encoding = resolveEncoding(model);
+  return {
+    model,
+    tokenLimit,
+    countTokens(text) {
+      return encoding.encode(text).length;
+    },
+    async chatCompletion(content) {
+      const response = await openai.chat.completions.create({
+        messages: [{ role: "user", content }],
+        model
+      });
+      return {
+        output: response.choices[0]?.message.content ?? "",
+        inputTokens: response.usage?.prompt_tokens ?? 0,
+        outputTokens: response.usage?.completion_tokens ?? 0
+      };
+    },
+    async chatFunctionJson(systemMessage, userMessage, schema, functionName) {
+      try {
+        const completion = await openai.chat.completions.create({
+          model,
+          messages: [
+            { role: "system", content: systemMessage },
+            { role: "user", content: userMessage }
+          ],
+          functions: [{ name: functionName, parameters: schema }],
+          function_call: { name: functionName }
+        });
+        const functionCall = completion.choices[0]?.message.function_call;
+        if (!functionCall) {
+          throw new AiResponseJsonError("No function call in OpenAI response");
+        }
+        const inputTokens = completion.usage?.prompt_tokens ?? 0;
+        const outputTokens = completion.usage?.completion_tokens ?? 0;
+        try {
+          const output = JSON.parse(functionCall.arguments);
+          return { output, inputTokens, outputTokens };
+        } catch (error) {
+          if (error instanceof SyntaxError) {
+            const msg = error.message.toLowerCase();
+            if (msg.includes("unexpected end of json input") || msg.includes("unterminated string in js")) {
+              throw new AiOutputLengthError(
+                `Output tokens exceeded: ${error.message}`,
+                { cause: error }
+              );
+            }
+            throw new AiResponseJsonError(
+              `Failed to parse JSON from OpenAI response: ${error.message}`,
+              { cause: error }
+            );
+          }
+          throw new AiResponseJsonError(
+            "Failed to parse JSON from OpenAI response (unknown reason)"
+          );
+        }
+      } catch (error) {
+        if (error instanceof AiError) throw error;
+        if (error instanceof OpenAI.APIError) {
+          const msg = error.message.toLowerCase();
+          if (msg.includes("maximum context")) {
+            throw new AiInputLengthError(`Text too long: ${error.message}`, {
+              cause: error
+            });
+          }
+          if (msg.includes("invalid schema")) {
+            throw new InvalidJsonSchemaError(`Invalid schema: ${error.message}`, {
+              cause: error
+            });
+          }
+          throw new AiError(`OpenAI error: ${error.message}`, { cause: error });
+        }
+        throw new AiError("OpenAI error (unknown)", {
+          cause: error instanceof Error ? error : void 0
+        });
+      }
+    }
+  };
+}
+
+// src/ai/parser.ts
+function createLocationParser(config) {
+  const backend = createOpenAiBackend(config);
+  return {
+    parseLocationsFromText: (text) => parseLocationsFromText(backend, text),
+    parseLocationsFromHtml: (html) => parseLocationsFromHtml(backend, html),
+    parseLocationsFromUrl: (url) => parseLocationsFromUrl(backend, url)
+  };
+}
+
+export { AiError, AiInputLengthError, AiOutputLengthError, AiResponseJsonError, DEFAULT_MODEL, DEFAULT_TOKEN_LIMIT, InvalidJsonSchemaError, LOCATIONS_FUNCTIONS_NAME, LOCATIONS_MAX_OUTPUT_TOKENS, LOCATIONS_PROMPT_VERSION, LOCATIONS_SCHEMA, LOCATIONS_SYSTEM_MESSAGE, NoLocationsFoundError, SUMMARIZATION_PROMPT, SUMMARIZE_PROMPT_VERSION, SummarizeTextError, createLocationParser, createOpenAiBackend, parseLocationsFromHtml, parseLocationsFromText, parseLocationsFromUrl, summarizeText };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map