@graphext/cuery 0.2.0

package/esm/src/tools/topics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"topics.d.ts","sourceRoot":"","sources":["../../../src/src/tools/topics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,gDAAgD,CAAC;AAEnE,OAAO,EAAsC,KAAK,QAAQ,EAAE,MAAM,cAAc,CAAC;AAEjF,OAAO,EAEN,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,MAAM,6BAA6B,CAAC;AAIrC,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AAuEpD;;;GAGG;AACH,eAAO,MAAM,KAAK;;;iBAShB,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,YAAY;;;;;iBAEvB,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,QAAQ;;;;;iBAEnB,CAAC;AAEH;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC,CAKpH;AACD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,YAAY;;;;;;;kBAmBvD;AAyCD;;GAEG;AACH,wBAAsB,WAAW,CAChC,IAAI,EAAE,MAAM,GAAG,IAAI,EACnB,QAAQ,EAAE,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,MAAM,EAClD,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAClC,KAAK,GAAE,MAAkB,EACzB,WAAW,GAAE,QAA4C,GACvD,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CA2B5B;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC3B,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAC3B,QAAQ,EAAE,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,EACzC,KAAK,GAAE,MAAkB,EACzB,WAAW,GAAE,QAA4C,EACzD,cAAc,GAAE,MAAY,GAC1B,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAgBnC;AAED,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,EACnC,OAAO,EACP,OAAY,EACZ,UAAc,EACd,YAAiB,EACjB,UAAgB,EAChB,KAAiB,EACjB,WAAgB,EAChB,UAAc,EACd,QAA6C,EAC7C,EAAE,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAsChD"}

package/esm/src/tools/topics.js

@@ -0,0 +1,229 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import { z } from '../../deps/jsr.io/@zod/zod/4.1.12/src/index.js';
+import { mapParallel } from '../async.js';
+import { askOpenAISafe } from '../openai.js';
+import { TopicSchema } from '../schemas/topics.schema.js';
+import { dedent, formatRecordsAttrWise } from '../utils.js';
+/**
+ * Levenshtein distance implementation for string similarity validation
+ */
+function levenshteinDistance(a, b) {
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    const matrix = Array(a.length + 1).fill(null).map(() => Array(b.length + 1).fill(0));
+    for (let i = 0; i <= a.length; i++) {
+        matrix[i][0] = i;
+    }
+    for (let j = 0; j <= b.length; j++) {
+        matrix[0][j] = j;
+    }
+    for (let i = 1; i <= a.length; i++) {
+        for (let j = 1; j <= b.length; j++) {
+            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[i][j] = Math.min(matrix[i - 1][j] + 1, // deletion
+            matrix[i][j - 1] + 1, // insertion
+            matrix[i - 1][j - 1] + cost // substitution
+            );
+        }
+    }
+    return matrix[a.length][b.length];
+}
+/**
+ * Validates that subtopics are sufficiently distinct from each other and from the parent topic.
+ */
+function validateSubtopics(topic, subtopics) {
+    const MIN_LDIST = 2;
+    const subtopicsLower = subtopics.map((st) => st.toLowerCase());
+    const errors = [];
+    for (let i = 0; i < subtopicsLower.length; i++) {
+        const st = subtopicsLower[i];
+        // Subtopics should not be too similar to their parent topic
+        if (levenshteinDistance(st, topic.toLowerCase()) < MIN_LDIST) {
+            errors.push(`Subtopic '${st}' too similar to parent topic '${topic}'.`);
+        }
+        // Subtopics should not be too similar to each other
+        for (let j = i + 1; j < subtopicsLower.length; j++) {
+            const other = subtopicsLower[j];
+            // Check Levenshtein distance for similarity
+            if (levenshteinDistance(st.replace(/ /g, ''), other.replace(/ /g, '')) < MIN_LDIST) {
+                errors.push(`Subtopic '${st}' too similar to other subtopic '${other}'.`);
+            }
+            // Check for permutations of words
+            const stWords = new Set(st.split(' '));
+            const otherWords = new Set(other.split(' '));
+            if (stWords.size === otherWords.size &&
+                [...stWords].every(word => otherWords.has(word))) {
+                errors.push(`Subtopic '${st}' is a duplicate (permutation) of subtopic '${other}'.`);
+            }
+        }
+    }
+    return errors;
+}
+/**
+ * Topic schema with subtopic validation
+ * Validates that subtopics are sufficiently distinct from each other and from the parent topic.
+ */
+export const Topic = TopicSchema.superRefine(({ topic, subtopics }, ctx) => {
+    const errors = validateSubtopics(topic, subtopics);
+    for (const error of errors) {
+        ctx.addIssue({
+            code: 'custom',
+            message: error,
+            path: ['subtopics']
+        });
+    }
+});
+/**
+ * Taxonomy schema WITHOUT validation for fallback parsing
+ */
+export const TaxonomyBase = z.object({
+    topics: z.array(TopicSchema)
+});
+/**
+ * Taxonomy schema with topic validation
+ */
+export const Taxonomy = z.object({
+    topics: z.array(Topic)
+});
+/**
+ * Converts a taxonomy in the format Record<string, Array<string>> to an array
+ * of topic objects compatible with the Topic schema.
+ */
+export function toTopics(taxonomy) {
+    return Object.entries(taxonomy).map(([topic, subtopics]) => ({
+        topic,
+        subtopics
+    }));
+}
+/**
+ * Creates a dynamic TopicLabel schema based on a topic hierarchy
+ * Validates that subtopic belongs to the correct topic
+ */
+export function createLabelSchema(taxonomy) {
+    const topicNames = taxonomy.topics.map(t => t.topic);
+    const allSubtopics = taxonomy.topics.flatMap(t => t.subtopics);
+    return z.object({
+        topic: z.enum(topicNames),
+        subtopic: z.enum(allSubtopics)
+    }).superRefine((data, ctx) => {
+        const topicEntry = taxonomy.topics.find(t => t.topic === data.topic);
+        const allowedSubtopics = topicEntry?.subtopics || [];
+        if (!allowedSubtopics.includes(data.subtopic)) {
+            ctx.addIssue({
+                code: 'custom',
+                message: `Subtopic '${data.subtopic}' is not a valid subtopic for topic '${data.topic}'. ` +
+                    `Allowed subtopics are: ${allowedSubtopics.join(', ')}.`,
+                path: ['subtopic']
+            });
+        }
+    });
+}
+const TOPICS_PROMPT = dedent(`
+# Instructions
+
+From the data records below, extract a two-level nested list of topics.
+The output should be a JSON object with top-level topics as keys and lists of subtopics as values.
+The top-level should not contain more than {n_topics} topics, and each top-level
+should not contain more than {n_subtopics} subtopics.
+
+Make sure top-level topics are generalizable and capture broad themes.
+Subtopics should represent more specific categories within each theme.
+Both topics and subtopics must be written in {language}.
+
+{instructions}
+
+# Data Records
+
+{records}
+`);
+const LABEL_PROMPT_SYSTEM = dedent(`
+You're task is to use the following hierarchy of topics and subtopics (in json format),
+to assign the correct topic and subtopic to each text in the input.
+You cannot invent new topics or subtopics.
+You cannot assign a subtopic that does not belong to the assigned topic.
+
+# Topics
+
+{taxonomy}
+`);
+const LABEL_PROMPT_USER = dedent(`
+Assign the correct topic and subtopic to the following text.
+
+# Text
+
+{text}
+`);
+/**
+ * Assigns a topic and subtopic to a single text using an LLM call.
+ */
+export async function assignTopic(text, taxonomy, labelSchema, model = 'gpt-5.1', modelParams = { reasoning: { effort: 'none' } }) {
+    if (text == null || text.trim() === '') {
+        return null;
+    }
+    const systemPrompt = LABEL_PROMPT_SYSTEM.replace('{taxonomy}', typeof taxonomy === 'string' ? taxonomy : JSON.stringify(taxonomy, null, 2));
+    const userPrompt = LABEL_PROMPT_USER.replace('{text}', text);
+    const prompts = [
+        { role: 'system', content: systemPrompt },
+        { role: 'user', content: userPrompt }
+    ];
+    try {
+        const { parsed, error } = await askOpenAISafe(prompts, model, labelSchema, modelParams);
+        if (error != null || parsed == null) {
+            return null;
+        }
+        return parsed;
+    }
+    catch (error) {
+        console.warn(`Failed to assign topic for text "${text.substring(0, 50)}...":`, error);
+        return null;
+    }
+}
+/**
+ * Assigns topics to multiple texts concurrently while preserving order.
+ */
+export function assignTopics(texts, taxonomy, model = 'gpt-5.1', modelParams = { reasoning: { effort: 'none' } }, maxConcurrency = 100) {
+    // Precompute normalized taxonomy and label schema once for whole batch
+    const normalizedTaxonomy = Array.isArray(taxonomy)
+        ? { topics: taxonomy }
+        : taxonomy;
+    const labelSchema = createLabelSchema(normalizedTaxonomy);
+    const serializedTaxonomy = JSON.stringify(normalizedTaxonomy, null, 2);
+    return mapParallel(texts, maxConcurrency, text => assignTopic(text, serializedTaxonomy, labelSchema, model, modelParams));
+}
+/**
+ * Extracts a topic hierarchy from an array of records using an LLM.
+ */
+export async function extractTopics({ records, nTopics = 10, nSubtopics = 5, instructions = '', maxSamples = 500, model = 'gpt-4.1', modelParams = {}, maxRetries = 8, language = 'The same language as the records' }) {
+    if (!records || records.length === 0) {
+        return { topics: [] };
+    }
+    const sampledRecords = records.length > maxSamples
+        ? records.slice(0, maxSamples)
+        : records;
+    const formattedRecords = formatRecordsAttrWise(sampledRecords);
+    const prompt = TOPICS_PROMPT
+        .replace('{n_topics}', String(nTopics))
+        .replace('{n_subtopics}', String(nSubtopics))
+        .replace('{instructions}', instructions)
+        .replace('{records}', formattedRecords)
+        .replace('{language}', language);
+    const { parsed, output_text, error } = await askOpenAISafe(prompt, model, Taxonomy, modelParams, maxRetries, 'return');
+    if (error != null) {
+        if (output_text == null) {
+            throw new Error('Failed to get response from OpenAI');
+        }
+        try {
+            const taxonomy = JSON.parse(output_text);
+            return TaxonomyBase.parse(taxonomy);
+        }
+        catch (parseError) {
+            throw new Error(`Failed to parse response even without validation: ${parseError instanceof Error ? parseError.message : String(parseError)}`);
+        }
+    }
+    if (parsed == null) {
+        throw new Error('Failed to parse response from OpenAI');
+    }
+    return parsed;
+}

package/esm/src/tools/translate.d.ts

@@ -0,0 +1,44 @@
+import type { BrandContext } from '../schemas/brand.schema.js';
+export interface translateParams {
+    keyword: string;
+    language?: string;
+    model?: string;
+    instructions?: string;
+    brandContext: BrandContext;
+}
+/**
+ * Translates a single Google search keyword into an equivalent LLM prompt.
+ */
+export declare function translate({ keyword, language, model, brandContext, instructions }: translateParams): Promise<string>;
+/**
+ * Translates multiple Google search keywords to LLM prompts concurrently while preserving order.
+ */
+export interface translateBatchParams {
+    keywords: Array<string>;
+    language?: string;
+    model?: string;
+    maxConcurrency?: number;
+    instructions?: string;
+    brandContext: BrandContext;
+}
+export declare function translateBatch(params: translateBatchParams): Promise<Array<string>>;
+export interface reverseTranslateParams {
+    prompt: string;
+    language?: string;
+    model?: string;
+}
+/**
+ * Converts a natural language prompt into an equivalent Google search keyword.
+ */
+export declare function reverseTranslate({ prompt, language, model }: reverseTranslateParams): Promise<string>;
+export interface reverseTranslateBatchParams {
+    prompts: Array<string>;
+    language?: string;
+    model?: string;
+    maxConcurrency?: number;
+}
+/**
+ * Converts multiple natural language prompts to Google search keywords concurrently while preserving order.
+ */
+export declare function reverseTranslateBatch(params: reverseTranslateBatchParams): Promise<Array<string>>;
+//# sourceMappingURL=translate.d.ts.map

package/esm/src/tools/translate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"translate.d.ts","sourceRoot":"","sources":["../../../src/src/tools/translate.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AA+C/D,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC9B,EAAE,OAAO,EAAE,QAAe,EAAE,KAAoB,EAAE,YAAY,EAAE,YAAiB,EAAE,EAAE,eAAe,GAClG,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED;;GAEG;AAEH,MAAM,WAAW,oBAAoB;IACpC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;CAC3B;AAED,wBAAsB,cAAc,CACnC,MAAM,EAAE,oBAAoB,GAC1B,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAQxB;AA4BD,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACrC,EAAE,MAAM,EAAE,QAAe,EAAE,KAAsB,EAAE,EAAE,sBAAsB,GACzE,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CAC1C,MAAM,EAAE,2BAA2B,GACjC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAQxB"}

package/esm/src/tools/translate.js

@@ -0,0 +1,112 @@
+import { mapParallel } from '../async.js';
+import { askOpenAISafe } from '../openai.js';
+import { dedent } from '../utils.js';
+const TRANSLATE_PROMPT = dedent(`
+You are an expert in understanding search intent and conversational AI interactions.
+
+Convert the following Google search keyword into an equivalent natural language prompt that
+someone would use when interacting with an LLM like ChatGPT. The LLM prompt should:
+
+- Be conversational and natural, as if speaking to an assistant
+- Capture the same intent and information need as the keyword
+- Be a complete sentence or question
+- Not be overly verbose - keep it concise but natural
+- Be written in "{language}"
+
+For example:
+- "best running shoes" → "What are the best running shoes?"
+- "how to train for marathon" → "How do I train for a marathon?"
+- "nike air max review" → "Can you give me a review of Nike Air Max shoes?"
+- "weather new york" → "What's the weather like in New York?"
+
+## Brand Context Rules
+
+You will receive brand context (sector, country). Follow these rules strictly:
+
+1. **DO NOT** change the core intent or meaning of the keyword
+2. **DO** use brand context to disambiguate terms when multiple interpretations exist
+3. The keyword's original intent must be preserved - brand context only helps clarify ambiguous terms
+
+**Example of correct disambiguation:**
+- Keyword: "geo"
+- Brand sector: "ai visibility search"
+- WRONG: "Tell me about geography" (ignores sector context)
+- CORRECT: "What is GEO in AI visibility and search optimization?" (uses sector to disambiguate the acronym)
+
+{additional_instructions}
+
+# Keyword
+
+{keyword}
+
+# Brand Context (for reference only)
+- Sector: {sector}
+- Country: {country}
+
+Return only the converted prompt, nothing else.
+`);
+/**
+ * Translates a single Google search keyword into an equivalent LLM prompt.
+ */
+export async function translate({ keyword, language = 'en', model = 'gpt-5-mini', brandContext, instructions = '' }) {
+    const prompt = TRANSLATE_PROMPT
+        .replace('{keyword}', keyword)
+        .replace('{language}', language)
+        .replace('{sector}', brandContext.sector)
+        .replace('{country}', brandContext.country)
+        .replace('{additional_instructions}', instructions);
+    const { parsed } = await askOpenAISafe(prompt, model);
+    if (!parsed) {
+        throw new Error('Failed to parse translation from OpenAI response');
+    }
+    return parsed;
+}
+export async function translateBatch(params) {
+    const { keywords, language = 'en', model = 'gpt-4.1-mini', maxConcurrency = 100, brandContext, instructions = '' } = params;
+    return mapParallel(keywords, maxConcurrency, kwd => translate({ keyword: kwd, language, model, brandContext, instructions }));
+}
+const REVERSE_TRANSLATE_PROMPT = dedent(`
+You are an expert in understanding search intent.
+
+Convert the following natural language prompt into an equivalent Google search keyword.
+The keyword should:
+
+- Be concise (typically 2-5 words)
+- Capture the core search intent
+- Remove conversational filler and politeness
+- Use common search terms
+
+For example:
+- "What are the best running shoes?" → "best running shoes"
+- "How do I train for a marathon?" → "how to train marathon"
+- "Can you give me a review of Nike Air Max shoes?" → "nike air max review"
+- "What's the weather like in New York?" → "weather new york"
+
+Make sure the keyword is in the language "{language}".
+
+# Prompt
+
+{prompt}
+
+Return only the keyword, nothing else.
+`);
+/**
+ * Converts a natural language prompt into an equivalent Google search keyword.
+ */
+export async function reverseTranslate({ prompt, language = 'en', model = 'gpt-4.1-mini' }) {
+    const requestPrompt = REVERSE_TRANSLATE_PROMPT
+        .replace('{prompt}', prompt)
+        .replace('{language}', language);
+    const { parsed } = await askOpenAISafe(requestPrompt, model);
+    if (!parsed) {
+        throw new Error('Failed to parse reverse translation from OpenAI response');
+    }
+    return parsed;
+}
+/**
+ * Converts multiple natural language prompts to Google search keywords concurrently while preserving order.
+ */
+export async function reverseTranslateBatch(params) {
+    const { prompts, language = 'en', model = 'gpt-5-mini', maxConcurrency = 100 } = params;
+    return mapParallel(prompts, maxConcurrency, p => reverseTranslate({ prompt: p, language, model }));
+}

package/esm/src/urls.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Parse a URL, adding scheme if missing (internal helper).
+ */
+export declare function parseUrl(url: string): URL;
+/**
+ * Extract the domain from a URL or hostname.
+ * Handles incomplete URLs (missing scheme), Google Translate redirects, and subdomains.
+ */
+export declare function extractDomain(url: string, withSubdomain?: boolean, resolveGoogleTranslate?: boolean): string;
+//# sourceMappingURL=urls.d.ts.map

package/esm/src/urls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"urls.d.ts","sourceRoot":"","sources":["../../src/src/urls.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAQzC;AAaD;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,GAAG,EAAE,MAAM,EACX,aAAa,GAAE,OAAe,EAC9B,sBAAsB,GAAE,OAAc,GACpC,MAAM,CA6BR"}

package/esm/src/urls.js

@@ -0,0 +1,49 @@
+import { parse } from 'tldts';
+/**
+ * Parse a URL, adding scheme if missing (internal helper).
+ */
+export function parseUrl(url) {
+    let normalizedUrl = url.trim();
+    if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://') && !normalizedUrl.startsWith('//')) {
+        normalizedUrl = 'http://' + normalizedUrl;
+    }
+    return new URL(normalizedUrl);
+}
+/**
+ * Check if a URL is a Google Translate URL (internal helper).
+ */
+function isGoogleTranslateUrl(parsedUrl) {
+    return (parsedUrl.hostname === 'translate.google.com' &&
+        parsedUrl.pathname.startsWith('/translate') &&
+        parsedUrl.searchParams.has('u'));
+}
+/**
+ * Extract the domain from a URL or hostname.
+ * Handles incomplete URLs (missing scheme), Google Translate redirects, and subdomains.
+ */
+export function extractDomain(url, withSubdomain = false, resolveGoogleTranslate = true) {
+    if (!url) {
+        return url;
+    }
+    try {
+        let targetUrl = url.trim();
+        if (resolveGoogleTranslate) {
+            const parsedUrl = parseUrl(targetUrl);
+            if (isGoogleTranslateUrl(parsedUrl)) {
+                const originalUrl = parsedUrl.searchParams.get('u');
+                if (originalUrl != null) {
+                    targetUrl = originalUrl;
+                }
+            }
+        }
+        const extracted = parse(targetUrl);
+        if (withSubdomain) {
+            const fqdn = extracted.hostname ?? '';
+            return fqdn.replace(/^www\./, '');
+        }
+        return extracted.domain ?? url;
+    }
+    catch {
+        return url;
+    }
+}

package/esm/src/utils.d.ts

@@ -0,0 +1,38 @@
+/**
+ * General utility functions for text processing, record formatting, and arrays.
+ */
+/**
+ * Remove leading whitespace from a string, similar to Python's textwrap.dedent.
+ * Also collapses newlines within paragraphs (except for markdown list items).
+ */
+export declare function dedent(text: string): string;
+/**
+ * Clean a string to make it suitable for use as a column name.
+ * Converts to lowercase, replaces non-alphanumeric characters with underscores,
+ * and removes consecutive underscores.
+ */
+export declare function cleanColumnName(name: string): string;
+/**
+ * Formats records in attribute-wise format (column-oriented).
+ * Each section shows all values for a single attribute across all records.
+ * Equivalent to records_attr_wise.jinja template.
+ *
+ * @param records - Array of record objects to format
+ * @returns Formatted string with records grouped by attribute
+ */
+export declare function formatRecordsAttrWise(records: Array<Record<string, unknown>>): string;
+/**
+ * Randomly sample n elements from an array using Fisher-Yates shuffle.
+ */
+export declare function sampleArray<T>(array: Array<T>, n: number): Array<T>;
+/**
+ * Deduplicates an array of records based on a single scalar field.
+ * Keeps the first occurrence of each unique value and removes subsequent duplicates.
+ */
+export declare function deduplicate<T extends Record<string, unknown>>(records: Array<T>, field: keyof T): Array<T>;
+/**
+ * Deduplicates an array of records based on a single scalar field.
+ * Keeps the last occurrence of each unique value, removing earlier duplicates.
+ */
+export declare function deduplicateLast<T extends Record<string, unknown>>(records: Array<T>, field: keyof T): Array<T>;
+//# sourceMappingURL=utils.d.ts.map

package/esm/src/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;;GAGG;AACH,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA8B3C;AAwCD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMpD;AAMD;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,MAAM,CAoCrF;AAMD;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAYnE;AAMD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5D,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EACjB,KAAK,EAAE,MAAM,CAAC,GACZ,KAAK,CAAC,CAAC,CAAC,CAoBV;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EACjB,KAAK,EAAE,MAAM,CAAC,GACZ,KAAK,CAAC,CAAC,CAAC,CAgBV"}

package/esm/src/utils.js

@@ -0,0 +1,175 @@
+/**
+ * General utility functions for text processing, record formatting, and arrays.
+ */
+// ============================================================================
+// Text Processing
+// ============================================================================
+/**
+ * Remove leading whitespace from a string, similar to Python's textwrap.dedent.
+ * Also collapses newlines within paragraphs (except for markdown list items).
+ */
+export function dedent(text) {
+    /**
+     * Check if a line is a markdown list item.
+     */
+    function isMarkdownListItem(line) {
+        const trimmedLine = line.trim();
+        // Unordered lists
+        if (trimmedLine.startsWith('- ') ||
+            trimmedLine.startsWith('* ') ||
+            trimmedLine.startsWith('+ ')) {
+            return true;
+        }
+        // Ordered lists: Detect a number or single letter followed by a dot and space
+        return /^\d+\. |^[a-zA-Z]\. /.test(trimmedLine);
+    }
+    // Clean doc: Remove leading/trailing whitespace and normalize indentation
+    const cleanedText = cleanDoc(text);
+    // Split into paragraphs
+    const paragraphs = cleanedText.split('\n\n');
+    // Replace newlines with spaces in non-list paragraphs
+    const processedParagraphs = paragraphs.map((p) => isMarkdownListItem(p) ? p : p.replace(/\n/g, ' '));
+    return processedParagraphs.join('\n\n').trim();
+}
+/**
+ * Clean up indentation in a multi-line string (similar to Python's inspect.cleandoc).
+ * Removes leading blank lines, trailing blank lines, and common leading whitespace.
+ */
+function cleanDoc(text) {
+    const lines = text.split('\n');
+    // Remove leading blank lines
+    while (lines.length > 0 && lines[0].trim() === '') {
+        lines.shift();
+    }
+    // Remove trailing blank lines
+    while (lines.length > 0 && lines[lines.length - 1].trim() === '') {
+        lines.pop();
+    }
+    if (lines.length === 0) {
+        return '';
+    }
+    // Find the minimum indentation (ignoring blank lines)
+    let minIndent = Infinity;
+    for (const line of lines) {
+        if (line.trim() !== '') {
+            const indent = line.match(/^[ \t]*/)?.[0].length ?? 0;
+            minIndent = Math.min(minIndent, indent);
+        }
+    }
+    // Remove the common leading whitespace
+    if (minIndent > 0 && minIndent !== Infinity) {
+        return lines.map((line) => line.trim() === '' ? line : line.slice(minIndent)).join('\n');
+    }
+    return lines.join('\n');
+}
+/**
+ * Clean a string to make it suitable for use as a column name.
+ * Converts to lowercase, replaces non-alphanumeric characters with underscores,
+ * and removes consecutive underscores.
+ */
+export function cleanColumnName(name) {
+    return name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '')
+        .replace(/_+/g, '_');
+}
+// ============================================================================
+// Record Formatting
+// ============================================================================
+/**
+ * Formats records in attribute-wise format (column-oriented).
+ * Each section shows all values for a single attribute across all records.
+ * Equivalent to records_attr_wise.jinja template.
+ *
+ * @param records - Array of record objects to format
+ * @returns Formatted string with records grouped by attribute
+ */
+export function formatRecordsAttrWise(records) {
+    if (!records || records.length === 0) {
+        return '';
+    }
+    const attributes = Object.keys(records[0]);
+    const sections = [];
+    for (const attrName of attributes) {
+        const title = attrName.replace(/_/g, ' ')
+            .split(' ')
+            .map(word => word.charAt(0).toUpperCase() + word.slice(1))
+            .join(' ');
+        const values = [];
+        for (const record of records) {
+            const value = record[attrName];
+            if (value == null || value === '') {
+                continue;
+            }
+            if (Array.isArray(value) || (typeof value === 'object' && value !== null)) {
+                values.push(JSON.stringify(value));
+            }
+            else {
+                values.push(String(value));
+            }
+        }
+        if (values.length > 0) {
+            sections.push(`## ${title}\n\n${values.join(', ')}`);
+        }
+    }
+    return sections.join('\n\n');
+}
+// ============================================================================
+// Array Utilities
+// ============================================================================
+/**
+ * Randomly sample n elements from an array using Fisher-Yates shuffle.
+ */
+export function sampleArray(array, n) {
+    if (n >= array.length) {
+        return [...array];
+    }
+    const shuffled = [...array];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return shuffled.slice(0, n);
+}
+// ============================================================================
+// Deduplication
+// ============================================================================
+/**
+ * Deduplicates an array of records based on a single scalar field.
+ * Keeps the first occurrence of each unique value and removes subsequent duplicates.
+ */
+export function deduplicate(records, field) {
+    const seen = new Set();
+    const result = [];
+    for (const record of records) {
+        const value = record[field];
+        // Skip records where the field is null or undefined
+        if (value == null) {
+            continue;
+        }
+        // Only add if we haven't seen this value before
+        if (!seen.has(value)) {
+            seen.add(value);
+            result.push(record);
+        }
+    }
+    return result;
+}
+/**
+ * Deduplicates an array of records based on a single scalar field.
+ * Keeps the last occurrence of each unique value, removing earlier duplicates.
+ */
+export function deduplicateLast(records, field) {
+    const map = new Map();
+    for (const record of records) {
+        const value = record[field];
+        // Skip records where the field is null or undefined
+        if (value == null) {
+            continue;
+        }
+        // Always update, so last occurrence wins
+        map.set(value, record);
+    }
+    return Array.from(map.values());
+}