@get-technology-inc/jamf-docs-mcp-server 3.0.6 → 3.0.8

package/dist/core/services/scraper.js

@@ -1,990 +0,0 @@
-/**
- * Web scraping service for Jamf documentation
- *
- * This module handles fetching and parsing HTML from learn.jamf.com
- * (Jamf documentation has moved from docs.jamf.com to learn.jamf.com)
- */
-import { httpGetText, httpGetJson, HttpError } from '../http-client.js';
-import * as cheerio from 'cheerio';
-import { sanitizeErrorMessage } from '../utils/sanitize.js';
-import { isAllowedHostname, extractLocaleFromUrl } from '../utils/url.js';
-import TurndownService from 'turndown';
-import { DOCS_BASE_URL, DOCS_API_URL, JAMF_PRODUCTS, JAMF_TOPICS, DOC_TYPE_LABEL_MAP, SELECTORS, CONTENT_LIMITS, TOKEN_CONFIG, PAGINATION_CONFIG, DEFAULT_LOCALE, SUPPORTED_LOCALES } from '../constants.js';
-import { JamfDocsError, JamfDocsErrorCode } from '../types.js';
-import { extractVersionFromBundleId, extractProductSlug, compareVersions } from '../utils/bundle.js';
-import { docTypeFromLabels } from '../utils/doc-type.js';
-import { estimateTokens, createTokenInfo, extractSections, extractSection, extractSummary, truncateToTokenLimit, calculatePagination } from './tokenizer.js';
-import { getBundleIdForVersion } from './metadata.js';
-// Initialize Turndown for HTML to Markdown conversion
-const turndown = new TurndownService({
-    headingStyle: 'atx',
-    codeBlockStyle: 'fenced',
-    bulletListMarker: '-',
-    emDelimiter: '*',
-    strongDelimiter: '**'
-});
-// Custom Turndown rules
-turndown.addRule('codeBlocks', {
-    filter: 'pre',
-    replacement: (content, node) => {
-        // Handle code element extraction safely
-        const nodeElement = node;
-        const codeElement = nodeElement.querySelector?.('code');
-        const language = codeElement?.className?.replace('language-', '') ?? '';
-        return `\n\`\`\`${language}\n${content.trim()}\n\`\`\`\n`;
-    }
-});
-// Rate limiter
-let lastRequestTime = 0;
-async function throttle(requestConfig) {
-    const now = Date.now();
-    const elapsed = now - lastRequestTime;
-    if (elapsed < requestConfig.rateLimitDelay) {
-        await new Promise(resolve => setTimeout(resolve, requestConfig.rateLimitDelay - elapsed));
-    }
-    lastRequestTime = Date.now();
-}
-// Re-export URL utilities for backward compatibility
-export { ALLOWED_HOSTNAMES, isAllowedHostname } from '../utils/url.js';
-/**
- * Strip locale prefix (e.g. /en-US/, /ja-JP/) from URL path.
- * Frontend URLs include locale in path but backend doesn't accept it.
- */
-export function stripLocalePrefix(urlStr) {
-    try {
-        const url = new URL(urlStr);
-        const segments = url.pathname.split('/').filter(Boolean);
-        const firstSegment = segments[0];
-        if (firstSegment !== undefined && firstSegment in SUPPORTED_LOCALES) {
-            url.pathname = `/${segments.slice(1).join('/')}`;
-            return url.toString();
-        }
-    }
-    catch {
-        // Invalid URL, return as-is
-    }
-    return urlStr;
-}
-// URL transformation between frontend (learn.jamf.com) and backend (learn-be.jamf.com)
-export function transformToBackendUrl(urlStr) {
-    const url = new URL(urlStr);
-    if (url.hostname === 'learn.jamf.com') {
-        url.hostname = 'learn-be.jamf.com';
-    }
-    return url.toString();
-}
-export function transformToFrontendUrl(urlStr) {
-    const url = new URL(urlStr);
-    if (url.hostname === 'learn-be.jamf.com') {
-        url.hostname = 'learn.jamf.com';
-    }
-    return url.toString();
-}
-// bundleId validation pattern
-const BUNDLE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*$/;
-function validateBundleId(bundleId) {
-    if (!BUNDLE_ID_PATTERN.test(bundleId)) {
-        throw new JamfDocsError(`Invalid bundle ID: ${bundleId}`, JamfDocsErrorCode.PARSE_ERROR);
-    }
-    return bundleId;
-}
-// HTML entity map for stripping HTML
-const HTML_ENTITIES = {
-    '&nbsp;': ' ', '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&#39;': "'"
-};
-/**
- * Strip HTML tags from a string
- * Uses iterative approach to handle nested/malformed tags
- */
-function stripHtml(html) {
-    // Truncate input before processing to prevent CPU exhaustion on large inputs
-    let text = html.length > CONTENT_LIMITS.MAX_SNIPPET_LENGTH * 2
-        ? html.slice(0, CONTENT_LIMITS.MAX_SNIPPET_LENGTH * 2)
-        : html;
-    // Iteratively remove HTML tags until none remain (handles nested cases like <scr<script>ipt>)
-    const MAX_ITERATIONS = 10;
-    let prev = '';
-    let iterations = 0;
-    while (prev !== text && iterations < MAX_ITERATIONS) {
-        prev = text;
-        text = text.replace(/<[^>]*>/g, '');
-        iterations++;
-    }
-    for (const [entity, char] of Object.entries(HTML_ENTITIES)) {
-        text = text.replaceAll(entity, char);
-    }
-    return text.replace(/\s+/g, ' ').trim();
-}
-// Minimum meaningful snippet length
-const MIN_SNIPPET_LENGTH = 50;
-// Breadcrumb/navigation patterns to filter
-const NAV_PATTERNS = [
-    /^Home\s*>/i,
-    /^[\w\s]+>\s*[\w\s]+>\s*[\w\s]+/, // "A > B > C" breadcrumb pattern
-];
-/**
- * Clean and validate a search result snippet
- * Filters navigation-only text and ensures minimum content quality
- */
-export function cleanSnippet(snippet, title, product) {
-    let cleaned = snippet;
-    // Strip navigation/breadcrumb prefixes
-    for (const pattern of NAV_PATTERNS) {
-        cleaned = cleaned.replace(pattern, '').trim();
-    }
-    // If snippet is too short after cleaning, use fallback
-    if (cleaned.length < MIN_SNIPPET_LENGTH) {
-        const productSuffix = product !== null && product !== '' ? ` \u2014 ${product}` : '';
-        return `${title}${productSuffix}`;
-    }
-    return cleaned;
-}
-/**
- * Handle HTTP errors and convert to JamfDocsError
- */
-function handleHttpError(error, url, resourceType) {
-    const status = error.status;
-    if (status === 404) {
-        throw new JamfDocsError(`${resourceType} not found: ${url}`, JamfDocsErrorCode.NOT_FOUND, url, 404);
-    }
-    if (status === 429) {
-        throw new JamfDocsError('Rate limited. Please wait and try again.', JamfDocsErrorCode.RATE_LIMITED, url, 429);
-    }
-    throw new JamfDocsError(`Network error: ${sanitizeErrorMessage(error.message)}`, JamfDocsErrorCode.NETWORK_ERROR, url);
-}
-/**
- * Build Accept-Language header value for a given locale
- */
-export function buildAcceptLanguage(locale) {
-    if (locale === 'en-US') {
-        return 'en-US,en;q=0.9';
-    }
-    return `${locale},en;q=0.5`;
-}
-/**
- * Fetch JSON from a URL with error handling
- */
-function makeFetchJson(requestConfig) {
-    return async (url, locale) => {
-        await throttle(requestConfig);
-        try {
-            return await httpGetJson(url, {
-                timeout: requestConfig.timeout,
-                headers: {
-                    'User-Agent': requestConfig.userAgent,
-                    'Accept-Language': buildAcceptLanguage(locale ?? DEFAULT_LOCALE)
-                }
-            });
-        }
-        catch (error) {
-            if (error instanceof HttpError) {
-                handleHttpError(error, url, 'Resource');
-            }
-            throw error;
-        }
-    };
-}
-/**
- * Fetch HTML from a URL with error handling
- */
-function makeFetchHtml(requestConfig) {
-    return async (url, locale) => {
-        await throttle(requestConfig);
-        try {
-            return await httpGetText(url, {
-                timeout: requestConfig.timeout,
-                headers: {
-                    'User-Agent': requestConfig.userAgent,
-                    'Accept': 'text/html,application/xhtml+xml',
-                    'Accept-Language': buildAcceptLanguage(locale ?? DEFAULT_LOCALE)
-                }
-            });
-        }
-        catch (error) {
-            if (error instanceof HttpError) {
-                handleHttpError(error, url, 'Article');
-            }
-            throw error;
-        }
-    };
-}
-/**
- * Fetch HTML from a URL with error handling.
- * Requires a ServerContext for request configuration.
- */
-export async function fetchHtml(ctx, url, locale) {
-    return makeFetchHtml(ctx.config.request)(url, locale);
-}
-/**
- * Clean HTML content by removing unwanted elements
- */
-function cleanHtml($) {
-    // Remove unwanted elements
-    $(SELECTORS.REMOVE).remove();
-    // Fix relative URLs
-    $('a[href^="/"]').each((_, el) => {
-        const href = $(el).attr('href');
-        if (href !== undefined && href !== '') {
-            $(el).attr('href', `${DOCS_BASE_URL}${href}`);
-        }
-    });
-    $('img[src^="/"]').each((_, el) => {
-        const src = $(el).attr('src');
-        if (src !== undefined && src !== '') {
-            $(el).attr('src', `${DOCS_BASE_URL}${src}`);
-        }
-    });
-}
-/**
- * Extract product and version from URL
- * Supports both old docs.jamf.com and new learn.jamf.com URL structures
- */
-function extractProductInfo(url) {
-    // learn.jamf.com structure: /{locale}/bundle/{product}-documentation/page/{page}.html
-    const bundleMatch = /\/bundle\/([^/]+)-documentation\//.exec(url);
-    if (bundleMatch !== null) {
-        const product = Object.values(JAMF_PRODUCTS).find(p => p.bundleId.includes(bundleMatch[1] ?? ''));
-        return { product: product?.name, version: 'current' };
-    }
-    // Legacy docs.jamf.com structure: /{version}/{product}/...
-    const pathParts = new URL(url).pathname.split('/').filter(Boolean);
-    const versionMatch = pathParts[0]?.match(/^\d+\.\d+(\.\d+)?$/);
-    if (versionMatch !== null && versionMatch !== undefined) {
-        const product = Object.values(JAMF_PRODUCTS).find(p => p.urlPattern.includes(pathParts[1] ?? ''));
-        return { product: product?.name, version: pathParts[0] };
-    }
-    // Check for unversioned paths
-    const matchedProduct = Object.values(JAMF_PRODUCTS).find(p => url.includes(p.bundleId) || url.includes(p.id));
-    return { product: matchedProduct?.name, version: matchedProduct !== undefined ? 'current' : undefined };
-}
-/** Pre-computed lowercase keywords per topic (avoids repeated toLowerCase on static data) */
-const TOPIC_KEYWORDS_LOWER = Object.fromEntries(Object.keys(JAMF_TOPICS).map(id => [
-    id,
-    JAMF_TOPICS[id].keywords.map(k => k.toLowerCase())
-]));
-const ALL_TOPIC_IDS = Object.keys(JAMF_TOPICS);
-/**
- * Apply filters with progressive relaxation when results are zero
- * Relaxation order: docType → topic → product
- */
-function applyFiltersWithFallback(allResults, activeFilters) {
-    // Apply all filters
-    let filtered = allResults;
-    for (const filter of activeFilters) {
-        filtered = filter.apply(filtered);
-    }
-    if (filtered.length > 0 || activeFilters.length === 0) {
-        return { filtered };
-    }
-    // Progressive relaxation
-    const relaxOrder = ['docType', 'topic', 'product'];
-    const removed = [];
-    const original = {};
-    for (const filterName of relaxOrder) {
-        if (filtered.length > 0) {
-            break;
-        }
-        const filterIndex = activeFilters.findIndex(f => f.name === filterName);
-        if (filterIndex === -1) {
-            continue;
-        }
-        const removedFilter = activeFilters[filterIndex];
-        if (removedFilter === undefined) {
-            continue;
-        }
-        removed.push(removedFilter.name);
-        original[removedFilter.name] = removedFilter.value;
-        activeFilters.splice(filterIndex, 1);
-        // Re-apply remaining filters
-        filtered = allResults;
-        for (const filter of activeFilters) {
-            filtered = filter.apply(filtered);
-        }
-    }
-    if (removed.length > 0) {
-        return {
-            filtered,
-            relaxation: {
-                removed,
-                original,
-                message: `No results with all filters applied. Removed filter(s): ${removed.join(', ')}. Try broader search terms or fewer filters.`
-            }
-        };
-    }
-    return { filtered };
-}
-/**
- * Select the best result entry for the requested version from leading + followers.
- * Returns the matching entry's URL, bundle_id, and whether the match was exact.
- */
-function selectVersionedEntry(leading, followers, requestedVersion) {
-    const leadingVersion = extractVersionFromBundleId(leading.bundle_id) ?? 'current';
-    // No version filter or 'current' — use leading
-    if (requestedVersion === undefined || requestedVersion === '' || requestedVersion === 'current') {
-        return { url: leading.url, bundleId: leading.bundle_id, version: leadingVersion, versionMatched: true };
-    }
-    // Check if leading already matches
-    if (leadingVersion === requestedVersion) {
-        return { url: leading.url, bundleId: leading.bundle_id, version: leadingVersion, versionMatched: true };
-    }
-    // Search followers for matching version
-    if (followers !== undefined && followers.length > 0) {
-        for (const follower of followers) {
-            const followerVersion = extractVersionFromBundleId(follower.bundle_id);
-            if (followerVersion === requestedVersion) {
-                return { url: follower.url, bundleId: follower.bundle_id, version: followerVersion, versionMatched: true };
-            }
-        }
-    }
-    // No match found — fallback to leading
-    return { url: leading.url, bundleId: leading.bundle_id, version: leadingVersion, versionMatched: false };
-}
-/**
- * Transform a Zoomin leading result into a SearchResultWithMeta
- */
-function transformZoominResult(wrapper, requestedVersion) {
-    const leading = wrapper.leading_result;
-    const versioned = selectVersionedEntry(leading, wrapper.follower_result, requestedVersion);
-    const { bundleId } = versioned;
-    const bundleSlug = extractProductSlug(bundleId);
-    const productEntry = bundleSlug !== null
-        ? Object.entries(JAMF_PRODUCTS).find(([id]) => id === bundleSlug)
-        : null;
-    const resultTitle = leading.title !== '' ? leading.title : 'Untitled';
-    const resultProduct = productEntry !== null && productEntry !== undefined
-        ? productEntry[1].name
-        : (leading.publication_title !== '' ? leading.publication_title : 'Jamf');
-    const rawSnippet = stripHtml(leading.snippet !== '' ? leading.snippet : '').slice(0, CONTENT_LIMITS.MAX_SNIPPET_LENGTH);
-    const { labels } = leading;
-    const labelKeys = (labels ?? []).map(l => l.key);
-    const searchResult = {
-        title: resultTitle,
-        url: transformToFrontendUrl(versioned.url !== '' ? versioned.url : ''),
-        snippet: cleanSnippet(rawSnippet, resultTitle, resultProduct),
-        product: resultProduct,
-        version: versioned.version,
-        docType: docTypeFromLabels(labels)
-    };
-    if (leading.score !== undefined) {
-        searchResult.relevance = leading.score;
-    }
-    return {
-        result: searchResult, bundleSlug, bundleId,
-        matchedTopics: matchTopics(searchResult.title, searchResult.snippet),
-        labelKeys,
-        versionMatched: versioned.versionMatched
-    };
-}
-/**
- * Match topic IDs from title + snippet text.
- */
-function matchTopics(title, snippet) {
-    const searchText = `${title} ${snippet}`.toLowerCase();
-    return ALL_TOPIC_IDS
-        .filter(topicId => TOPIC_KEYWORDS_LOWER[topicId].some(kw => searchText.includes(kw)));
-}
-const BUNDLE_ID_REGEX = /\/bundle\/([^/]+)\//;
-/**
- * Convert a flat SearchResult (e.g. from a SearchProvider) into a
- * SearchResultWithMeta so it can flow through the core post-processing pipeline.
- */
-function toSearchResultWithMeta(result) {
-    const bundleMatch = BUNDLE_ID_REGEX.exec(result.url);
-    const bundleId = bundleMatch?.[1] ?? '';
-    const bundleSlug = bundleId !== '' ? extractProductSlug(bundleId) : null;
-    const labelKeys = result.docType !== undefined
-        ? [DOC_TYPE_LABEL_MAP[result.docType]]
-        : [];
-    return {
-        result,
-        bundleSlug,
-        bundleId,
-        matchedTopics: matchTopics(result.title, result.snippet),
-        labelKeys,
-        versionMatched: true,
-    };
-}
-/**
- * Build active filters from search params for progressive relaxation
- */
-function buildActiveFilters(params) {
-    const activeFilters = [];
-    if (params.product !== undefined) {
-        activeFilters.push({
-            name: 'product',
-            value: params.product,
-            apply: (results) => results.filter(r => r.bundleSlug === params.product)
-        });
-    }
-    if (params.topic !== undefined) {
-        const topicFilter = params.topic;
-        activeFilters.push({
-            name: 'topic',
-            value: topicFilter,
-            apply: (results) => results.filter(r => r.matchedTopics.includes(topicFilter))
-        });
-    }
-    if (params.docType !== undefined) {
-        const docTypeFilter = params.docType;
-        const targetLabelKey = DOC_TYPE_LABEL_MAP[docTypeFilter];
-        if (targetLabelKey !== undefined) {
-            activeFilters.push({
-                name: 'docType',
-                value: docTypeFilter,
-                apply: (results) => results.filter(r => {
-                    if (r.labelKeys.length === 0) {
-                        return true;
-                    }
-                    return r.labelKeys.includes(targetLabelKey);
-                })
-            });
-        }
-    }
-    return activeFilters;
-}
-/**
- * Truncate search results to fit within token budget.
- * Returns the truncated results and info about omitted items.
- */
-function truncateSearchResults(paginatedResults, maxTokens) {
-    const tokenCount = estimateTokens(paginatedResults.map(r => `${r.title}\n${r.snippet}\n${r.url}`).join('\n\n'));
-    if (tokenCount <= maxTokens) {
-        return { finalResults: paginatedResults, truncated: false };
-    }
-    let runningTokens = 0;
-    const finalResults = [];
-    for (const result of paginatedResults) {
-        const resultTokens = estimateTokens(`${result.title}\n${result.snippet}\n${result.url}`);
-        if (runningTokens + resultTokens > maxTokens) {
-            break;
-        }
-        finalResults.push(result);
-        runningTokens += resultTokens;
-    }
-    const omittedResults = paginatedResults.slice(finalResults.length);
-    const truncatedContent = {
-        omittedCount: omittedResults.length,
-        omittedItems: omittedResults.map(r => ({
-            title: r.title,
-            estimatedTokens: estimateTokens(`${r.title}\n${r.snippet}\n${r.url}`)
-        }))
-    };
-    return { finalResults, truncated: true, truncatedContent };
-}
-/**
- * Fetch search results from API or cache
- */
-async function fetchSearchResults(params, locale, cacheKey, cache, fetchJson, log) {
-    const cached = await cache.get(cacheKey);
-    if (cached !== null) {
-        return cached;
-    }
-    const hasClientFilter = params.docType !== undefined || (params.version !== undefined && params.version !== 'current');
-    const fetchLimit = hasClientFilter
-        ? Math.min(CONTENT_LIMITS.MAX_SEARCH_RESULTS * CONTENT_LIMITS.FILTER_OVERFETCH_MULTIPLIER, CONTENT_LIMITS.FILTER_OVERFETCH_CAP)
-        : CONTENT_LIMITS.MAX_SEARCH_RESULTS;
-    const apiUrl = new URL(`${DOCS_API_URL}/api/search`);
-    apiUrl.searchParams.set('q', params.query);
-    apiUrl.searchParams.set('rpp', fetchLimit.toString());
-    if (locale !== DEFAULT_LOCALE) {
-        apiUrl.searchParams.set('lang', locale);
-    }
-    if (params.product !== undefined) {
-        const productDef = JAMF_PRODUCTS[params.product];
-        apiUrl.searchParams.set('label', productDef.searchLabel);
-    }
-    log.debug(`Query: "${params.query}", Product: ${params.product ?? 'all'}, Topic: ${params.topic ?? 'all'}, Locale: ${locale}, URL: ${apiUrl.toString()}`);
-    const response = await fetchJson(apiUrl.toString(), locale);
-    const results = response.Results
-        .filter((wrapper) => wrapper.leading_result !== null && wrapper.leading_result !== undefined)
-        .filter(wrapper => {
-        const resultUrl = wrapper.leading_result.url;
-        if (resultUrl !== '' && !isAllowedHostname(resultUrl)) {
-            log.warning(`Skipping result with unexpected hostname: ${resultUrl}`);
-            return false;
-        }
-        return true;
-    })
-        .map(wrapper => transformZoominResult(wrapper, params.version));
-    await cache.set(cacheKey, results);
-    return results;
-}
-/**
- * Normalize a title for deduplication: lowercase, strip special chars, collapse whitespace.
- */
-function normalizeTitle(title) {
-    return title
-        .toLowerCase()
-        .replace(/[^a-z0-9\s]/g, '')
-        .replace(/\s+/g, ' ')
-        .trim();
-}
-/**
- * Deduplicate search results by product + page slug, keeping the latest version.
- * Falls back to title-based dedup when page slugs differ but titles match.
- * Only applied when no version filter is specified.
- */
-function deduplicateByLatestVersion(results) {
-    const seen = new Map();
-    const titleIndex = new Map(); // titleKey → slug key
-    for (const item of results) {
-        const pageMatch = /\/page\/([^?#]+)/.exec(item.result.url);
-        if (pageMatch === null) {
-            seen.set(`_no_page_${seen.size}`, item);
-            continue;
-        }
-        const pageSlug = pageMatch[1] ?? '';
-        const productSlug = item.bundleSlug ?? '_unknown';
-        const slugKey = `${productSlug}:${pageSlug}`;
-        const titleKey = `${productSlug}:title:${normalizeTitle(item.result.title)}`;
-        // Check slug key first, then fall back to title key
-        const existingSlugKey = seen.has(slugKey) ? slugKey : null;
-        const existingTitleRef = titleIndex.get(titleKey);
-        const matchKey = existingSlugKey ?? (existingTitleRef !== undefined && seen.has(existingTitleRef) ? existingTitleRef : null);
-        if (matchKey === null) {
-            seen.set(slugKey, item);
-            titleIndex.set(titleKey, slugKey);
-        }
-        else {
-            const existing = seen.get(matchKey);
-            const existingVersion = existing.result.version ?? '';
-            const newVersion = item.result.version ?? '';
-            if (compareVersions(newVersion, existingVersion) > 0) {
-                seen.set(matchKey, item);
-            }
-        }
-    }
-    const kept = new Set(seen.values());
-    return results.filter(r => kept.has(r));
-}
-/**
- * Resolve search results from provider or Zoomin API fallback.
- */
-async function resolveSearchResults(ctx, params, log) {
-    if (ctx.searchProvider) {
-        const provided = await ctx.searchProvider.search(params);
-        if (provided !== null) {
-            return provided.map(toSearchResultWithMeta);
-        }
-    }
-    const fetchJson = makeFetchJson(ctx.config.request);
-    const locale = params.language ?? DEFAULT_LOCALE;
-    const cacheKey = `${locale}:search:${JSON.stringify({
-        query: params.query, product: params.product, topic: params.topic,
-        docType: params.docType, version: params.version, limit: params.limit,
-        maxTokens: params.maxTokens, page: 1
-    })}`;
-    return fetchSearchResults(params, locale, cacheKey, ctx.cache, fetchJson, log);
-}
-/**
- * Search Jamf documentation using Zoomin Search API
- */
-export async function searchDocumentation(ctx, params) {
-    const log = ctx.logger.createLogger('scraper');
-    const page = params.page ?? PAGINATION_CONFIG.DEFAULT_PAGE;
-    const pageSize = params.limit ?? CONTENT_LIMITS.DEFAULT_SEARCH_RESULTS;
-    const maxTokens = params.maxTokens ?? TOKEN_CONFIG.DEFAULT_MAX_TOKENS;
-    let allResults;
-    try {
-        allResults = await resolveSearchResults(ctx, params, log);
-    }
-    catch (error) {
-        log.error(`Search error: ${String(error)}`);
-        if (error instanceof JamfDocsError) {
-            throw error;
-        }
-        allResults = [];
-    }
-    // Build and apply filters with progressive relaxation
-    const activeFilters = buildActiveFilters(params);
-    const { filtered: filteredResults, relaxation: filterRelaxation } = applyFiltersWithFallback(allResults, activeFilters);
-    // Deduplicate cross-version results when no version filter
-    const dedupedResults = params.version === undefined || params.version === ''
-        ? deduplicateByLatestVersion(filteredResults)
-        : filteredResults;
-    // Calculate pagination
-    const paginationInfo = calculatePagination(dedupedResults.length, page, pageSize);
-    // Get paginated results
-    const paginatedResults = dedupedResults
-        .slice(paginationInfo.startIndex, paginationInfo.endIndex)
-        .map(r => r.result);
-    // Truncate results to fit within token budget
-    const { finalResults, truncated, truncatedContent } = truncateSearchResults(paginatedResults, maxTokens);
-    const finalTokenCount = estimateTokens(finalResults.map(r => `${r.title}\n${r.snippet}\n${r.url}`).join('\n\n'));
-    // Generate versionNote only when version was requested but not found in some results
-    const requestedVersion = params.version;
-    const hasVersionMismatch = requestedVersion !== undefined
-        && requestedVersion !== 'current'
-        && requestedVersion !== ''
-        && dedupedResults.some(r => !r.versionMatched);
-    const versionNote = hasVersionMismatch
-        ? `Version "${requestedVersion}" was not available for some results. Showing the latest version instead.`
-        : undefined;
-    return {
-        results: finalResults,
-        pagination: {
-            page: paginationInfo.page,
-            pageSize: paginationInfo.pageSize,
-            totalPages: paginationInfo.totalPages,
-            totalItems: dedupedResults.length,
-            hasNext: paginationInfo.hasNext,
-            hasPrev: paginationInfo.hasPrev
-        },
-        tokenInfo: {
-            tokenCount: finalTokenCount,
-            truncated,
-            maxTokens
-        },
-        ...(filterRelaxation !== undefined ? { filterRelaxation } : {}),
-        ...(versionNote !== undefined ? { versionNote } : {}),
-        ...(truncatedContent !== undefined ? { truncatedContent } : {})
-    };
-}
-/**
- * Fetch and parse a documentation article
- * Uses backend URL (learn-be.jamf.com) for pre-rendered content
- */
-export async function fetchArticle(ctx, url, options = {}) {
-    if (ctx.articleProvider) {
-        const provided = await ctx.articleProvider.getArticle(url, options);
-        if (provided !== null)
-            return provided;
-    }
-    const fetchHtmlInner = makeFetchHtml(ctx.config.request);
-    const maxTokens = options.maxTokens ?? TOKEN_CONFIG.DEFAULT_MAX_TOKENS;
-    // Infer locale from URL if not explicitly provided (e.g., /ja-JP/bundle/...)
-    const locale = options.locale ?? extractLocaleFromUrl(url);
-    // Strip locale prefix before transforming — backend doesn't accept /en-US/ etc. in path
-    const cleanUrl = stripLocalePrefix(url);
-    const displayUrl = transformToFrontendUrl(cleanUrl);
-    const articleFetchUrl = transformToBackendUrl(cleanUrl);
-    const cacheKey = `${locale}:article:${displayUrl}`;
-    // Check cache for raw article (without section/token processing)
-    let rawArticle = await ctx.cache.get(cacheKey);
-    if (rawArticle === null) {
-        // Fetch HTML from backend URL (pre-rendered content, not SPA shell)
-        const html = await fetchHtmlInner(articleFetchUrl, locale);
-        const $ = cheerio.load(html);
-        // Clean content
-        cleanHtml($);
-        // Extract content
-        const contentHtml = $(SELECTORS.CONTENT).html() ?? '';
-        const extractedTitle = $(SELECTORS.TITLE).first().text().trim();
-        const title = extractedTitle !== '' ? extractedTitle : 'Untitled';
-        // Convert to Markdown and strip Turndown anchor artifacts from headings
-        // Turndown converts HTML anchors like <a href="#">...</a> inside headings to [](#)
-        const content = turndown.turndown(contentHtml)
-            .replace(/^(#{1,6}\s+)\[([^\]]*)\]\(#[^)]*\)/gm, '$1$2');
-        // Extract breadcrumb
-        const breadcrumb = $(SELECTORS.BREADCRUMB)
-            .map((_, el) => $(el).text().trim())
-            .get()
-            .filter(Boolean);
-        // Extract related articles
-        const relatedArticles = options.includeRelated === true
-            ? $(SELECTORS.RELATED).map((_, el) => {
-                const rawHref = $(el).attr('href') ?? '';
-                // Skip empty or hash-only URLs (same-page anchors)
-                if (rawHref === '' || rawHref.startsWith('#')) {
-                    return { title: '', url: '' };
-                }
-                // Resolve relative URLs against the article's fetch URL
-                let resolvedUrl;
-                try {
-                    resolvedUrl = new URL(rawHref, articleFetchUrl).toString();
-                }
-                catch {
-                    resolvedUrl = rawHref;
-                }
-                return {
-                    title: $(el).text().trim(),
-                    url: transformToFrontendUrl(resolvedUrl)
-                };
-            }).get().filter(r => r.title !== '' && r.url !== '')
-            : undefined;
-        // Extract product info from URL
-        const { product, version } = extractProductInfo(displayUrl);
-        rawArticle = {
-            title,
-            content,
-            url: displayUrl,
-            product,
-            version,
-            breadcrumb: breadcrumb.length > 0 ? breadcrumb : undefined,
-            relatedArticles: relatedArticles !== undefined && relatedArticles.length > 0 ? relatedArticles : undefined
-        };
-        // Cache raw article
-        await ctx.cache.set(cacheKey, rawArticle);
-    }
-    // At this point rawArticle is guaranteed to be non-null
-    const article = rawArticle;
-    // Extract all sections
-    const allSections = extractSections(article.content);
-    // Handle summaryOnly mode
-    if (options.summaryOnly === true) {
-        const summaryResult = extractSummary(article.content, article.title, maxTokens);
-        // Build summary content
-        let summaryContent = `## Summary\n\n${summaryResult.summary}\n\n`;
-        summaryContent += `## Article Outline (${summaryResult.outline.length} sections)\n\n`;
-        for (const section of summaryResult.outline) {
-            const indent = '  '.repeat(Math.max(0, section.level - 1));
-            summaryContent += `${indent}- ${section.title} (~${section.tokenCount} tokens)\n`;
-        }
-        summaryContent += `\n*Estimated read time: ${summaryResult.estimatedReadTime} min | Total: ${summaryResult.totalTokens.toLocaleString()} tokens*\n`;
-        return {
-            ...article,
-            content: summaryContent,
-            tokenInfo: summaryResult.tokenInfo,
-            sections: allSections
-        };
-    }
-    // Handle section extraction if requested
-    let processedContent;
-    let tokenInfo;
-    if (options.section !== undefined && options.section !== '') {
-        // Extract specific section
-        const sectionResult = extractSection(article.content, options.section, maxTokens);
-        processedContent = sectionResult.content;
-        tokenInfo = sectionResult.tokenInfo;
-        if (sectionResult.section === null) {
-            // Section not found, return error info
-            processedContent = `*Section "${options.section}" not found.*\n\n**Available sections:**\n${allSections.map(s => `- ${s.title}`).join('\n')}`;
-            tokenInfo = createTokenInfo(processedContent, maxTokens);
-        }
-    }
-    else {
-        // Apply token limit with smart truncation
-        const truncateResult = truncateToTokenLimit(article.content, maxTokens);
-        processedContent = truncateResult.content;
-        tokenInfo = truncateResult.tokenInfo;
-    }
-    return {
-        ...article,
-        content: processedContent,
-        tokenInfo,
-        sections: allSections
-    };
-}
-/**
- * Discover the latest bundle version for a product via search API
- */
-async function discoverLatestBundleId(product, fetchJson, log) {
-    const productInfo = JAMF_PRODUCTS[product];
-    const baseBundleId = productInfo.bundleId;
-    try {
-        // Search for any document from this product to discover the latest version
-        const apiUrl = `${DOCS_API_URL}/api/search?q=${encodeURIComponent(productInfo.name)}&rpp=10`;
-        const response = await fetchJson(apiUrl);
-        // Find a result with a matching bundle prefix
-        for (const wrapper of response.Results) {
-            if (wrapper.leading_result === null || wrapper.leading_result === undefined) {
-                continue;
-            }
-            const bundleId = wrapper.leading_result.bundle_id;
-            if (bundleId.startsWith(baseBundleId)) {
-                return validateBundleId(bundleId);
-            }
-        }
-    }
-    catch (error) {
-        log.error(`Error discovering bundle version for ${product}: ${String(error)}`);
-    }
-    return null;
-}
-/**
- * Parse TOC HTML from Zoomin backend
- */
-function parseTocHtml(html) {
-    const $ = cheerio.load(html);
-    const toc = [];
-    // Parse the nested list structure
-    $('ul.list-links > li.toc').each((_, el) => {
-        const entry = parseTocEntry($, el);
-        if (entry !== null) {
-            toc.push(entry);
-        }
-    });
-    return toc;
-}
-function parseTocEntry($, el) {
-    const $el = $(el);
-    const $link = $el.children('.inner').children('a').first();
-    const title = $link.text().trim();
-    let url = $link.attr('href') ?? '';
-    if (title === '' || url === '') {
-        return null;
-    }
-    // Transform to frontend URL for display
-    url = transformToFrontendUrl(url);
-    const entry = { title, url };
-    // Check for children
-    const $children = $el.children('ul.list-links');
-    if ($children.length > 0) {
-        const children = [];
-        $children.children('li.toc').each((_, childEl) => {
-            const childEntry = parseTocEntry($, childEl);
-            if (childEntry !== null) {
-                children.push(childEntry);
-            }
-        });
-        if (children.length > 0) {
-            entry.children = children;
-        }
-    }
-    return entry;
-}
-/**
- * Convert TOC entry to string for token counting
- */
-function tocEntryToString(entry, depth = 0) {
-    const indent = '  '.repeat(depth);
-    const childrenStr = entry.children?.map(c => tocEntryToString(c, depth + 1)).join('') ?? '';
-    return `${indent}- ${entry.title}\n${childrenStr}`;
-}
-/**
- * Count total TOC entries including children
- */
-function countTocEntries(entries) {
-    return entries.reduce((count, entry) => count + 1 + (entry.children !== undefined ? countTocEntries(entry.children) : 0), 0);
-}
-/**
- * Build a flat TOC from search results when backend TOC endpoint returns empty.
- * Searches for all articles under the given product and constructs TocEntry list.
- */
-async function buildTocFromSearch(product, locale, fetchJson, log) {
-    try {
-        const apiUrl = new URL(`${DOCS_API_URL}/api/search`);
-        apiUrl.searchParams.set('q', JAMF_PRODUCTS[product].name);
-        apiUrl.searchParams.set('rpp', CONTENT_LIMITS.MAX_SEARCH_RESULTS.toString());
-        if (locale !== DEFAULT_LOCALE) {
-            apiUrl.searchParams.set('lang', locale);
-        }
-        const response = await fetchJson(apiUrl.toString(), locale);
-        const seen = new Set();
-        const entries = [];
-        for (const wrapper of response.Results) {
-            const leading = wrapper.leading_result;
-            if (leading === null || leading === undefined) {
-                continue;
-            }
-            const bundleSlug = extractProductSlug(leading.bundle_id);
-            if (bundleSlug !== product) {
-                continue;
-            }
-            const url = transformToFrontendUrl(leading.url);
-            if (seen.has(url)) {
-                continue;
-            }
-            seen.add(url);
-            const title = leading.title !== '' ? leading.title : 'Untitled';
-            entries.push({ title, url });
-        }
-        if (entries.length > 0) {
-            log.info(`Built fallback TOC with ${entries.length} entries from search for ${product}`);
-        }
-        return entries;
-    }
-    catch (error) {
-        log.error(`Failed to build TOC from search for ${product}: ${String(error)}`);
-        return [];
-    }
-}
-/**
- * Fetch table of contents for a product
- * Uses backend TOC endpoint (learn-be.jamf.com/bundle/{bundleId}/toc)
- */
-export async function fetchTableOfContents(ctx, product, version = 'current', options = {}) {
-    if (ctx.tocProvider) {
-        const provided = await ctx.tocProvider.getTableOfContents(product, version, options);
-        if (provided !== null)
-            return provided;
-    }
-    const log = ctx.logger.createLogger('scraper');
-    const fetchJson = makeFetchJson(ctx.config.request);
-    const page = options.page ?? PAGINATION_CONFIG.DEFAULT_PAGE;
-    const maxTokens = options.maxTokens ?? TOKEN_CONFIG.DEFAULT_MAX_TOKENS;
-    const locale = options.locale ?? DEFAULT_LOCALE;
-    const cacheKey = `${locale}:toc:${product}:${version}`;
-    // Check cache
-    let allToc = await ctx.cache.get(cacheKey);
-    if (allToc === null) {
-        // Get bundle ID for the specified version (or latest if current)
-        let bundleId = await getBundleIdForVersion(ctx, product, version);
-        // Fallback to discovery if metadata service fails
-        bundleId ??= await discoverLatestBundleId(product, fetchJson, log);
-        if (bundleId === null || bundleId === '') {
-            throw new JamfDocsError(`Could not find bundle for ${product} version ${version}`, JamfDocsErrorCode.NOT_FOUND);
-        }
-        // Fetch TOC from backend (may 404 for some products)
-        const tocUrl = `${DOCS_API_URL}/bundle/${bundleId}/toc`;
-        log.debug(`Fetching TOC from: ${tocUrl}, Locale: ${locale}`);
-        allToc = [];
-        try {
-            const tocJson = await fetchJson(tocUrl, locale);
-            // Parse TOC
-            for (const [, html] of Object.entries(tocJson)) {
-                if (typeof html === 'string' && html.includes('<ul')) {
-                    const entries = parseTocHtml(html);
-                    allToc.push(...entries);
-                }
-            }
-        }
-        catch (error) {
-            // TOC endpoint may 404 or fail for some products — fall through to search fallback
-            log.error(`Backend TOC request failed for ${product}: ${error instanceof Error ? error.message : String(error)}`);
-        }
-        // Fallback: build TOC from search results when backend returns empty or fails
-        if (allToc.length === 0) {
-            log.info(`No TOC entries for ${product}, falling back to search-based discovery`);
-            allToc = await buildTocFromSearch(product, locale, fetchJson, log);
-        }
-        // Cache result
-        await ctx.cache.set(cacheKey, allToc);
-    }
-    // Count total entries
-    const totalItems = countTocEntries(allToc);
-    // For TOC, we paginate at the top-level entries
-    const topLevelCount = allToc.length;
-    const pageSize = PAGINATION_CONFIG.DEFAULT_PAGE_SIZE;
-    const paginationInfo = calculatePagination(topLevelCount, page, pageSize);
-    // Get paginated top-level entries (with their children)
-    const paginatedToc = allToc.slice(paginationInfo.startIndex, paginationInfo.endIndex);
-    // Calculate tokens for paginated TOC
-    const tocText = paginatedToc.map(e => tocEntryToString(e)).join('');
-    let tokenCount = estimateTokens(tocText);
-    let truncated = false;
-    let finalToc = paginatedToc;
-    // Truncate if over token limit
-    if (tokenCount > maxTokens) {
-        truncated = true;
-        finalToc = [];
-        let runningTokens = 0;
-        for (const entry of paginatedToc) {
-            const entryText = tocEntryToString(entry);
-            const entryTokens = estimateTokens(entryText);
-            if (runningTokens + entryTokens > maxTokens) {
-                break;
-            }
-            finalToc.push(entry);
-            runningTokens += entryTokens;
-        }
-        tokenCount = runningTokens;
-    }
-    return {
-        toc: finalToc,
-        pagination: {
-            page: paginationInfo.page,
-            pageSize: paginationInfo.pageSize,
-            totalPages: paginationInfo.totalPages,
-            totalItems,
-            hasNext: paginationInfo.hasNext,
-            hasPrev: paginationInfo.hasPrev
-        },
-        tokenInfo: {
-            tokenCount,
-            truncated,
-            maxTokens
-        }
-    };
-}
-//# sourceMappingURL=scraper.js.map