@pinkpixel/sugarstitch 1.0.0

package/dist/scraper.js

@@ -0,0 +1,719 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PROFILES_FILE = void 0;
+exports.sanitizeFilename = sanitizeFilename;
+exports.normalizeUrl = normalizeUrl;
+exports.dedupeStrings = dedupeStrings;
+exports.getSelectorPresets = getSelectorPresets;
+exports.isSelectorPresetId = isSelectorPresetId;
+exports.getSelectorPreset = getSelectorPreset;
+exports.loadSiteProfiles = loadSiteProfiles;
+exports.sanitizeSelectorOverrides = sanitizeSelectorOverrides;
+exports.loadExistingData = loadExistingData;
+exports.previewPattern = previewPattern;
+exports.scrapeUrls = scrapeUrls;
+const axios_1 = __importDefault(require("axios"));
+const cheerio = __importStar(require("cheerio"));
+const fs = __importStar(require("fs/promises"));
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const promises_1 = require("stream/promises");
+const REQUEST_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+};
+const REQUEST_TIMEOUT_MS = 15000;
+const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.avif']);
+const DEFAULT_DELAY_MS = 2000;
+const DEFAULT_CRAWL_MAX_DEPTH = 2;
+const DEFAULT_CRAWL_MAX_URLS = 100;
+const DEFAULT_CRAWL_MAX_PAGINATION_PAGES = 20;
+exports.DEFAULT_PROFILES_FILE = 'sugarstitch.profiles.json';
+const ARTICLE_TEXT_SELECTORS = ['.entry-content', '.wp-block-post-content', '.post-content', '.article-body', 'article', 'main'];
+const SELECTOR_PRESETS = {
+    generic: {
+        id: 'generic',
+        label: 'Generic / Custom',
+        description: 'A broad default for custom sites and general article-style pattern pages.',
+        titleSelectors: ['h1'],
+        descriptionSelectors: ['.entry-content p', '.post-content p', '.article-body p', 'main p'],
+        materialsSelectors: ['.materials-list li', '.pattern-materials li', '.supply-list li'],
+        instructionsSelectors: ['.instructions-step', '.instruction-step', '.pattern-steps li', 'ol li'],
+        imageSelectors: ['.entry-content img', '.post-content img', '.article-body img', 'main img']
+    },
+    wordpress: {
+        id: 'wordpress',
+        label: 'WordPress Article',
+        description: 'Best for blog-style posts using common WordPress article wrappers.',
+        titleSelectors: ['h1.entry-title', 'article h1', 'h1'],
+        descriptionSelectors: ['.entry-content p', '.wp-block-post-content p', '.post-content p'],
+        materialsSelectors: ['.entry-content ul li', '.wp-block-post-content ul li', '.materials-list li'],
+        instructionsSelectors: ['.entry-content ol li', '.wp-block-post-content ol li', '.instructions-step'],
+        imageSelectors: ['.entry-content img', '.wp-block-post-content img', '.post-content img']
+    },
+    woocommerce: {
+        id: 'woocommerce',
+        label: 'WooCommerce Product',
+        description: 'Best for WooCommerce product pages with product galleries and description areas.',
+        titleSelectors: ['.product_title', '.entry-summary h1', 'h1.product_title', 'h1'],
+        descriptionSelectors: [
+            '.woocommerce-product-details__short-description p',
+            '.woocommerce-Tabs-panel--description p',
+            '.entry-summary p'
+        ],
+        materialsSelectors: [
+            '.woocommerce-Tabs-panel--description ul li',
+            '.woocommerce-Tabs-panel ul li',
+            '.product_meta + div ul li'
+        ],
+        instructionsSelectors: [
+            '.woocommerce-Tabs-panel--description ol li',
+            '.woocommerce-Tabs-panel ol li',
+            '.instruction-step'
+        ],
+        imageSelectors: ['.woocommerce-product-gallery img', '.images img', '.woocommerce-product-gallery__image img']
+    }
+};
+function defaultLogger(message) {
+    console.log(message);
+}
+function sanitizeFilename(name) {
+    const sanitized = name
+        .replace(/[^a-z0-9]/gi, '_')
+        .replace(/_+/g, '_')
+        .replace(/^_+|_+$/g, '')
+        .toLowerCase();
+    return sanitized || 'untitled_pattern';
+}
+function normalizeUrl(input) {
+    try {
+        const url = new URL(input.trim());
+        return ['http:', 'https:'].includes(url.protocol) ? url.href : null;
+    }
+    catch {
+        return null;
+    }
+}
+function dedupeStrings(items) {
+    return [...new Set(items)];
+}
+function getSelectorPresets() {
+    return Object.values(SELECTOR_PRESETS);
+}
+function isSelectorPresetId(value) {
+    return value in SELECTOR_PRESETS;
+}
+function getSelectorPreset(presetId = 'generic') {
+    return SELECTOR_PRESETS[presetId];
+}
+async function loadSiteProfiles(profilesPath = path.resolve(process.cwd(), exports.DEFAULT_PROFILES_FILE)) {
+    try {
+        const fileContent = await fs.readFile(profilesPath, 'utf-8');
+        const parsed = JSON.parse(fileContent);
+        if (!Array.isArray(parsed.profiles)) {
+            throw new Error('Profiles file must contain a "profiles" array.');
+        }
+        return parsed.profiles.map(profile => {
+            if (!profile?.id || !profile?.label) {
+                throw new Error('Each profile must include both "id" and "label".');
+            }
+            if (profile.preset && !isSelectorPresetId(profile.preset)) {
+                throw new Error(`Profile "${profile.id}" references unknown preset "${profile.preset}".`);
+            }
+            return {
+                id: profile.id,
+                label: profile.label,
+                description: profile.description,
+                preset: profile.preset ?? 'generic',
+                selectorOverrides: sanitizeSelectorOverrides(profile.selectorOverrides)
+            };
+        });
+    }
+    catch (error) {
+        if (error?.code === 'ENOENT') {
+            return [];
+        }
+        if (error instanceof SyntaxError) {
+            throw new Error(`Profiles file contains invalid JSON: ${error.message}`);
+        }
+        throw error;
+    }
+}
+function sanitizeSelectorOverrides(overrides) {
+    if (!overrides)
+        return undefined;
+    const cleaned = {};
+    for (const [key, value] of Object.entries(overrides)) {
+        const trimmed = value?.trim();
+        if (trimmed) {
+            cleaned[key] = trimmed;
+        }
+    }
+    return Object.keys(cleaned).length > 0 ? cleaned : undefined;
+}
+function resolveSelectors(preset, overrides) {
+    const cleanedOverrides = sanitizeSelectorOverrides(overrides);
+    return {
+        titleSelectors: cleanedOverrides?.titleSelector ? [cleanedOverrides.titleSelector, ...preset.titleSelectors] : preset.titleSelectors,
+        descriptionSelectors: cleanedOverrides?.descriptionSelector ? [cleanedOverrides.descriptionSelector, ...preset.descriptionSelectors] : preset.descriptionSelectors,
+        materialsSelectors: cleanedOverrides?.materialsSelector ? [cleanedOverrides.materialsSelector, ...preset.materialsSelectors] : preset.materialsSelectors,
+        instructionsSelectors: cleanedOverrides?.instructionsSelector ? [cleanedOverrides.instructionsSelector, ...preset.instructionsSelectors] : preset.instructionsSelectors,
+        imageSelectors: cleanedOverrides?.imageSelector ? [cleanedOverrides.imageSelector, ...preset.imageSelectors] : preset.imageSelectors
+    };
+}
+async function resolveStrategy(options) {
+    const profilesPath = options.profilesPath ?? path.resolve(process.cwd(), exports.DEFAULT_PROFILES_FILE);
+    const availableProfiles = await loadSiteProfiles(profilesPath);
+    if (options.profileId) {
+        const profile = availableProfiles.find(item => item.id === options.profileId);
+        if (!profile) {
+            throw new Error(`Unknown profile "${options.profileId}" in ${profilesPath}.`);
+        }
+        const profilePresetId = profile.preset ?? 'generic';
+        return {
+            presetId: profilePresetId,
+            preset: getSelectorPreset(profilePresetId),
+            selectorOverrides: sanitizeSelectorOverrides({
+                ...profile.selectorOverrides,
+                ...options.selectorOverrides
+            }),
+            profileId: profile.id,
+            profileLabel: profile.label
+        };
+    }
+    const presetId = options.preset ?? 'generic';
+    return {
+        presetId,
+        preset: getSelectorPreset(presetId),
+        selectorOverrides: sanitizeSelectorOverrides(options.selectorOverrides)
+    };
+}
+async function loadExistingData(outputPath) {
+    try {
+        const fileContent = await fs.readFile(outputPath, 'utf-8');
+        const parsed = JSON.parse(fileContent);
+        if (!Array.isArray(parsed)) {
+            throw new Error('Output file must contain a JSON array.');
+        }
+        return parsed;
+    }
+    catch (error) {
+        if (error?.code === 'ENOENT') {
+            return [];
+        }
+        if (error instanceof SyntaxError) {
+            throw new Error(`Output file contains invalid JSON: ${error.message}`);
+        }
+        throw error;
+    }
+}
+function getImageExtension(fileUrl) {
+    try {
+        const pathname = new URL(fileUrl).pathname;
+        const ext = path.extname(pathname).toLowerCase();
+        return IMAGE_EXTENSIONS.has(ext) ? ext : '.jpg';
+    }
+    catch {
+        return '.jpg';
+    }
+}
+async function downloadFile(url, dest, logger) {
+    try {
+        const response = await axios_1.default.get(url, {
+            responseType: 'stream',
+            headers: REQUEST_HEADERS,
+            timeout: REQUEST_TIMEOUT_MS
+        });
+        await (0, promises_1.pipeline)(response.data, (0, fs_1.createWriteStream)(dest));
+        return true;
+    }
+    catch (error) {
+        logger(`\n❌ Shit, couldn't download file ${url}: ${error.message}`);
+        return false;
+    }
+}
+async function fetchHtml(url) {
+    const { data } = await axios_1.default.get(url, {
+        headers: REQUEST_HEADERS,
+        timeout: REQUEST_TIMEOUT_MS
+    });
+    return data;
+}
+function shouldSkipHref(href) {
+    const normalized = href.trim().toLowerCase();
+    return (normalized.length === 0 ||
+        normalized.startsWith('#') ||
+        normalized.startsWith('mailto:') ||
+        normalized.startsWith('tel:') ||
+        normalized.startsWith('javascript:'));
+}
+function normalizeLanguageValue(value) {
+    const normalized = value?.trim().toLowerCase();
+    return normalized ? normalized : undefined;
+}
+function detectLanguageFromUrl(url) {
+    const langParam = url.searchParams.get('lang')?.trim().toLowerCase();
+    if (langParam)
+        return langParam;
+    const pathname = url.pathname.toLowerCase();
+    if (pathname.includes('french'))
+        return 'french';
+    if (pathname.includes('portuguese'))
+        return 'portuguese';
+    if (pathname.includes('english'))
+        return 'english';
+    return undefined;
+}
+function matchesLanguage(url, language) {
+    const normalizedLanguage = normalizeLanguageValue(language);
+    if (!normalizedLanguage)
+        return true;
+    const detectedLanguage = detectLanguageFromUrl(url);
+    if (!detectedLanguage)
+        return true;
+    return detectedLanguage === normalizedLanguage;
+}
+function looksLikeHtmlPage(url) {
+    const pathname = url.pathname.toLowerCase();
+    if (pathname.endsWith('.pdf'))
+        return false;
+    const ext = path.extname(pathname);
+    return ext === '' || ['.html', '.htm', '.php', '.asp', '.aspx'].includes(ext);
+}
+function detectMaxPaginationPages($) {
+    const dataPages = $('.jet-listing-grid__items[data-pages]').first().attr('data-pages');
+    const parsedDataPages = dataPages ? Number.parseInt(dataPages, 10) : Number.NaN;
+    if (Number.isFinite(parsedDataPages) && parsedDataPages > 1) {
+        return parsedDataPages;
+    }
+    let maxPage = 1;
+    $('a[href*="/page/"]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href)
+            return;
+        const match = href.match(/\/page\/(\d+)\//i);
+        if (!match)
+            return;
+        const page = Number.parseInt(match[1], 10);
+        if (Number.isFinite(page)) {
+            maxPage = Math.max(maxPage, page);
+        }
+    });
+    return maxPage > 1 ? maxPage : undefined;
+}
+function buildPaginatedUrl(seedUrl, pageNumber) {
+    const parsed = new URL(seedUrl);
+    const segments = parsed.pathname.replace(/\/+$/, '').split('/').filter(Boolean);
+    if (segments[segments.length - 2] === 'page') {
+        segments[segments.length - 1] = String(pageNumber);
+    }
+    else {
+        segments.push('page', String(pageNumber));
+    }
+    parsed.pathname = `/${segments.join('/')}/`;
+    return parsed.href;
+}
+async function expandPaginatedSeedUrls(startUrls, logger, crawl) {
+    if (!crawl?.enabled || !crawl.paginate) {
+        return startUrls;
+    }
+    const maxPaginationPages = Math.max(1, crawl.maxPaginationPages ?? DEFAULT_CRAWL_MAX_PAGINATION_PAGES);
+    const expanded = new Set(startUrls);
+    for (const startUrl of startUrls) {
+        let html;
+        try {
+            html = await fetchHtml(startUrl);
+        }
+        catch (error) {
+            logger(`⚠️ Couldn't inspect pagination for ${startUrl}: ${error.message}`);
+            continue;
+        }
+        const $ = cheerio.load(html);
+        const detectedMaxPages = detectMaxPaginationPages($);
+        if (!detectedMaxPages || detectedMaxPages <= 1) {
+            continue;
+        }
+        const pageLimit = Math.min(detectedMaxPages, maxPaginationPages);
+        logger(`📚 Pagination detected. Adding listing pages 2 through ${pageLimit} for ${startUrl}`);
+        for (let page = 2; page <= pageLimit; page++) {
+            expanded.add(buildPaginatedUrl(startUrl, page));
+        }
+    }
+    return [...expanded];
+}
+async function discoverUrls(startUrls, logger, crawl) {
+    if (!crawl?.enabled) {
+        return startUrls;
+    }
+    const maxDepth = Math.max(0, crawl.maxDepth ?? DEFAULT_CRAWL_MAX_DEPTH);
+    const sameDomainOnly = crawl.sameDomainOnly ?? true;
+    const maxDiscoveredUrls = Math.max(1, crawl.maxDiscoveredUrls ?? DEFAULT_CRAWL_MAX_URLS);
+    const language = normalizeLanguageValue(crawl.language);
+    const linkPattern = crawl.linkPattern?.trim();
+    const patternRegex = linkPattern ? new RegExp(linkPattern, 'i') : null;
+    const queue = [];
+    const visited = new Set();
+    const discovered = new Set();
+    const seedUrls = await expandPaginatedSeedUrls(startUrls, logger, crawl);
+    for (const startUrl of seedUrls) {
+        const parsed = new URL(startUrl);
+        if (!matchesLanguage(parsed, language))
+            continue;
+        queue.push({ url: startUrl, depth: 0, rootHost: parsed.host });
+        discovered.add(startUrl);
+    }
+    logger(`\n🕸️ Crawl mode is on. Exploring up to ${maxDepth} link level(s) deep...`);
+    while (queue.length > 0 && discovered.size < maxDiscoveredUrls) {
+        const current = queue.shift();
+        if (visited.has(current.url))
+            continue;
+        visited.add(current.url);
+        logger(`🔎 Discovering links at depth ${current.depth}: ${current.url}`);
+        let html;
+        try {
+            html = await fetchHtml(current.url);
+        }
+        catch (error) {
+            logger(`⚠️ Couldn't crawl ${current.url}: ${error.message}`);
+            continue;
+        }
+        const $ = cheerio.load(html);
+        $('a').each((_, el) => {
+            if (discovered.size >= maxDiscoveredUrls)
+                return;
+            const href = $(el).attr('href');
+            if (!href || shouldSkipHref(href))
+                return;
+            try {
+                const absolute = new URL(href, current.url);
+                if (!['http:', 'https:'].includes(absolute.protocol))
+                    return;
+                if (sameDomainOnly && absolute.host !== current.rootHost)
+                    return;
+                if (!matchesLanguage(absolute, language))
+                    return;
+                if (!looksLikeHtmlPage(absolute))
+                    return;
+                const normalized = absolute.href;
+                const matchesPattern = !patternRegex || patternRegex.test(normalized) || patternRegex.test($(el).text().trim());
+                if (!matchesPattern)
+                    return;
+                if (discovered.has(normalized))
+                    return;
+                discovered.add(normalized);
+                if (current.depth < maxDepth) {
+                    queue.push({ url: normalized, depth: current.depth + 1, rootHost: current.rootHost });
+                }
+            }
+            catch { }
+        });
+        if (queue.length > 0) {
+            await sleep(500);
+        }
+    }
+    logger(`✨ Discovery finished. Found ${discovered.size} page URL(s) to scrape.`);
+    return [...discovered];
+}
+function firstTextMatch($, selectors, fallback) {
+    for (const selector of selectors) {
+        const text = $(selector).first().text().trim();
+        if (text)
+            return text;
+    }
+    return fallback;
+}
+function collectTextMatches($, selectors) {
+    for (const selector of selectors) {
+        const values = [];
+        $(selector).each((_, el) => {
+            const value = $(el).text().trim();
+            if (value)
+                values.push(value);
+        });
+        if (values.length > 0) {
+            return dedupeStrings(values);
+        }
+    }
+    return [];
+}
+function collectAssetUrls($, selectors, pageUrl) {
+    for (const selector of selectors) {
+        const urls = [];
+        $(selector).each((_, el) => {
+            const src = $(el).attr('src');
+            if (!src)
+                return;
+            try {
+                const absoluteUrl = new URL(src, pageUrl).href;
+                if (!absoluteUrl.includes('logo') && !absoluteUrl.includes('icon')) {
+                    urls.push(absoluteUrl);
+                }
+            }
+            catch { }
+        });
+        if (urls.length > 0) {
+            return dedupeStrings(urls);
+        }
+    }
+    return [];
+}
+function normalizeTextBlock(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}
+function extractPageText($) {
+    for (const selector of ARTICLE_TEXT_SELECTORS) {
+        const container = $(selector).first();
+        if (!container.length)
+            continue;
+        const blocks = [];
+        container.find('h2, h3, h4, p, li').each((_, el) => {
+            const text = normalizeTextBlock($(el).text());
+            if (text) {
+                blocks.push(text);
+            }
+        });
+        const uniqueBlocks = dedupeStrings(blocks);
+        if (uniqueBlocks.length > 0) {
+            return uniqueBlocks.join('\n\n');
+        }
+    }
+    const fallbackText = normalizeTextBlock($('body').text());
+    return fallbackText;
+}
+async function writeTextArtifact(preview, workingDirectory, safeTitle) {
+    const textDir = path.resolve(workingDirectory, 'texts', safeTitle);
+    await fs.mkdir(textDir, { recursive: true });
+    const textFile = path.resolve(textDir, 'pattern.txt');
+    const outputText = [
+        `Title: ${preview.title}`,
+        `Source URL: ${preview.sourceUrl}`,
+        `Preset: ${preview.presetLabel}`,
+        preview.profileLabel ? `Profile: ${preview.profileLabel}` : '',
+        preview.description ? `Description: ${preview.description}` : '',
+        preview.materials.length > 0 ? `Materials:\n- ${preview.materials.join('\n- ')}` : '',
+        preview.instructions.length > 0 ? `Instructions:\n- ${preview.instructions.join('\n- ')}` : '',
+        preview.pageText ? `Page Text:\n${preview.pageText}` : ''
+    ].filter(Boolean).join('\n\n');
+    await fs.writeFile(textFile, outputText, 'utf-8');
+    return `texts/${safeTitle}/pattern.txt`;
+}
+async function extractPatternPreview(url, logger, strategy) {
+    logger(`\n💖 Purring along... Fetching data from: ${url}`);
+    logger(`🪡 Using selector preset: ${strategy.preset.label}`);
+    if (strategy.profileLabel) {
+        logger(`🧁 Loaded site profile: ${strategy.profileLabel}`);
+    }
+    if (strategy.selectorOverrides) {
+        logger('🧷 Advanced selector overrides are active for this run.');
+    }
+    const { data } = await axios_1.default.get(url, {
+        headers: REQUEST_HEADERS,
+        timeout: REQUEST_TIMEOUT_MS
+    });
+    const $ = cheerio.load(data);
+    const selectors = resolveSelectors(strategy.preset, strategy.selectorOverrides);
+    const title = firstTextMatch($, selectors.titleSelectors, 'Untitled Pattern');
+    const description = firstTextMatch($, selectors.descriptionSelectors, 'No description found.');
+    const materials = collectTextMatches($, selectors.materialsSelectors);
+    const instructions = collectTextMatches($, selectors.instructionsSelectors);
+    const pageText = extractPageText($);
+    const imageUrls = collectAssetUrls($, selectors.imageSelectors, url);
+    const pdfUrls = [];
+    $('a').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href || !href.toLowerCase().includes('.pdf'))
+            return;
+        try {
+            const absoluteUrl = new URL(href, url).href;
+            pdfUrls.push(absoluteUrl);
+        }
+        catch { }
+    });
+    return {
+        title,
+        description,
+        materials,
+        instructions,
+        pageText,
+        sourceUrl: url,
+        imageUrls,
+        pdfUrls: dedupeStrings(pdfUrls),
+        preset: strategy.presetId,
+        presetLabel: strategy.preset.label,
+        selectorOverrides: strategy.selectorOverrides,
+        profileId: strategy.profileId,
+        profileLabel: strategy.profileLabel
+    };
+}
+async function scrapePattern(url, workingDirectory, logger, strategy) {
+    try {
+        const preview = await extractPatternPreview(url, logger, strategy);
+        const uniqueImageUrls = dedupeStrings(preview.imageUrls);
+        const uniquePdfUrls = dedupeStrings(preview.pdfUrls);
+        const safeTitle = sanitizeFilename(preview.title);
+        const localImages = [];
+        const localPdfs = [];
+        const localTextFile = await writeTextArtifact(preview, workingDirectory, safeTitle);
+        logger(`📝 Saved page text to ./${localTextFile}`);
+        if (uniqueImageUrls.length > 0) {
+            const imageDir = path.resolve(workingDirectory, 'images', safeTitle);
+            await fs.mkdir(imageDir, { recursive: true });
+            logger(`✨ Found ${uniqueImageUrls.length} images! Downloading to ./images/${safeTitle}/...`);
+            for (let i = 0; i < uniqueImageUrls.length; i++) {
+                const imgUrl = uniqueImageUrls[i];
+                const ext = getImageExtension(imgUrl);
+                const filename = `image_${i + 1}${ext}`;
+                const destPath = path.resolve(imageDir, filename);
+                if (await downloadFile(imgUrl, destPath, logger)) {
+                    localImages.push(`images/${safeTitle}/${filename}`);
+                }
+            }
+        }
+        if (uniquePdfUrls.length > 0) {
+            const pdfDir = path.resolve(workingDirectory, 'pdfs', safeTitle);
+            await fs.mkdir(pdfDir, { recursive: true });
+            logger(`📄 Holy shit, found ${uniquePdfUrls.length} PDFs! Downloading to ./pdfs/${safeTitle}/...`);
+            for (let i = 0; i < uniquePdfUrls.length; i++) {
+                const pdfUrl = uniquePdfUrls[i];
+                let filename = path.basename(new URL(pdfUrl).pathname);
+                if (!filename.toLowerCase().endsWith('.pdf'))
+                    filename = `pattern_${i + 1}.pdf`;
+                const destPath = path.resolve(pdfDir, filename);
+                if (await downloadFile(pdfUrl, destPath, logger)) {
+                    localPdfs.push(`pdfs/${safeTitle}/${filename}`);
+                }
+            }
+        }
+        return {
+            title: preview.title,
+            description: preview.description,
+            materials: preview.materials,
+            instructions: preview.instructions,
+            sourceUrl: url,
+            localImages,
+            localPdfs,
+            localTextFile
+        };
+    }
+    catch (error) {
+        logger(`\n❌ Ah shit, something broke while fetching the URL: ${error.message}`);
+        return null;
+    }
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+async function previewPattern(options, logger = defaultLogger) {
+    const normalizedUrl = normalizeUrl(options.url);
+    if (!normalizedUrl) {
+        throw new Error(`That doesn't look like a valid URL: ${options.url}`);
+    }
+    const strategy = await resolveStrategy({
+        preset: options.preset,
+        selectorOverrides: options.selectorOverrides,
+        profileId: options.profileId,
+        profilesPath: options.profilesPath
+    });
+    return extractPatternPreview(normalizedUrl, logger, strategy);
+}
+async function scrapeUrls(options) {
+    const logger = options.logger ?? defaultLogger;
+    const workingDirectory = options.workingDirectory ?? process.cwd();
+    const strategy = await resolveStrategy({
+        preset: options.preset,
+        selectorOverrides: options.selectorOverrides,
+        profileId: options.profileId,
+        profilesPath: options.profilesPath
+    });
+    const startingUrls = dedupeStrings(options.urls
+        .map(normalizeUrl)
+        .filter((url) => Boolean(url)));
+    if (startingUrls.length === 0) {
+        throw new Error('No valid http(s) URLs were provided.');
+    }
+    const normalizedUrls = await discoverUrls(startingUrls, logger, options.crawl);
+    const existingData = await loadExistingData(options.outputPath);
+    const knownSourceUrls = new Set(existingData.map(item => item.sourceUrl));
+    const newPatterns = [];
+    let skippedCount = 0;
+    let failedCount = 0;
+    for (let i = 0; i < normalizedUrls.length; i++) {
+        const currentUrl = normalizedUrls[i];
+        logger(`\n======================================================`);
+        logger(`🧵 Scraping pattern ${i + 1} of ${normalizedUrls.length}`);
+        logger(`======================================================`);
+        if (knownSourceUrls.has(currentUrl)) {
+            skippedCount += 1;
+            logger(`⚠️ Already scraped ${currentUrl}. Skipping before download to save time.`);
+            continue;
+        }
+        const data = await scrapePattern(currentUrl, workingDirectory, logger, strategy);
+        if (!data) {
+            failedCount += 1;
+        }
+        else if (!knownSourceUrls.has(data.sourceUrl)) {
+            existingData.push(data);
+            newPatterns.push(data);
+            knownSourceUrls.add(data.sourceUrl);
+            await fs.writeFile(options.outputPath, JSON.stringify(existingData, null, 2), 'utf-8');
+            logger(`✨ Badass! Appended data to ${path.basename(options.outputPath)} 🧁`);
+        }
+        else {
+            skippedCount += 1;
+            logger(`⚠️ Looks like we already have the data for "${data.title}". Skipping JSON append to keep it clean.`);
+        }
+        if (i < normalizedUrls.length - 1) {
+            logger(`\n⏳ Taking a quick 2-second breather so we don't get blocked...`);
+            await sleep(DEFAULT_DELAY_MS);
+        }
+    }
+    logger(`\n🎉 All done! Your data is ready to roll. You crushed it.`);
+    return {
+        scrapedCount: newPatterns.length,
+        skippedCount,
+        failedCount,
+        outputPath: options.outputPath,
+        outputDirectory: workingDirectory,
+        patterns: newPatterns,
+        preset: strategy.presetId,
+        selectorOverrides: strategy.selectorOverrides,
+        profileId: strategy.profileId,
+        profileLabel: strategy.profileLabel,
+        discoveredUrlCount: normalizedUrls.length
+    };
+}