@larkiny/astro-github-loader 0.9.0

package/dist/github.content.js

@@ -0,0 +1,1016 @@
+import { existsSync, promises as fs } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import path, { join, dirname, basename, extname } from "node:path";
+import picomatch from "picomatch";
+import { globalLinkTransform, generateAutoLinkMappings } from "./github.link-transform.js";
+import { getLatestCommitInfo, loadImportState, createConfigId } from "./github.dryrun.js";
+import { INVALID_SERVICE_RESPONSE, INVALID_STRING_ERROR, INVALID_URL_ERROR, } from "./github.constants.js";
+/**
+ * Generates a unique identifier from a file path by removing the extension
+ * @param filePath - The file path to generate ID from
+ * @return {string} The generated identifier as a string with extension removed
+ * @internal
+ */
+export function generateId(filePath) {
+    let id = filePath;
+    // Remove file extension for ID generation
+    const lastDotIndex = id.lastIndexOf('.');
+    if (lastDotIndex > 0) {
+        id = id.substring(0, lastDotIndex);
+    }
+    return id;
+}
+/**
+ * Applies path mapping logic to get the final filename for a file
+ *
+ * Supports two types of path mappings:
+ * - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
+ * - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
+ *
+ * @param filePath - Original source file path
+ * @param matchedPattern - The pattern that matched this file
+ * @param options - Import options containing path mappings
+ * @returns Final filename after applying path mapping logic
+ * @internal
+ */
+export function applyRename(filePath, matchedPattern, options) {
+    if (options?.includes && matchedPattern && matchedPattern.index < options.includes.length) {
+        const includePattern = options.includes[matchedPattern.index];
+        if (includePattern.pathMappings) {
+            // First check for exact file match (current behavior - backwards compatible)
+            if (includePattern.pathMappings[filePath]) {
+                const mappingValue = includePattern.pathMappings[filePath];
+                return typeof mappingValue === 'string' ? mappingValue : mappingValue.target;
+            }
+            // Then check for folder-to-folder mappings
+            for (const [sourceFolder, mappingValue] of Object.entries(includePattern.pathMappings)) {
+                // Check if this is a folder mapping (ends with /) and file is within it
+                if (sourceFolder.endsWith('/') && filePath.startsWith(sourceFolder)) {
+                    // Replace the source folder path with target folder path
+                    const targetFolder = typeof mappingValue === 'string' ? mappingValue : mappingValue.target;
+                    const relativePath = filePath.slice(sourceFolder.length);
+                    return path.posix.join(targetFolder, relativePath);
+                }
+            }
+        }
+    }
+    // Return original filename if no path mapping found
+    return basename(filePath);
+}
+/**
+ * Generates a local file path based on the matched pattern and file path
+ * @param filePath - The original file path from the repository
+ * @param matchedPattern - The pattern that matched this file (or null if no includes specified)
+ * @param options - Import options containing includes patterns for path mapping lookups
+ * @return {string} The local file path where this content should be stored
+ * @internal
+ */
+export function generatePath(filePath, matchedPattern, options) {
+    if (matchedPattern) {
+        // Extract the directory part from the pattern (before any glob wildcards)
+        const pattern = matchedPattern.pattern;
+        const beforeGlob = pattern.split(/[*?{]/)[0];
+        // Remove the pattern prefix from the file path to get the relative path
+        let relativePath = filePath;
+        if (beforeGlob && filePath.startsWith(beforeGlob)) {
+            relativePath = filePath.substring(beforeGlob.length);
+            // Remove leading slash if present
+            if (relativePath.startsWith('/')) {
+                relativePath = relativePath.substring(1);
+            }
+        }
+        // If no relative path remains, use just the filename
+        if (!relativePath) {
+            relativePath = basename(filePath);
+        }
+        // Apply path mapping logic
+        const finalFilename = applyRename(filePath, matchedPattern, options);
+        // Always apply path mapping if applyRename returned something different from the original basename
+        // OR if there are pathMappings configured (since empty string mappings might return same basename)
+        const hasPathMappings = options?.includes?.[matchedPattern.index]?.pathMappings &&
+            Object.keys(options.includes[matchedPattern.index].pathMappings).length > 0;
+        if (finalFilename !== basename(filePath) || hasPathMappings) {
+            // Check if applyRename returned a full path (contains path separators) or just a filename
+            if (finalFilename.includes('/') || finalFilename.includes('\\')) {
+                // applyRename returned a full relative path - need to extract relative part
+                // Remove the pattern prefix to get the relative path within the pattern context
+                const beforeGlob = pattern.split(/[*?{]/)[0];
+                if (beforeGlob && finalFilename.startsWith(beforeGlob)) {
+                    relativePath = finalFilename.substring(beforeGlob.length);
+                    // Remove leading slash if present
+                    if (relativePath.startsWith('/')) {
+                        relativePath = relativePath.substring(1);
+                    }
+                }
+                else {
+                    relativePath = finalFilename;
+                }
+            }
+            else {
+                // applyRename returned just a filename
+                // If the filename is different due to pathMapping, use it directly
+                // This handles cases where pathMappings flatten directory structures
+                relativePath = finalFilename;
+            }
+        }
+        return join(matchedPattern.basePath, relativePath);
+    }
+    // Should not happen since we always use includes
+    throw new Error("No matched pattern provided - includes are required");
+}
+/**
+ * Synchronizes a file by ensuring the target directory exists and then writing the specified content to the file at the given path.
+ *
+ * @param {string} path - The path of the file to synchronize, including its directory and filename.
+ * @param {string} content - The content to write into the file.
+ * @return {Promise<void>} - A promise that resolves when the file has been successfully written.
+ * @internal
+ */
+export async function syncFile(path, content) {
+    const dir = path.substring(0, path.lastIndexOf("/"));
+    // Ensure the directory exists
+    if (dir && !existsSync(dir)) {
+        await fs.mkdir(dir, { recursive: true });
+    }
+    // Write the file to the filesystem and store
+    await fs.writeFile(path, content, "utf-8");
+}
+/**
+ * Default asset patterns for common image and media file types
+ * @internal
+ */
+const DEFAULT_ASSET_PATTERNS = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.ico', '.bmp'];
+/**
+ * Checks if a file path should be included and returns the matching pattern
+ * @param filePath - The file path to check (relative to the repository root)
+ * @param options - Import options containing includes patterns
+ * @returns Object with include status and matched pattern, or null if not included
+ * @internal
+ */
+export function shouldIncludeFile(filePath, options) {
+    const { includes } = options;
+    // If no include patterns specified, include all files
+    if (!includes || includes.length === 0) {
+        return { included: true, matchedPattern: null };
+    }
+    // Check each include pattern to find a match
+    for (let i = 0; i < includes.length; i++) {
+        const includePattern = includes[i];
+        const matcher = picomatch(includePattern.pattern);
+        if (matcher(filePath)) {
+            return {
+                included: true,
+                matchedPattern: {
+                    pattern: includePattern.pattern,
+                    basePath: includePattern.basePath,
+                    index: i
+                }
+            };
+        }
+    }
+    // No patterns matched
+    return { included: false, matchedPattern: null };
+}
+/**
+ * Detects asset references in markdown content using regex patterns
+ * @param content - The markdown content to parse
+ * @param assetPatterns - File extensions to treat as assets
+ * @returns Array of detected asset paths
+ * @internal
+ */
+export function detectAssets(content, assetPatterns = DEFAULT_ASSET_PATTERNS) {
+    const assets = [];
+    const patterns = assetPatterns.map(ext => ext.toLowerCase());
+    // Match markdown images: ![alt](path)
+    const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
+    let match;
+    while ((match = imageRegex.exec(content)) !== null) {
+        const assetPath = match[1];
+        // Only include relative paths and assets matching our patterns
+        if (assetPath.startsWith('./') || assetPath.startsWith('../') || !assetPath.includes('://')) {
+            const ext = extname(assetPath).toLowerCase();
+            if (patterns.includes(ext)) {
+                assets.push(assetPath);
+            }
+        }
+    }
+    // Match HTML img tags: <img src="path">
+    const htmlImgRegex = /<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi;
+    while ((match = htmlImgRegex.exec(content)) !== null) {
+        const assetPath = match[1];
+        if (assetPath.startsWith('./') || assetPath.startsWith('../') || !assetPath.includes('://')) {
+            const ext = extname(assetPath).toLowerCase();
+            if (patterns.includes(ext)) {
+                assets.push(assetPath);
+            }
+        }
+    }
+    return [...new Set(assets)]; // Remove duplicates
+}
+/**
+ * Downloads an asset from GitHub and saves it locally
+ * @param octokit - GitHub API client
+ * @param owner - Repository owner
+ * @param repo - Repository name
+ * @param ref - Git reference
+ * @param assetPath - Path to the asset in the repository
+ * @param localPath - Local path where the asset should be saved
+ * @param signal - Abort signal for cancellation
+ * @returns Promise that resolves when the asset is downloaded
+ * @internal
+ */
+export async function downloadAsset(octokit, owner, repo, ref, assetPath, localPath, signal) {
+    try {
+        const { data } = await octokit.rest.repos.getContent({
+            owner,
+            repo,
+            path: assetPath,
+            ref,
+            request: { signal },
+        });
+        if (Array.isArray(data) || data.type !== 'file' || !data.download_url) {
+            throw new Error(`Asset ${assetPath} is not a valid file (type: ${data.type}, downloadUrl: ${data.download_url})`);
+        }
+        const response = await fetch(data.download_url, { signal });
+        if (!response.ok) {
+            throw new Error(`Failed to download asset: ${response.status} ${response.statusText}`);
+        }
+        const buffer = await response.arrayBuffer();
+        const dir = dirname(localPath);
+        if (!existsSync(dir)) {
+            await fs.mkdir(dir, { recursive: true });
+        }
+        await fs.writeFile(localPath, new Uint8Array(buffer));
+    }
+    catch (error) {
+        if (error.status === 404) {
+            throw new Error(`Asset not found: ${assetPath}`);
+        }
+        throw error;
+    }
+}
+/**
+ * Transforms asset references in markdown content to use local paths
+ * @param content - The markdown content to transform
+ * @param assetMap - Map of original asset paths to new local paths
+ * @returns Transformed content with updated asset references
+ * @internal
+ */
+export function transformAssetReferences(content, assetMap) {
+    let transformedContent = content;
+    for (const [originalPath, newPath] of assetMap) {
+        // Transform markdown images
+        const imageRegex = new RegExp(`(!)\\[([^\\]]*)\\]\\(\\s*${escapeRegExp(originalPath)}\\s*\\)`, 'g');
+        transformedContent = transformedContent.replace(imageRegex, `$1[$2](${newPath})`);
+        // Transform HTML img tags
+        const htmlRegex = new RegExp(`(<img[^>]+src\\s*=\\s*["'])${escapeRegExp(originalPath)}(["'][^>]*>)`, 'gi');
+        transformedContent = transformedContent.replace(htmlRegex, `$1${newPath}$2`);
+    }
+    return transformedContent;
+}
+/**
+ * Escapes special regex characters in a string
+ * @internal
+ */
+function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Processes assets in markdown content by detecting, downloading, and transforming references
+ * @param content - The markdown content to process
+ * @param options - Configuration options including asset settings
+ * @param octokit - GitHub API client
+ * @param signal - Abort signal for cancellation
+ * @returns Promise that resolves to transformed content
+ * @internal
+ */
+async function processAssets(content, filePath, options, octokit, logger, signal) {
+    const { owner, repo, ref = 'main', assetsPath, assetsBaseUrl, assetPatterns } = options;
+    logger.verbose(`🖼️  Processing assets for ${filePath}`);
+    logger.debug(`    assetsPath: ${assetsPath}`);
+    logger.debug(`    assetsBaseUrl: ${assetsBaseUrl}`);
+    if (!assetsPath || !assetsBaseUrl) {
+        logger.verbose(`    ⏭️  Skipping asset processing - missing assetsPath or assetsBaseUrl`);
+        return { content, assetsDownloaded: 0, assetsCached: 0 };
+    }
+    // Detect assets in the content
+    const detectedAssets = detectAssets(content, assetPatterns);
+    logger.verbose(`    📸 Detected ${detectedAssets.length} assets`);
+    if (detectedAssets.length > 0) {
+        logger.debug(`    Assets: ${detectedAssets.join(', ')}`);
+    }
+    if (detectedAssets.length === 0) {
+        return { content, assetsDownloaded: 0, assetsCached: 0 };
+    }
+    const assetMap = new Map();
+    let assetsDownloaded = 0;
+    let assetsCached = 0;
+    // Process each detected asset
+    await Promise.all(detectedAssets.map(async (assetPath) => {
+        logger.logAssetProcessing("Processing", assetPath);
+        try {
+            // Resolve the asset path relative to the current markdown file
+            const resolvedAssetPath = resolveAssetPath(filePath, assetPath);
+            logger.debug(`    🔗 Resolved path: ${resolvedAssetPath}`);
+            // Generate unique filename to avoid conflicts
+            const originalFilename = basename(assetPath);
+            const ext = extname(originalFilename);
+            const nameWithoutExt = basename(originalFilename, ext);
+            const uniqueFilename = `${nameWithoutExt}-${Date.now()}${ext}`;
+            const localPath = join(assetsPath, uniqueFilename);
+            logger.debug(`    💾 Local path: ${localPath}`);
+            // Check if asset already exists (simple cache check)
+            if (existsSync(localPath)) {
+                logger.logAssetProcessing("Cached", assetPath);
+                assetsCached++;
+            }
+            else {
+                // Download the asset
+                logger.logAssetProcessing("Downloading", assetPath, `from ${owner}/${repo}@${ref}:${resolvedAssetPath}`);
+                await downloadAsset(octokit, owner, repo, ref, resolvedAssetPath, localPath, signal);
+                logger.logAssetProcessing("Downloaded", assetPath);
+                assetsDownloaded++;
+            }
+            // Generate URL for the transformed reference
+            const assetUrl = `${assetsBaseUrl}/${uniqueFilename}`.replace(/\/+/g, '/');
+            logger.debug(`    🔄 Transform: ${assetPath} -> ${assetUrl}`);
+            // Map the transformation
+            assetMap.set(assetPath, assetUrl);
+        }
+        catch (error) {
+            logger.warn(`    ❌ Failed to process asset ${assetPath}: ${error}`);
+        }
+    }));
+    logger.verbose(`    🗺️  Processed ${assetMap.size} assets: ${assetsDownloaded} downloaded, ${assetsCached} cached`);
+    // Transform the content with new asset references
+    const transformedContent = transformAssetReferences(content, assetMap);
+    return { content: transformedContent, assetsDownloaded, assetsCached };
+}
+/**
+ * Resolves an asset path relative to a base path
+ * @internal
+ */
+function resolveAssetPath(basePath, assetPath) {
+    if (assetPath.startsWith('./')) {
+        return join(dirname(basePath), assetPath.slice(2));
+    }
+    else if (assetPath.startsWith('../')) {
+        return join(dirname(basePath), assetPath);
+    }
+    return assetPath;
+}
+/**
+ * Synchronizes an entry by fetching its contents, validating its metadata, and storing or rendering it as needed.
+ *
+ * @param {LoaderContext} context - The loader context containing the required utilities, metadata, and configuration.
+ * @param {Object} urls - Object containing URL data.
+ * @param {string | URL | null} urls.url - The URL of the entry to fetch. Throws an error if null or invalid.
+ * @param {string} urls.editUrl - The URL for editing the entry.
+ * @param {RootOptions} options - Configuration settings for processing the entry such as file paths and custom options.
+ * @param {any} octokit - GitHub API client for downloading assets.
+ * @param {RequestInit} [init] - Optional parameter for customizing the fetch request.
+ * @return {Promise<void>} Resolves when the entry has been successfully processed and stored. Throws errors if invalid URL, missing configuration, or other issues occur.
+ * @internal
+ */
+export async function syncEntry(context, { url, editUrl }, filePath, options, octokit, init = {}) {
+    // Exit on null or if the URL is invalid
+    if (url === null || (typeof url !== "string" && !(url instanceof URL))) {
+        throw new TypeError(INVALID_URL_ERROR);
+    }
+    // Validate URL
+    if (typeof url === "string")
+        url = new URL(url);
+    const { meta, store, generateDigest, entryTypes, logger, parseData, config } = context;
+    function configForFile(file) {
+        const ext = file.split(".").at(-1);
+        if (!ext) {
+            logger.warn(`No extension found for ${file}`);
+            return;
+        }
+        return entryTypes?.get(`.${ext}`);
+    }
+    // Custom ID, TODO: Allow custom id generators
+    let id = generateId(filePath);
+    init.headers = getHeaders({
+        init: init.headers,
+        meta,
+        id,
+    });
+    let res = await fetch(url, init);
+    if (res.status === 304) {
+        // Only skip if the local file actually exists
+        const includeResult = shouldIncludeFile(filePath, options);
+        const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
+        const fileUrl = pathToFileURL(relativePath);
+        if (existsSync(fileURLToPath(fileUrl))) {
+            logger.info(`Skipping ${id} as it has not changed`);
+            return;
+        }
+        else {
+            logger.info(`File ${id} missing locally, re-fetching despite 304`);
+            // File is missing locally, fetch without ETag headers
+            const freshInit = { ...init };
+            freshInit.headers = new Headers(init.headers);
+            freshInit.headers.delete('If-None-Match');
+            freshInit.headers.delete('If-Modified-Since');
+            res = await fetch(url, freshInit);
+            if (!res.ok)
+                throw new Error(res.statusText);
+        }
+    }
+    if (!res.ok)
+        throw new Error(res.statusText);
+    let contents = await res.text();
+    const entryType = configForFile(filePath || "tmp.md");
+    if (!entryType)
+        throw new Error("No entry type found");
+    // Process assets FIRST if configuration is provided - before content transforms
+    // This ensures asset detection works with original markdown links before they get transformed
+    if (options.assetsPath && options.assetsBaseUrl) {
+        try {
+            // Create a dummy logger for syncEntry since it uses Astro's logger
+            const dummyLogger = {
+                verbose: (msg) => logger.info(msg),
+                debug: (msg) => logger.debug(msg),
+                warn: (msg) => logger.warn(msg),
+                logAssetProcessing: (action, path, details) => {
+                    const msg = details ? `Asset ${action}: ${path} - ${details}` : `Asset ${action}: ${path}`;
+                    logger.info(msg);
+                }
+            };
+            const assetResult = await processAssets(contents, filePath, options, octokit, dummyLogger, init.signal || undefined);
+            contents = assetResult.content;
+        }
+        catch (error) {
+            logger.warn(`Asset processing failed for ${id}: ${error.message}`);
+        }
+    }
+    // Apply content transforms if provided - both global and pattern-specific
+    // This runs after asset processing so transforms work with processed content
+    const includeResultForTransforms = shouldIncludeFile(filePath, options);
+    const transformsToApply = [];
+    // Add global transforms first
+    if (options.transforms && options.transforms.length > 0) {
+        transformsToApply.push(...options.transforms);
+    }
+    // Add pattern-specific transforms
+    if (includeResultForTransforms.included && includeResultForTransforms.matchedPattern && options.includes) {
+        const matchedInclude = options.includes[includeResultForTransforms.matchedPattern.index];
+        if (matchedInclude.transforms && matchedInclude.transforms.length > 0) {
+            transformsToApply.push(...matchedInclude.transforms);
+        }
+    }
+    if (transformsToApply.length > 0) {
+        const transformContext = {
+            id,
+            path: filePath,
+            options,
+            matchedPattern: includeResultForTransforms.included ? includeResultForTransforms.matchedPattern : undefined,
+        };
+        for (const transform of transformsToApply) {
+            try {
+                contents = transform(contents, transformContext);
+            }
+            catch (error) {
+                logger.warn(`Transform failed for ${id}: ${error}`);
+            }
+        }
+    }
+    const includeResult = shouldIncludeFile(filePath, options);
+    const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
+    const fileUrl = pathToFileURL(relativePath);
+    const { body, data } = await entryType.getEntryInfo({
+        contents,
+        fileUrl: fileUrl,
+    });
+    const existingEntry = store.get(id);
+    const digest = generateDigest(contents);
+    if (existingEntry &&
+        existingEntry.digest === digest &&
+        existingEntry.filePath) {
+        return;
+    }
+    // Write file to path
+    if (!existsSync(fileURLToPath(fileUrl))) {
+        logger.verbose(`Writing ${id} to ${fileUrl}`);
+        await syncFile(fileURLToPath(fileUrl), contents);
+    }
+    const parsedData = await parseData({
+        id,
+        data,
+        filePath: fileUrl.toString(),
+    });
+    if (entryType.getRenderFunction) {
+        logger.verbose(`Rendering ${id}`);
+        const render = await entryType.getRenderFunction(config);
+        let rendered = undefined;
+        try {
+            rendered = await render?.({
+                id,
+                data,
+                body,
+                filePath: fileUrl.toString(),
+                digest,
+            });
+        }
+        catch (error) {
+            logger.error(`Error rendering ${id}: ${error.message}`);
+        }
+        store.set({
+            id,
+            data: parsedData,
+            body,
+            filePath: relativePath,
+            digest,
+            rendered,
+        });
+    }
+    else if ("contentModuleTypes" in entryType) {
+        store.set({
+            id,
+            data: parsedData,
+            body,
+            filePath: relativePath,
+            digest,
+            deferredRender: true,
+        });
+    }
+    else {
+        store.set({ id, data: parsedData, body, filePath: relativePath, digest });
+    }
+    syncHeaders({
+        headers: res.headers,
+        meta,
+        id,
+    });
+}
+/**
+ * Converts a given GitHub repository path into a collection entry by fetching the content
+ * from the GitHub repository using the provided Octokit instance and options.
+ * Handles both files and directories, recursively processing directories if needed.
+ * @internal
+ */
+export async function toCollectionEntry({ context, octokit, options, signal, force = false, }) {
+    const { owner, repo, ref = "main" } = options || {};
+    if (typeof repo !== "string" || typeof owner !== "string")
+        throw new TypeError(INVALID_STRING_ERROR);
+    // Get logger from context - it should be our Logger instance (initialize early)
+    const logger = context.logger;
+    // Repository-level caching - simple all-or-nothing approach
+    const configName = options.name || `${owner}/${repo}`;
+    const configId = createConfigId(options);
+    if (!force) {
+        try {
+            const state = await loadImportState(process.cwd());
+            const currentState = state.imports[configId];
+            if (currentState && currentState.lastCommitSha) {
+                logger.debug(`🔍 Checking repository changes for ${configName}...`);
+                const latestCommit = await getLatestCommitInfo(octokit, options, signal);
+                if (latestCommit && currentState.lastCommitSha === latestCommit.sha) {
+                    logger.info(`✅ Repository ${configName} unchanged (${latestCommit.sha.slice(0, 7)}) - skipping import`);
+                    return {
+                        processed: 0,
+                        updated: 0,
+                        unchanged: 0,
+                        assetsDownloaded: 0,
+                        assetsCached: 0,
+                    };
+                }
+                else if (latestCommit) {
+                    logger.info(`🔄 Repository ${configName} changed (${currentState.lastCommitSha?.slice(0, 7) || 'unknown'} -> ${latestCommit.sha.slice(0, 7)}) - proceeding with import`);
+                }
+            }
+            else {
+                logger.debug(`📥 First time importing ${configName} - no previous state found`);
+            }
+        }
+        catch (error) {
+            logger.warn(`Failed to check repository state for ${configName}: ${error instanceof Error ? error.message : String(error)}`);
+            // Continue with import if state check fails
+        }
+    }
+    else {
+        logger.info(`🔄 Force mode enabled for ${configName} - proceeding with full import`);
+    }
+    // Get all unique directory prefixes from include patterns to limit scanning
+    const directoriesToScan = new Set();
+    if (options.includes && options.includes.length > 0) {
+        for (const includePattern of options.includes) {
+            // Extract directory part from pattern (before any glob wildcards)
+            const pattern = includePattern.pattern;
+            const beforeGlob = pattern.split(/[*?{]/)[0];
+            const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+            directoriesToScan.add(dirPart);
+        }
+    }
+    else {
+        // If no includes specified, scan from root
+        directoriesToScan.add('');
+    }
+    // Collect all files first (with content transforms applied)
+    const allFiles = [];
+    for (const dirPath of directoriesToScan) {
+        const files = await collectFilesRecursively(dirPath);
+        allFiles.push(...files);
+    }
+    // Track statistics
+    const stats = {
+        processed: 0,
+        updated: 0,
+        unchanged: 0,
+        assetsDownloaded: 0,
+        assetsCached: 0,
+    };
+    // Apply link transformation if configured
+    let processedFiles = allFiles;
+    if (options.linkTransform) {
+        logger.verbose(`Applying link transformation to ${allFiles.length} files`);
+        // Generate automatic link mappings from pathMappings
+        const autoGeneratedMappings = options.includes
+            ? generateAutoLinkMappings(options.includes, options.linkTransform.stripPrefixes)
+            : [];
+        // Combine auto-generated mappings with user-defined mappings
+        const allLinkMappings = [
+            ...autoGeneratedMappings,
+            ...(options.linkTransform.linkMappings || [])
+        ];
+        logger.debug(`Generated ${autoGeneratedMappings.length} automatic link mappings from pathMappings`);
+        processedFiles = globalLinkTransform(allFiles, {
+            stripPrefixes: options.linkTransform.stripPrefixes,
+            customHandlers: options.linkTransform.customHandlers,
+            linkMappings: allLinkMappings,
+            logger,
+        });
+    }
+    // Now store all processed files
+    stats.processed = processedFiles.length;
+    for (const file of processedFiles) {
+        logger.logFileProcessing("Storing", file.sourcePath);
+        const result = await storeProcessedFile(file, context, options);
+        if (result) {
+            stats.updated++;
+        }
+        else {
+            stats.unchanged++;
+        }
+    }
+    return stats;
+    // Helper function to collect files without storing them
+    async function collectFilesRecursively(path) {
+        const collectedFiles = [];
+        // Fetch the content
+        const { data, status } = await octokit.rest.repos.getContent({
+            owner,
+            repo,
+            path,
+            ref,
+            request: { signal },
+        });
+        if (status !== 200)
+            throw new Error(INVALID_SERVICE_RESPONSE);
+        // Handle single file
+        if (!Array.isArray(data)) {
+            const filePath = data.path;
+            if (data.type === "file") {
+                const fileData = await collectFileData({ url: data.download_url, editUrl: data.url }, filePath);
+                if (fileData) {
+                    collectedFiles.push(fileData);
+                }
+            }
+            return collectedFiles;
+        }
+        // Directory listing - process files and recurse into subdirectories
+        const filteredEntries = data
+            .filter(({ type, path }) => {
+            // Always include directories for recursion
+            if (type === "dir")
+                return true;
+            // Apply filtering logic to files
+            if (type === "file") {
+                return shouldIncludeFile(path, options).included;
+            }
+            return false;
+        });
+        for (const { type, path, download_url, url } of filteredEntries) {
+            if (type === "dir") {
+                // Recurse into subdirectory
+                const subDirFiles = await collectFilesRecursively(path);
+                collectedFiles.push(...subDirFiles);
+            }
+            else if (type === "file") {
+                // Process file
+                const fileData = await collectFileData({ url: download_url, editUrl: url }, path);
+                if (fileData) {
+                    collectedFiles.push(fileData);
+                }
+            }
+        }
+        return collectedFiles;
+    }
+    // Helper function to collect file data with content transforms applied
+    async function collectFileData({ url, editUrl }, filePath) {
+        if (url === null || typeof url !== "string") {
+            return null;
+        }
+        const urlObj = new URL(url);
+        // Determine if file needs renaming and generate appropriate ID
+        const includeCheck = shouldIncludeFile(filePath, options);
+        const matchedPattern = includeCheck.included ? includeCheck.matchedPattern : null;
+        // Check if this file has a path mapping
+        const hasPathMapping = matchedPattern &&
+            options?.includes &&
+            matchedPattern.index < options.includes.length &&
+            options.includes[matchedPattern.index].pathMappings &&
+            options.includes[matchedPattern.index].pathMappings[filePath];
+        // Generate ID based on appropriate path
+        const id = hasPathMapping ?
+            generateId(generatePath(filePath, matchedPattern, options)) : // Use path-mapped path for ID
+            generateId(filePath); // Use original path for ID
+        const finalPath = generatePath(filePath, matchedPattern, options);
+        let contents;
+        logger.logFileProcessing("Fetching", filePath, `from ${urlObj.toString()}`);
+        // Download file content
+        const init = { signal, headers: getHeaders({ init: {}, meta: context.meta, id }) };
+        let res = null;
+        // Fetch with retries (simplified version of syncEntry logic)
+        for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+                res = await fetch(urlObj, init);
+                if (res.ok)
+                    break;
+            }
+            catch (error) {
+                if (attempt === 2)
+                    throw error;
+                await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
+            }
+        }
+        if (!res) {
+            throw new Error(`No response received for ${urlObj.toString()}`);
+        }
+        if (res.status === 304) {
+            // File not modified, read existing content from disk if it exists
+            const includeResult = shouldIncludeFile(filePath, options);
+            const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
+            const fileUrl = pathToFileURL(relativePath);
+            if (existsSync(fileURLToPath(fileUrl))) {
+                logger.logFileProcessing("Using cached", filePath, "304 not modified");
+                const { promises: fs } = await import('node:fs');
+                contents = await fs.readFile(fileURLToPath(fileUrl), 'utf-8');
+            }
+            else {
+                // File is missing locally, re-fetch without cache headers
+                logger.logFileProcessing("Re-fetching", filePath, "missing locally despite 304");
+                const freshInit = { ...init };
+                freshInit.headers = new Headers(init.headers);
+                freshInit.headers.delete('If-None-Match');
+                freshInit.headers.delete('If-Modified-Since');
+                res = await fetch(urlObj, freshInit);
+                if (!res.ok) {
+                    throw new Error(`Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || 'Unknown error'}`);
+                }
+                contents = await res.text();
+            }
+        }
+        else if (!res.ok) {
+            throw new Error(`Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || 'Unknown error'}`);
+        }
+        else {
+            contents = await res.text();
+        }
+        // Process assets FIRST if configuration is provided
+        let fileAssetsDownloaded = 0;
+        let fileAssetsCached = 0;
+        if (options.assetsPath && options.assetsBaseUrl) {
+            try {
+                const assetResult = await processAssets(contents, filePath, options, octokit, logger, signal);
+                contents = assetResult.content;
+                fileAssetsDownloaded = assetResult.assetsDownloaded;
+                fileAssetsCached = assetResult.assetsCached;
+            }
+            catch (error) {
+                logger.warn(`Asset processing failed for ${id}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        // Apply content transforms
+        const includeResult = shouldIncludeFile(filePath, options);
+        const transformsToApply = [];
+        // Add global transforms first
+        if (options.transforms && options.transforms.length > 0) {
+            transformsToApply.push(...options.transforms);
+        }
+        // Add pattern-specific transforms
+        if (includeResult.included && includeResult.matchedPattern && options.includes) {
+            const matchedInclude = options.includes[includeResult.matchedPattern.index];
+            if (matchedInclude.transforms && matchedInclude.transforms.length > 0) {
+                transformsToApply.push(...matchedInclude.transforms);
+            }
+        }
+        if (transformsToApply.length > 0) {
+            const transformContext = {
+                id,
+                path: filePath,
+                options,
+                matchedPattern: includeResult.included ? includeResult.matchedPattern : undefined,
+            };
+            for (const transform of transformsToApply) {
+                try {
+                    contents = transform(contents, transformContext);
+                }
+                catch (error) {
+                    context.logger?.warn(`Transform failed for ${id}: ${error}`);
+                }
+            }
+        }
+        // Build link context for this file
+        const linkContext = includeResult.included && includeResult.matchedPattern ? {
+            sourcePath: filePath,
+            targetPath: finalPath,
+            basePath: includeResult.matchedPattern.basePath,
+            pathMappings: options.includes?.[includeResult.matchedPattern.index]?.pathMappings,
+            matchedPattern: includeResult.matchedPattern,
+        } : undefined;
+        // Use the finalPath we already computed
+        return {
+            sourcePath: filePath,
+            targetPath: finalPath,
+            content: contents,
+            id,
+            linkContext,
+        };
+    }
+    // Helper function to store a processed file
+    async function storeProcessedFile(file, context, options) {
+        const { store, generateDigest, entryTypes, logger, parseData, config } = context;
+        function configForFile(filePath) {
+            const ext = filePath.split(".").at(-1);
+            if (!ext) {
+                logger.warn(`No extension found for ${filePath}`);
+                return;
+            }
+            return entryTypes?.get(`.${ext}`);
+        }
+        const entryType = configForFile(file.sourcePath || "tmp.md");
+        if (!entryType)
+            throw new Error("No entry type found");
+        const fileUrl = pathToFileURL(file.targetPath);
+        const { body, data } = await entryType.getEntryInfo({
+            contents: file.content,
+            fileUrl: fileUrl,
+        });
+        // Generate digest for storage (repository-level caching handles change detection)
+        const digest = generateDigest(file.content);
+        const existingEntry = store.get(file.id);
+        if (existingEntry) {
+            logger.debug(`🔄 File ${file.id} - updating`);
+        }
+        else {
+            logger.debug(`📄 File ${file.id} - adding`);
+        }
+        // Write file to disk
+        if (!existsSync(fileURLToPath(fileUrl))) {
+            logger.verbose(`Writing ${file.id} to ${fileUrl}`);
+            await syncFile(fileURLToPath(fileUrl), file.content);
+        }
+        const parsedData = await parseData({
+            id: file.id,
+            data,
+            filePath: fileUrl.toString(),
+        });
+        // Store in content store
+        if (entryType.getRenderFunction) {
+            logger.verbose(`Rendering ${file.id}`);
+            const render = await entryType.getRenderFunction(config);
+            let rendered = undefined;
+            try {
+                rendered = await render?.({
+                    id: file.id,
+                    data,
+                    body,
+                    filePath: fileUrl.toString(),
+                    digest,
+                });
+            }
+            catch (error) {
+                logger.error(`Error rendering ${file.id}: ${error.message}`);
+            }
+            logger.debug(`🔍 Storing collection entry: ${file.id} (${file.sourcePath} -> ${file.targetPath})`);
+            store.set({
+                id: file.id,
+                data: parsedData,
+                body,
+                filePath: file.targetPath,
+                digest,
+                rendered,
+            });
+        }
+        else if ("contentModuleTypes" in entryType) {
+            store.set({
+                id: file.id,
+                data: parsedData,
+                body,
+                filePath: file.targetPath,
+                digest,
+                deferredRender: true,
+            });
+        }
+        else {
+            store.set({
+                id: file.id,
+                data: parsedData,
+                body,
+                filePath: file.targetPath,
+                digest
+            });
+        }
+        return { id: file.id, filePath: file.targetPath };
+    }
+    async function processDirectoryRecursively(path) {
+        // Fetch the content
+        const { data, status } = await octokit.rest.repos.getContent({
+            owner,
+            repo,
+            path,
+            ref,
+            request: { signal },
+        });
+        if (status !== 200)
+            throw new Error(INVALID_SERVICE_RESPONSE);
+        // Matches for regular files
+        if (!Array.isArray(data)) {
+            const filePath = data.path;
+            switch (data.type) {
+                // Return
+                case "file":
+                    return await syncEntry(context, { url: data.download_url, editUrl: data.url }, filePath, options, octokit, { signal });
+                default:
+                    throw new Error("Invalid type");
+            }
+        }
+        // Directory listing with filtering - process sequentially
+        const filteredEntries = data
+            .filter(({ type, path }) => {
+            // Always include directories for recursion
+            if (type === "dir")
+                return true;
+            // Apply filtering logic to files
+            if (type === "file") {
+                return shouldIncludeFile(path, options).included;
+            }
+            return false;
+        });
+        const results = [];
+        for (const { type, path, download_url, url } of filteredEntries) {
+            switch (type) {
+                // Recurse
+                case "dir":
+                    results.push(await processDirectoryRecursively(path));
+                    break;
+                // Return
+                case "file":
+                    results.push(await syncEntry(context, { url: download_url, editUrl: url }, path, options, octokit, { signal }));
+                    break;
+                default:
+                    throw new Error("Invalid type");
+            }
+        }
+        return results;
+    } // End of processDirectoryRecursively function
+}
+/**
+ * Get the headers needed to make a conditional request.
+ * Uses the etag and last-modified values from the meta store.
+ * @internal
+ */
+export function getHeaders({ init, meta, id, }) {
+    const tag = `${id}-etag`;
+    const lastModifiedTag = `${id}-last-modified`;
+    const etag = meta.get(tag);
+    const lastModified = meta.get(lastModifiedTag);
+    const headers = new Headers(init);
+    if (etag) {
+        headers.set("If-None-Match", etag);
+    }
+    else if (lastModified) {
+        headers.set("If-Modified-Since", lastModified);
+    }
+    return headers;
+}
+/**
+ * Store the etag or last-modified headers from a response in the meta store.
+ * @internal
+ */
+export function syncHeaders({ headers, meta, id, }) {
+    const etag = headers.get("etag");
+    const lastModified = headers.get("last-modified");
+    const tag = `${id}-etag`;
+    const lastModifiedTag = `${id}-last-modified`;
+    meta.delete(tag);
+    meta.delete(lastModifiedTag);
+    if (etag) {
+        meta.set(tag, etag);
+    }
+    else if (lastModified) {
+        meta.set(lastModifiedTag, lastModified);
+    }
+}