@larkiny/astro-github-loader 0.11.3 → 0.13.0

package/src/github.content.ts

@@ -1,542 +1,236 @@
-import { existsSync, promises as fs } from "node:fs";
+import { existsSync } from "node:fs";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import path, { join, dirname, basename, extname } from "node:path";
-import picomatch from "picomatch";
-import { globalLinkTransform, generateAutoLinkMappings, type ImportedFile } from "./github.link-transform.js";
-import type { Logger } from "./github.logger.js";
-
+import path from "node:path";
 import {
-  INVALID_SERVICE_RESPONSE,
-  INVALID_STRING_ERROR,
-  INVALID_URL_ERROR,
-} from "./github.constants.js";
-
-import type { LoaderContext, CollectionEntryOptions, ImportOptions, RenderedContent, MatchedPattern } from "./github.types.js";
-
-export interface ImportStats {
-  processed: number;
-  updated: number;
-  unchanged: number;
-  assetsDownloaded?: number;
-  assetsCached?: number;
-}
-
-/**
- * Generates a unique identifier from a file path by removing the extension
- * @param filePath - The file path to generate ID from
- * @return {string} The generated identifier as a string with extension removed
- * @internal
- */
-export function generateId(filePath: string): string {
-  let id = filePath;
-
-  // Remove file extension for ID generation
-  const lastDotIndex = id.lastIndexOf('.');
-  if (lastDotIndex > 0) {
-    id = id.substring(0, lastDotIndex);
-  }
-  return id;
-}
-
-
-/**
- * Applies path mapping logic to get the final filename for a file
- *
- * Supports two types of path mappings:
- * - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
- * - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
- *
- * @param filePath - Original source file path
- * @param matchedPattern - The pattern that matched this file
- * @param options - Import options containing path mappings
- * @returns Final filename after applying path mapping logic
- * @internal
- */
-export function applyRename(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string {
-  if (options?.includes && matchedPattern && matchedPattern.index < options.includes.length) {
-    const includePattern = options.includes[matchedPattern.index];
-
-    if (includePattern.pathMappings) {
-      // First check for exact file match (current behavior - backwards compatible)
-      if (includePattern.pathMappings[filePath]) {
-        const mappingValue = includePattern.pathMappings[filePath];
-        return typeof mappingValue === 'string' ? mappingValue : mappingValue.target;
-      }
-
-      // Then check for folder-to-folder mappings
-      for (const [sourceFolder, mappingValue] of Object.entries(includePattern.pathMappings)) {
-        // Check if this is a folder mapping (ends with /) and file is within it
-        if (sourceFolder.endsWith('/') && filePath.startsWith(sourceFolder)) {
-          // Replace the source folder path with target folder path
-          const targetFolder = typeof mappingValue === 'string' ? mappingValue : mappingValue.target;
-          const relativePath = filePath.slice(sourceFolder.length);
-          return path.posix.join(targetFolder, relativePath);
-        }
-      }
-    }
-  }
-
-  // Return original filename if no path mapping found
-  return basename(filePath);
-}
-
-/**
- * Generates a local file path based on the matched pattern and file path
- * @param filePath - The original file path from the repository
- * @param matchedPattern - The pattern that matched this file (or null if no includes specified)
- * @param options - Import options containing includes patterns for path mapping lookups
- * @return {string} The local file path where this content should be stored
- * @internal
- */
-export function generatePath(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string {
-  if (matchedPattern) {
-    // Extract the directory part from the pattern (before any glob wildcards)
-    const pattern = matchedPattern.pattern;
-    const beforeGlob = pattern.split(/[*?{]/)[0];
-    
-    // Remove the pattern prefix from the file path to get the relative path
-    let relativePath = filePath;
-    if (beforeGlob && filePath.startsWith(beforeGlob)) {
-      relativePath = filePath.substring(beforeGlob.length);
-      // Remove leading slash if present
-      if (relativePath.startsWith('/')) {
-        relativePath = relativePath.substring(1);
-      }
-    }
-    
-    // If no relative path remains, use just the filename
-    if (!relativePath) {
-      relativePath = basename(filePath);
-    }
-
-    // Apply path mapping logic
-    const finalFilename = applyRename(filePath, matchedPattern, options);
-    // Always apply path mapping if applyRename returned something different from the original basename
-    // OR if there are pathMappings configured (since empty string mappings might return same basename)
-    const hasPathMappings = options?.includes?.[matchedPattern.index]?.pathMappings &&
-                           Object.keys(options.includes[matchedPattern.index].pathMappings!).length > 0;
-    if (finalFilename !== basename(filePath) || hasPathMappings) {
-      // Check if applyRename returned a full path (contains path separators) or just a filename
-      if (finalFilename.includes('/') || finalFilename.includes('\\')) {
-        // applyRename returned a full relative path - need to extract relative part
-        // Remove the pattern prefix to get the relative path within the pattern context
-        const beforeGlob = pattern.split(/[*?{]/)[0];
-        if (beforeGlob && finalFilename.startsWith(beforeGlob)) {
-          relativePath = finalFilename.substring(beforeGlob.length);
-          // Remove leading slash if present
-          if (relativePath.startsWith('/')) {
-            relativePath = relativePath.substring(1);
-          }
-        } else {
-          relativePath = finalFilename;
-        }
-      } else {
-        // applyRename returned just a filename
-        // If the filename is different due to pathMapping, use it directly
-        // This handles cases where pathMappings flatten directory structures
-        relativePath = finalFilename;
-      }
-    }
-
-    return join(matchedPattern.basePath, relativePath);
-  }
-  
-  // Should not happen since we always use includes
-  throw new Error("No matched pattern provided - includes are required");
-}
+  globalLinkTransform,
+  generateAutoLinkMappings,
+  type ImportedFile,
+} from "./github.link-transform.js";
+import { Octokit } from "octokit";
+import { INVALID_STRING_ERROR } from "./github.constants.js";
+
+import type {
+  CollectionEntryOptions,
+  ExtendedLoaderContext,
+  ImportOptions,
+  TransformFunction,
+} from "./github.types.js";
+
+// Decomposed modules
+import {
+  type ImportStats,
+  generateId,
+  generatePath,
+  shouldIncludeFile,
+  getHeaders,
+} from "./github.paths.js";
+import { resolveAssetConfig, processAssets } from "./github.assets.js";
+import { storeProcessedFile } from "./github.storage.js";
+
+// Re-export items that used to live in this module so existing internal
+// consumers can migrate gradually (cleanup.ts, spec files, etc.).
+export {
+  type ImportStats,
+  generateId,
+  generatePath,
+  shouldIncludeFile,
+  applyRename,
+  getHeaders,
+  syncHeaders,
+} from "./github.paths.js";
+export { syncFile } from "./github.storage.js";
+export {
+  resolveAssetConfig,
+  detectAssets,
+  downloadAsset,
+  transformAssetReferences,
+} from "./github.assets.js";
 
 /**
- * Synchronizes a file by ensuring the target directory exists and then writing the specified content to the file at the given path.
- *
- * @param {string} path - The path of the file to synchronize, including its directory and filename.
- * @param {string} content - The content to write into the file.
- * @return {Promise<void>} - A promise that resolves when the file has been successfully written.
+ * Validates that a basePath is relative and does not escape the project root.
  * @internal
  */
-export async function syncFile(path: string, content: string) {
-  const dir = path.substring(0, path.lastIndexOf("/"));
-
-  // Ensure the directory exists
-  if (dir && !existsSync(dir)) {
-    await fs.mkdir(dir, { recursive: true });
+function validateBasePath(basePath: string, projectRoot: string): void {
+  if (path.isAbsolute(basePath)) {
+    throw new Error(`basePath must be relative, got absolute path: ${basePath}`);
+  }
+  const resolved = path.resolve(projectRoot, basePath);
+  const normalized = path.normalize(resolved);
+  if (!normalized.startsWith(path.normalize(projectRoot))) {
+    throw new Error(
+      `basePath "${basePath}" resolves outside project root`,
+    );
   }
-
-  // Write the file to the filesystem and store
-  await fs.writeFile(path, content, "utf-8");
 }
 
-/**
- * Default asset patterns for common image and media file types
- * @internal
- */
-const DEFAULT_ASSET_PATTERNS = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.ico', '.bmp'];
+const GITHUB_IDENTIFIER_RE = /^[a-zA-Z0-9._-]+$/;
+const GITHUB_REF_RE = /^[a-zA-Z0-9._\-/]+$/;
 
 /**
- * Checks if a file path should be included and returns the matching pattern
- * @param filePath - The file path to check (relative to the repository root)
- * @param options - Import options containing includes patterns
- * @returns Object with include status and matched pattern, or null if not included
+ * Validates a GitHub owner or repo identifier.
  * @internal
  */
-export function shouldIncludeFile(filePath: string, options: ImportOptions): { included: true; matchedPattern: MatchedPattern | null } | { included: false; matchedPattern: null } {
-  const { includes } = options;
-  
-  // If no include patterns specified, include all files
-  if (!includes || includes.length === 0) {
-    return { included: true, matchedPattern: null };
+function validateGitHubIdentifier(value: string, name: string): void {
+  if (!value || value.length > 100) {
+    throw new Error(`Invalid ${name}: must be 1-100 characters`);
   }
-  
-  // Check each include pattern to find a match
-  for (let i = 0; i < includes.length; i++) {
-    const includePattern = includes[i];
-    const matcher = picomatch(includePattern.pattern);
-    
-    if (matcher(filePath)) {
-      return {
-        included: true,
-        matchedPattern: {
-          pattern: includePattern.pattern,
-          basePath: includePattern.basePath,
-          index: i
-        }
-      };
-    }
+  if (!GITHUB_IDENTIFIER_RE.test(value)) {
+    throw new Error(
+      `Invalid ${name}: "${value}" contains disallowed characters`,
+    );
   }
-  
-  // No patterns matched
-  return { included: false, matchedPattern: null };
 }
 
 /**
- * Detects asset references in markdown content using regex patterns
- * @param content - The markdown content to parse
- * @param assetPatterns - File extensions to treat as assets
- * @returns Array of detected asset paths
+ * Validates a GitHub ref (branch/tag name). More permissive than identifiers — allows `/`.
  * @internal
  */
-export function detectAssets(content: string, assetPatterns: string[] = DEFAULT_ASSET_PATTERNS): string[] {
-  const assets: string[] = [];
-  const patterns = assetPatterns.map(ext => ext.toLowerCase());
-  
-  // Match markdown images: ![alt](path)
-  const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
-  let match;
-  
-  while ((match = imageRegex.exec(content)) !== null) {
-    const assetPath = match[1];
-    // Only include relative paths and assets matching our patterns
-    if (assetPath.startsWith('./') || assetPath.startsWith('../') || !assetPath.includes('://')) {
-      const ext = extname(assetPath).toLowerCase();
-      if (patterns.includes(ext)) {
-        assets.push(assetPath);
-      }
-    }
-  }
-  
-  // Match HTML img tags: <img src="path">
-  const htmlImgRegex = /<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi;
-  while ((match = htmlImgRegex.exec(content)) !== null) {
-    const assetPath = match[1];
-    if (assetPath.startsWith('./') || assetPath.startsWith('../') || !assetPath.includes('://')) {
-      const ext = extname(assetPath).toLowerCase();
-      if (patterns.includes(ext)) {
-        assets.push(assetPath);
-      }
-    }
+function validateGitHubRef(value: string): void {
+  if (!value || value.length > 256) {
+    throw new Error(`Invalid ref: must be 1-256 characters`);
   }
-  
-  return [...new Set(assets)]; // Remove duplicates
-}
-
-/**
- * Downloads an asset from GitHub and saves it locally
- * @param octokit - GitHub API client
- * @param owner - Repository owner
- * @param repo - Repository name  
- * @param ref - Git reference
- * @param assetPath - Path to the asset in the repository
- * @param localPath - Local path where the asset should be saved
- * @param signal - Abort signal for cancellation
- * @returns Promise that resolves when the asset is downloaded
- * @internal
- */
-export async function downloadAsset(
-  octokit: any,
-  owner: string,
-  repo: string,
-  ref: string,
-  assetPath: string,
-  localPath: string,
-  signal?: AbortSignal
-): Promise<void> {
-  try {
-    const { data } = await octokit.rest.repos.getContent({
-      owner,
-      repo,
-      path: assetPath,
-      ref,
-      request: { signal },
-    });
-
-    if (Array.isArray(data) || data.type !== 'file' || !data.download_url) {
-      throw new Error(`Asset ${assetPath} is not a valid file (type: ${data.type}, downloadUrl: ${data.download_url})`);
-    }
-
-    const response = await fetch(data.download_url, { signal });
-    if (!response.ok) {
-      throw new Error(`Failed to download asset: ${response.status} ${response.statusText}`);
-    }
-
-    const buffer = await response.arrayBuffer();
-    const dir = dirname(localPath);
-
-    if (!existsSync(dir)) {
-      await fs.mkdir(dir, { recursive: true });
-    }
-
-    await fs.writeFile(localPath, new Uint8Array(buffer));
-  } catch (error: any) {
-    if (error.status === 404) {
-      throw new Error(`Asset not found: ${assetPath}`);
-    }
-    throw error;
-  }
-}
-
-/**
- * Transforms asset references in markdown content to use local paths
- * @param content - The markdown content to transform
- * @param assetMap - Map of original asset paths to new local paths
- * @returns Transformed content with updated asset references
- * @internal
- */
-export function transformAssetReferences(content: string, assetMap: Map<string, string>): string {
-  let transformedContent = content;
-  
-  for (const [originalPath, newPath] of assetMap) {
-    // Transform markdown images
-    const imageRegex = new RegExp(`(!)\\[([^\\]]*)\\]\\(\\s*${escapeRegExp(originalPath)}\\s*\\)`, 'g');
-    transformedContent = transformedContent.replace(imageRegex, `$1[$2](${newPath})`);
-    
-    // Transform HTML img tags
-    const htmlRegex = new RegExp(`(<img[^>]+src\\s*=\\s*["'])${escapeRegExp(originalPath)}(["'][^>]*>)`, 'gi');
-    transformedContent = transformedContent.replace(htmlRegex, `$1${newPath}$2`);
+  if (!GITHUB_REF_RE.test(value)) {
+    throw new Error(
+      `Invalid ref: "${value}" contains disallowed characters`,
+    );
   }
-  
-  return transformedContent;
-}
-
-/**
- * Escapes special regex characters in a string
- * @internal
- */
-function escapeRegExp(string: string): string {
-  return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
 
 /**
- * Processes assets in markdown content by detecting, downloading, and transforming references
- * @param content - The markdown content to process
- * @param options - Configuration options including asset settings
- * @param octokit - GitHub API client
- * @param signal - Abort signal for cancellation
- * @returns Promise that resolves to transformed content
+ * Collects file data by downloading content and applying transforms.
+ * Extracted from the nested closure inside toCollectionEntry for clarity.
  * @internal
  */
-async function processAssets(
-  content: string,
+async function collectFileData(
+  { url, editUrl: _editUrl }: { url: string | null; editUrl: string },
   filePath: string,
   options: ImportOptions,
-  octokit: any,
-  logger: Logger,
-  signal?: AbortSignal
-): Promise<{ content: string; assetsDownloaded: number; assetsCached: number }> {
-  const { owner, repo, ref = 'main', assetsPath, assetsBaseUrl, assetPatterns } = options;
-
-  logger.verbose(`🖼️  Processing assets for ${filePath}`);
-  logger.debug(`    assetsPath: ${assetsPath}`);
-  logger.debug(`    assetsBaseUrl: ${assetsBaseUrl}`);
-
-  if (!assetsPath || !assetsBaseUrl) {
-    logger.verbose(`    ⏭️  Skipping asset processing - missing assetsPath or assetsBaseUrl`);
-    return { content, assetsDownloaded: 0, assetsCached: 0 };
-  }
-
-  // Detect assets in the content
-  const detectedAssets = detectAssets(content, assetPatterns);
-  logger.verbose(`    📸 Detected ${detectedAssets.length} assets`);
-  if (detectedAssets.length > 0) {
-    logger.debug(`    Assets: ${detectedAssets.join(', ')}`);
-  }
-
-  if (detectedAssets.length === 0) {
-    return { content, assetsDownloaded: 0, assetsCached: 0 };
-  }
-
-  const assetMap = new Map<string, string>();
-  let assetsDownloaded = 0;
-  let assetsCached = 0;
+  context: ExtendedLoaderContext,
+  octokit: Octokit,
+  signal?: AbortSignal,
+): Promise<ImportedFile | null> {
+  const logger = context.logger;
+
+  if (url === null || typeof url !== "string") {
+    return null;
+  }
+
+  const urlObj = new URL(url);
+
+  // Determine if file needs renaming and generate appropriate ID
+  const includeCheck = shouldIncludeFile(filePath, options);
+  const matchedPattern = includeCheck.included
+    ? includeCheck.matchedPattern
+    : null;
+
+  // Check if this file has a path mapping
+  const hasPathMapping =
+    matchedPattern &&
+    options?.includes &&
+    matchedPattern.index < options.includes.length &&
+    options.includes[matchedPattern.index].pathMappings &&
+    options.includes[matchedPattern.index].pathMappings![filePath];
+
+  // Generate ID based on appropriate path
+  const id = hasPathMapping
+    ? generateId(generatePath(filePath, matchedPattern, options)) // Use path-mapped path for ID
+    : generateId(filePath); // Use original path for ID
+
+  const finalPath = generatePath(filePath, matchedPattern, options);
+  let contents: string;
+
+  logger.logFileProcessing("Fetching", filePath, `from ${urlObj.toString()}`);
+
+  // Download file content
+  const init = {
+    signal,
+    headers: getHeaders({ init: {}, meta: context.meta, id }),
+  };
+  let res: Response | null = null;
 
-  // Process each detected asset
-  await Promise.all(detectedAssets.map(async (assetPath) => {
-    logger.logAssetProcessing("Processing", assetPath);
+  // Fetch with retries (simplified version of syncEntry logic)
+  for (let attempt = 0; attempt < 3; attempt++) {
     try {
-      // Resolve the asset path relative to the current markdown file
-      const resolvedAssetPath = resolveAssetPath(filePath, assetPath);
-      logger.debug(`    🔗 Resolved path: ${resolvedAssetPath}`);
-
-      // Generate unique filename to avoid conflicts
-      const originalFilename = basename(assetPath);
-      const ext = extname(originalFilename);
-      const nameWithoutExt = basename(originalFilename, ext);
-      const uniqueFilename = `${nameWithoutExt}-${Date.now()}${ext}`;
-      const localPath = join(assetsPath, uniqueFilename);
-      logger.debug(`    💾 Local path: ${localPath}`);
-
-      // Check if asset already exists (simple cache check)
-      if (existsSync(localPath)) {
-        logger.logAssetProcessing("Cached", assetPath);
-        assetsCached++;
-      } else {
-        // Download the asset
-        logger.logAssetProcessing("Downloading", assetPath, `from ${owner}/${repo}@${ref}:${resolvedAssetPath}`);
-        await downloadAsset(octokit, owner, repo, ref, resolvedAssetPath, localPath, signal);
-        logger.logAssetProcessing("Downloaded", assetPath);
-        assetsDownloaded++;
-      }
-
-      // Generate URL for the transformed reference
-      const assetUrl = `${assetsBaseUrl}/${uniqueFilename}`.replace(/\/+/g, '/');
-      logger.debug(`    🔄 Transform: ${assetPath} -> ${assetUrl}`);
-
-      // Map the transformation
-      assetMap.set(assetPath, assetUrl);
+      res = await fetch(urlObj, init);
+      if (res.ok) break;
     } catch (error) {
-      logger.warn(`    ❌ Failed to process asset ${assetPath}: ${error}`);
+      if (attempt === 2) throw error;
+      await new Promise((resolve) =>
+        setTimeout(resolve, 1000 * (attempt + 1)),
+      );
     }
-  }));
-
-  logger.verbose(`    🗺️  Processed ${assetMap.size} assets: ${assetsDownloaded} downloaded, ${assetsCached} cached`);
-
-  // Transform the content with new asset references
-  const transformedContent = transformAssetReferences(content, assetMap);
-  return { content: transformedContent, assetsDownloaded, assetsCached };
-}
-
-/**
- * Resolves an asset path relative to a base path
- * @internal
- */
-function resolveAssetPath(basePath: string, assetPath: string): string {
-  if (assetPath.startsWith('./')) {
-    return join(dirname(basePath), assetPath.slice(2));
-  } else if (assetPath.startsWith('../')) {
-    return join(dirname(basePath), assetPath);
-  }
-  return assetPath;
-}
-
-/**
- * Synchronizes an entry by fetching its contents, validating its metadata, and storing or rendering it as needed.
- *
- * @param {LoaderContext} context - The loader context containing the required utilities, metadata, and configuration.
- * @param {Object} urls - Object containing URL data.
- * @param {string | URL | null} urls.url - The URL of the entry to fetch. Throws an error if null or invalid.
- * @param {string} urls.editUrl - The URL for editing the entry.
- * @param {RootOptions} options - Configuration settings for processing the entry such as file paths and custom options.
- * @param {any} octokit - GitHub API client for downloading assets.
- * @param {RequestInit} [init] - Optional parameter for customizing the fetch request.
- * @return {Promise<void>} Resolves when the entry has been successfully processed and stored. Throws errors if invalid URL, missing configuration, or other issues occur.
- * @internal
- */
-export async function syncEntry(
-  context: LoaderContext,
-  { url, editUrl }: { url: string | URL | null; editUrl: string },
-  filePath: string,
-  options: ImportOptions,
-  octokit: any,
-  init: RequestInit = {},
-) {
-  // Exit on null or if the URL is invalid
-  if (url === null || (typeof url !== "string" && !(url instanceof URL))) {
-    throw new TypeError(INVALID_URL_ERROR);
   }
-  // Validate URL
-  if (typeof url === "string") url = new URL(url);
 
-  const { meta, store, generateDigest, entryTypes, logger, parseData, config } =
-    context;
-
-  function configForFile(file: string) {
-    const ext = file.split(".").at(-1);
-    if (!ext) {
-      logger.warn(`No extension found for ${file}`);
-      return;
-    }
-    return entryTypes?.get(`.${ext}`);
+  if (!res) {
+    throw new Error(`No response received for ${urlObj.toString()}`);
   }
-  // Custom ID, TODO: Allow custom id generators
-  let id = generateId(filePath);
-
-  init.headers = getHeaders({
-    init: init.headers,
-    meta,
-    id,
-  });
-
-  let res = await fetch(url, init);
 
   if (res.status === 304) {
-    // Only skip if the local file actually exists
+    // File not modified, read existing content from disk if it exists
     const includeResult = shouldIncludeFile(filePath, options);
-    const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
+    const relativePath = generatePath(
+      filePath,
+      includeResult.included ? includeResult.matchedPattern : null,
+      options,
+    );
     const fileUrl = pathToFileURL(relativePath);
-    
+
     if (existsSync(fileURLToPath(fileUrl))) {
-      logger.info(`Skipping ${id} as it has not changed`);
-      return;
+      logger.logFileProcessing("Using cached", filePath, "304 not modified");
+      const { promises: fs } = await import("node:fs");
+      contents = await fs.readFile(fileURLToPath(fileUrl), "utf-8");
     } else {
-      logger.info(`File ${id} missing locally, re-fetching despite 304`);
-      // File is missing locally, fetch without ETag headers
+      // File is missing locally, re-fetch without cache headers
+      logger.logFileProcessing(
+        "Re-fetching",
+        filePath,
+        "missing locally despite 304",
+      );
       const freshInit = { ...init };
       freshInit.headers = new Headers(init.headers);
-      freshInit.headers.delete('If-None-Match');
-      freshInit.headers.delete('If-Modified-Since');
-      
-      res = await fetch(url, freshInit);
-      if (!res.ok) throw new Error(res.statusText);
+      freshInit.headers.delete("If-None-Match");
+      freshInit.headers.delete("If-Modified-Since");
+
+      res = await fetch(urlObj, freshInit);
+      if (!res.ok) {
+        throw new Error(
+          `Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || "Unknown error"}`,
+        );
+      }
+      contents = await res.text();
     }
+  } else if (!res.ok) {
+    throw new Error(
+      `Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || "Unknown error"}`,
+    );
+  } else {
+    contents = await res.text();
   }
-  if (!res.ok) throw new Error(res.statusText);
-  let contents = await res.text();
-  const entryType = configForFile(filePath || "tmp.md");
-  if (!entryType) throw new Error("No entry type found");
 
-  // Process assets FIRST if configuration is provided - before content transforms
-  // This ensures asset detection works with original markdown links before they get transformed
-  if (options.assetsPath && options.assetsBaseUrl) {
+  // Process assets FIRST if configuration is provided (or co-located defaults apply)
+  const resolvedAssetConfig = resolveAssetConfig(options, filePath);
+  if (resolvedAssetConfig) {
     try {
-      // Create a dummy logger for syncEntry since it uses Astro's logger
-      const dummyLogger = {
-        verbose: (msg: string) => logger.info(msg),
-        debug: (msg: string) => logger.debug(msg),
-        warn: (msg: string) => logger.warn(msg),
-        logAssetProcessing: (action: string, path: string, details?: string) => {
-          const msg = details ? `Asset ${action}: ${path} - ${details}` : `Asset ${action}: ${path}`;
-          logger.info(msg);
-        }
-      };
-      const assetResult = await processAssets(contents, filePath, options, octokit, dummyLogger as Logger, init.signal || undefined);
+      const optionsWithAssets = { ...options, ...resolvedAssetConfig };
+      const assetResult = await processAssets(
+        contents,
+        filePath,
+        optionsWithAssets,
+        octokit,
+        logger,
+        signal,
+      );
       contents = assetResult.content;
-    } catch (error: any) {
-      logger.warn(`Asset processing failed for ${id}: ${error.message}`);
+    } catch (error) {
+      logger.warn(
+        `Asset processing failed for ${id}: ${error instanceof Error ? error.message : String(error)}`,
+      );
     }
   }
 
-  // Apply content transforms if provided - both global and pattern-specific
-  // This runs after asset processing so transforms work with processed content
-  const includeResultForTransforms = shouldIncludeFile(filePath, options);
-  const transformsToApply: any[] = [];
+  // Apply content transforms
+  const includeResult = shouldIncludeFile(filePath, options);
+  const transformsToApply: TransformFunction[] = [];
 
   // Add global transforms first
   if (options.transforms && options.transforms.length > 0) {
@@ -544,8 +238,13 @@ export async function syncEntry(
   }
 
   // Add pattern-specific transforms
-  if (includeResultForTransforms.included && includeResultForTransforms.matchedPattern && options.includes) {
-    const matchedInclude = options.includes[includeResultForTransforms.matchedPattern.index];
+  if (
+    includeResult.included &&
+    includeResult.matchedPattern &&
+    options.includes
+  ) {
+    const matchedInclude =
+      options.includes[includeResult.matchedPattern.index];
     if (matchedInclude.transforms && matchedInclude.transforms.length > 0) {
       transformsToApply.push(...matchedInclude.transforms);
     }
@@ -556,90 +255,43 @@ export async function syncEntry(
       id,
       path: filePath,
       options,
-      matchedPattern: includeResultForTransforms.included ? includeResultForTransforms.matchedPattern : undefined,
+      matchedPattern:
+        includeResult.included && includeResult.matchedPattern
+          ? includeResult.matchedPattern
+          : undefined,
     };
 
     for (const transform of transformsToApply) {
       try {
         contents = transform(contents, transformContext);
       } catch (error) {
-        logger.warn(`Transform failed for ${id}: ${error}`);
+        context.logger?.warn(`Transform failed for ${id}: ${error}`);
       }
     }
   }
 
-  const includeResult = shouldIncludeFile(filePath, options);
-  const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
-  const fileUrl = pathToFileURL(relativePath);
-  const { body, data } = await entryType.getEntryInfo({
-    contents,
-    fileUrl: fileUrl,
-  });
-
-  const existingEntry = store.get(id);
-
-  const digest = generateDigest(contents);
-
-  if (
-    existingEntry &&
-    existingEntry.digest === digest &&
-    existingEntry.filePath
-  ) {
-    return;
-  }
-  // Write file to path
-  if (!existsSync(fileURLToPath(fileUrl))) {
-    (logger as any).verbose(`Writing ${id} to ${fileUrl}`);
-    await syncFile(fileURLToPath(fileUrl), contents);
-  }
-
-  const parsedData = await parseData({
-    id,
-    data,
-    filePath: fileUrl.toString(),
-  });
-
-  if (entryType.getRenderFunction) {
-    (logger as any).verbose(`Rendering ${id}`);
-    const render = await entryType.getRenderFunction(config);
-    let rendered: RenderedContent | undefined = undefined;
-    try {
-      rendered = await render?.({
-        id,
-        data,
-        body,
-        filePath: fileUrl.toString(),
-        digest,
-      });
-    } catch (error: any) {
-      logger.error(`Error rendering ${id}: ${error.message}`);
-    }
-    store.set({
-      id,
-      data: parsedData,
-      body,
-      filePath: relativePath,
-      digest,
-      rendered,
-    });
-  } else if ("contentModuleTypes" in entryType) {
-    store.set({
-      id,
-      data: parsedData,
-      body,
-      filePath: relativePath,
-      digest,
-      deferredRender: true,
-    });
-  } else {
-    store.set({ id, data: parsedData, body, filePath: relativePath, digest });
-  }
+  // Build link context for this file
+  const linkContext =
+    includeResult.included && includeResult.matchedPattern
+      ? {
+          sourcePath: filePath,
+          targetPath: finalPath,
+          basePath: includeResult.matchedPattern.basePath,
+          pathMappings:
+            options.includes?.[includeResult.matchedPattern.index]
+              ?.pathMappings,
+          matchedPattern: includeResult.matchedPattern,
+        }
+      : undefined;
 
-  syncHeaders({
-    headers: res.headers,
-    meta,
+  // Use the finalPath we already computed
+  return {
+    sourcePath: filePath,
+    targetPath: finalPath,
+    content: contents,
     id,
-  });
+    linkContext,
+  };
 }
 
 /**
@@ -660,8 +312,23 @@ export async function toCollectionEntry({
   if (typeof repo !== "string" || typeof owner !== "string")
     throw new TypeError(INVALID_STRING_ERROR);
 
-  // Get logger from context - it should be our Logger instance (initialize early)
-  const logger = context.logger as unknown as Logger;
+  // Validate identifiers to prevent injection into API calls / URLs
+  validateGitHubIdentifier(owner, "owner");
+  validateGitHubIdentifier(repo, "repo");
+  if (ref !== "main") validateGitHubRef(ref);
+
+  // Validate include pattern basePaths don't escape the project
+  const projectRoot = process.cwd();
+  if (options.includes) {
+    for (const inc of options.includes) {
+      validateBasePath(inc.basePath, projectRoot);
+    }
+  }
+  if (options.assetsPath) {
+    validateBasePath(options.assetsPath, projectRoot);
+  }
+
+  const logger = context.logger;
 
   /**
    * OPTIMIZATION: Use Git Trees API for efficient file discovery
@@ -693,7 +360,7 @@ export async function toCollectionEntry({
     repo,
     sha: ref,
     per_page: 1,
-    request: { signal }
+    request: { signal },
   });
 
   if (commits.length === 0) {
@@ -711,19 +378,23 @@ export async function toCollectionEntry({
     repo,
     tree_sha: treeSha,
     recursive: "true",
-    request: { signal }
+    request: { signal },
   });
 
   logger.debug(`Retrieved ${treeData.tree.length} items from repository tree`);
 
   // Filter tree to only include files (not dirs/submodules) that match our patterns
-  const fileEntries = treeData.tree.filter((item: any) => {
-    if (item.type !== 'blob') return false; // Only process files (blobs)
-    const includeCheck = shouldIncludeFile(item.path, options);
-    return includeCheck.included;
-  });
-
-  logger.info(`Found ${fileEntries.length} files matching include patterns (filtered from ${treeData.tree.length} total items)`);
+  const fileEntries = treeData.tree.filter(
+    (item: { type?: string; path?: string }) => {
+      if (item.type !== "blob") return false; // Only process files (blobs)
+      const includeCheck = shouldIncludeFile(item.path!, options);
+      return includeCheck.included;
+    },
+  );
+
+  logger.info(
+    `Found ${fileEntries.length} files matching include patterns (filtered from ${treeData.tree.length} total items)`,
+  );
 
   // Collect all files first (with content transforms applied)
   const allFiles: ImportedFile[] = [];
@@ -731,12 +402,17 @@ export async function toCollectionEntry({
   for (const treeItem of fileEntries) {
     const filePath = treeItem.path;
     // Construct the download URL (raw.githubusercontent.com format)
-    const downloadUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${commitSha}/${filePath}`;
-    const editUrl = treeItem.url || ''; // Git blob URL (use empty string as fallback)
+    const encodedPath = filePath.split("/").map(encodeURIComponent).join("/");
+    const downloadUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${commitSha}/${encodedPath}`;
+    const editUrl = treeItem.url || ""; // Git blob URL (use empty string as fallback)
 
     const fileData = await collectFileData(
       { url: downloadUrl, editUrl },
-      filePath
+      filePath,
+      options,
+      context,
+      octokit,
+      signal,
     );
 
     if (fileData) {
@@ -760,16 +436,21 @@ export async function toCollectionEntry({
 
     // Generate automatic link mappings from pathMappings
     const autoGeneratedMappings = options.includes
-      ? generateAutoLinkMappings(options.includes, options.linkTransform.stripPrefixes)
+      ? generateAutoLinkMappings(
+          options.includes,
+          options.linkTransform.stripPrefixes,
+        )
       : [];
 
     // Combine auto-generated mappings with user-defined mappings
     const allLinkMappings = [
       ...autoGeneratedMappings,
-      ...(options.linkTransform.linkMappings || [])
+      ...(options.linkTransform.linkMappings || []),
     ];
 
-    logger.debug(`Generated ${autoGeneratedMappings.length} automatic link mappings from pathMappings`);
+    logger.debug(
+      `Generated ${autoGeneratedMappings.length} automatic link mappings from pathMappings`,
+    );
 
     processedFiles = globalLinkTransform(allFiles, {
       stripPrefixes: options.linkTransform.stripPrefixes,
@@ -792,383 +473,4 @@ export async function toCollectionEntry({
   }
 
   return stats;
-
-  // Helper function to collect file data with content transforms applied
-  async function collectFileData(
-    { url, editUrl }: { url: string | null; editUrl: string },
-    filePath: string
-  ): Promise<ImportedFile | null> {
-    if (url === null || typeof url !== "string") {
-      return null;
-    }
-
-    const urlObj = new URL(url);
-
-    // Determine if file needs renaming and generate appropriate ID
-    const includeCheck = shouldIncludeFile(filePath, options);
-    const matchedPattern = includeCheck.included ? includeCheck.matchedPattern : null;
-
-    // Check if this file has a path mapping
-    const hasPathMapping = matchedPattern &&
-      options?.includes &&
-      matchedPattern.index < options.includes.length &&
-      options.includes[matchedPattern.index].pathMappings &&
-      options.includes[matchedPattern.index].pathMappings![filePath];
-
-    // Generate ID based on appropriate path
-    const id = hasPathMapping ?
-      generateId(generatePath(filePath, matchedPattern, options)) : // Use path-mapped path for ID
-      generateId(filePath); // Use original path for ID
-
-    const finalPath = generatePath(filePath, matchedPattern, options);
-    let contents: string;
-
-    logger.logFileProcessing("Fetching", filePath, `from ${urlObj.toString()}`);
-
-    // Download file content
-    const init = { signal, headers: getHeaders({ init: {}, meta: context.meta, id }) };
-    let res: Response | null = null;
-
-    // Fetch with retries (simplified version of syncEntry logic)
-    for (let attempt = 0; attempt < 3; attempt++) {
-      try {
-        res = await fetch(urlObj, init);
-        if (res.ok) break;
-      } catch (error) {
-        if (attempt === 2) throw error;
-        await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
-      }
-    }
-
-    if (!res) {
-      throw new Error(`No response received for ${urlObj.toString()}`);
-    }
-
-    if (res.status === 304) {
-      // File not modified, read existing content from disk if it exists
-      const includeResult = shouldIncludeFile(filePath, options);
-      const relativePath = generatePath(filePath, includeResult.included ? includeResult.matchedPattern : null, options);
-      const fileUrl = pathToFileURL(relativePath);
-
-      if (existsSync(fileURLToPath(fileUrl))) {
-        logger.logFileProcessing("Using cached", filePath, "304 not modified");
-        const { promises: fs } = await import('node:fs');
-        contents = await fs.readFile(fileURLToPath(fileUrl), 'utf-8');
-      } else {
-        // File is missing locally, re-fetch without cache headers
-        logger.logFileProcessing("Re-fetching", filePath, "missing locally despite 304");
-        const freshInit = { ...init };
-        freshInit.headers = new Headers(init.headers);
-        freshInit.headers.delete('If-None-Match');
-        freshInit.headers.delete('If-Modified-Since');
-
-        res = await fetch(urlObj, freshInit);
-        if (!res.ok) {
-          throw new Error(`Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || 'Unknown error'}`);
-        }
-        contents = await res.text();
-      }
-    } else if (!res.ok) {
-      throw new Error(`Failed to fetch file content from ${urlObj.toString()}: ${res.status} ${res.statusText || 'Unknown error'}`);
-    } else {
-      contents = await res.text();
-    }
-
-    // Process assets FIRST if configuration is provided
-    let fileAssetsDownloaded = 0;
-    let fileAssetsCached = 0;
-    if (options.assetsPath && options.assetsBaseUrl) {
-      try {
-        const assetResult = await processAssets(contents, filePath, options, octokit, logger, signal);
-        contents = assetResult.content;
-        fileAssetsDownloaded = assetResult.assetsDownloaded;
-        fileAssetsCached = assetResult.assetsCached;
-      } catch (error) {
-        logger.warn(`Asset processing failed for ${id}: ${error instanceof Error ? error.message : String(error)}`);
-      }
-    }
-
-    // Apply content transforms
-    const includeResult = shouldIncludeFile(filePath, options);
-    const transformsToApply: any[] = [];
-
-    // Add global transforms first
-    if (options.transforms && options.transforms.length > 0) {
-      transformsToApply.push(...options.transforms);
-    }
-
-    // Add pattern-specific transforms
-    if (includeResult.included && includeResult.matchedPattern && options.includes) {
-      const matchedInclude = options.includes[includeResult.matchedPattern.index];
-      if (matchedInclude.transforms && matchedInclude.transforms.length > 0) {
-        transformsToApply.push(...matchedInclude.transforms);
-      }
-    }
-
-    if (transformsToApply.length > 0) {
-      const transformContext = {
-        id,
-        path: filePath,
-        options,
-        matchedPattern: includeResult.included ? includeResult.matchedPattern : undefined,
-      };
-
-      for (const transform of transformsToApply) {
-        try {
-          contents = transform(contents, transformContext);
-        } catch (error) {
-          context.logger?.warn(`Transform failed for ${id}: ${error}`);
-        }
-      }
-    }
-
-    // Build link context for this file
-    const linkContext = includeResult.included && includeResult.matchedPattern ? {
-      sourcePath: filePath,
-      targetPath: finalPath,
-      basePath: includeResult.matchedPattern.basePath,
-      pathMappings: options.includes?.[includeResult.matchedPattern.index]?.pathMappings,
-      matchedPattern: includeResult.matchedPattern,
-    } : undefined;
-
-    // Use the finalPath we already computed
-    return {
-      sourcePath: filePath,
-      targetPath: finalPath,
-      content: contents,
-      id,
-      linkContext,
-    };
-  }
-
-  // Helper function to store a processed file
-  async function storeProcessedFile(
-    file: ImportedFile,
-    context: any,
-    clear: boolean
-  ): Promise<any> {
-    const { store, generateDigest, entryTypes, logger, parseData, config } = context;
-
-    function configForFile(filePath: string) {
-      const ext = filePath.split(".").at(-1);
-      if (!ext) {
-        logger.warn(`No extension found for ${filePath}`);
-        return;
-      }
-      return entryTypes?.get(`.${ext}`);
-    }
-
-    const entryType = configForFile(file.sourcePath || "tmp.md");
-    if (!entryType) throw new Error("No entry type found");
-
-    const fileUrl = pathToFileURL(file.targetPath);
-    const { body, data } = await entryType.getEntryInfo({
-      contents: file.content,
-      fileUrl: fileUrl,
-    });
-
-    // Generate digest for storage (repository-level caching handles change detection)
-    const digest = generateDigest(file.content);
-    const existingEntry = store.get(file.id);
-
-    if (existingEntry) {
-      logger.debug(`🔄 File ${file.id} - updating`);
-    } else {
-      logger.debug(`📄 File ${file.id} - adding`);
-    }
-
-    // Write file to disk
-    if (!existsSync(fileURLToPath(fileUrl))) {
-      logger.verbose(`Writing ${file.id} to ${fileUrl}`);
-      await syncFile(fileURLToPath(fileUrl), file.content);
-    }
-
-    const parsedData = await parseData({
-      id: file.id,
-      data,
-      filePath: fileUrl.toString(),
-    });
-
-    // When clear mode is enabled, delete the existing entry before setting the new one.
-    // This provides atomic replacement without breaking Astro's content collection,
-    // as opposed to calling store.clear() which empties everything at once.
-    if (clear && existingEntry) {
-      logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
-      store.delete(file.id);
-    }
-
-    // Store in content store
-    if (entryType.getRenderFunction) {
-      logger.verbose(`Rendering ${file.id}`);
-      const render = await entryType.getRenderFunction(config);
-      let rendered = undefined;
-      try {
-        rendered = await render?.({
-          id: file.id,
-          data,
-          body,
-          filePath: fileUrl.toString(),
-          digest,
-        });
-      } catch (error: any) {
-        logger.error(`Error rendering ${file.id}: ${error.message}`);
-      }
-      logger.debug(`🔍 Storing collection entry: ${file.id} (${file.sourcePath} -> ${file.targetPath})`);
-      store.set({
-        id: file.id,
-        data: parsedData,
-        body,
-        filePath: file.targetPath,
-        digest,
-        rendered,
-      });
-    } else if ("contentModuleTypes" in entryType) {
-      store.set({
-        id: file.id,
-        data: parsedData,
-        body,
-        filePath: file.targetPath,
-        digest,
-        deferredRender: true,
-      });
-    } else {
-      store.set({
-        id: file.id,
-        data: parsedData,
-        body,
-        filePath: file.targetPath,
-        digest
-      });
-    }
-
-    return { id: file.id, filePath: file.targetPath };
-  }
-
-  async function processDirectoryRecursively(path: string): Promise<any> {
-    // Fetch the content
-    const { data, status } = await octokit.rest.repos.getContent({
-      owner,
-      repo,
-      path,
-      ref,
-      request: { signal },
-    });
-    if (status !== 200) throw new Error(INVALID_SERVICE_RESPONSE);
-
-    // Matches for regular files
-    if (!Array.isArray(data)) {
-      const filePath = data.path;
-      switch (data.type) {
-        // Return
-        case "file":
-          return await syncEntry(
-            context,
-            { url: data.download_url, editUrl: data.url },
-            filePath,
-            options,
-            octokit,
-            { signal },
-          );
-        default:
-          throw new Error("Invalid type");
-      }
-    }
-
-    // Directory listing with filtering - process sequentially
-    const filteredEntries = data
-      .filter(({ type, path }) => {
-        // Always include directories for recursion
-        if (type === "dir") return true;
-        // Apply filtering logic to files
-        if (type === "file") {
-          return shouldIncludeFile(path, options).included;
-        }
-        return false;
-      });
-    
-    const results = [];
-    for (const { type, path, download_url, url } of filteredEntries) {
-      switch (type) {
-        // Recurse
-        case "dir":
-          results.push(await processDirectoryRecursively(path));
-          break;
-        // Return
-        case "file":
-          results.push(await syncEntry(
-            context,
-            { url: download_url, editUrl: url },
-            path,
-            options,
-            octokit,
-            { signal },
-          ));
-          break;
-        default:
-          throw new Error("Invalid type");
-      }
-    }
-    return results;
-  } // End of processDirectoryRecursively function
 }
-
-
-
-/**
- * Get the headers needed to make a conditional request.
- * Uses the etag and last-modified values from the meta store.
- * @internal
- */
-export function getHeaders({
-                                        init,
-                                        meta,
-                                        id,
-                                      }: {
-  /** Initial headers to include */
-  init?: RequestInit["headers"];
-  /** Meta store to get etag and last-modified values from */
-  meta: LoaderContext["meta"];
-  id: string;
-}): Headers {
-  const tag = `${id}-etag`;
-  const lastModifiedTag = `${id}-last-modified`;
-  const etag = meta.get(tag);
-  const lastModified = meta.get(lastModifiedTag);
-  const headers = new Headers(init);
-
-  if (etag) {
-    headers.set("If-None-Match", etag);
-  } else if (lastModified) {
-    headers.set("If-Modified-Since", lastModified);
-  }
-  return headers;
-}
-
-/**
- * Store the etag or last-modified headers from a response in the meta store.
- * @internal
- */
-export function syncHeaders({
-                                          headers,
-                                          meta,
-                                          id,
-                                        }: {
-  /** Headers from the response */
-  headers: Headers;
-  /** Meta store to store etag and last-modified values in */
-  meta: LoaderContext["meta"];
-  /** id string */
-  id: string;
-}) {
-  const etag = headers.get("etag");
-  const lastModified = headers.get("last-modified");
-  const tag = `${id}-etag`;
-  const lastModifiedTag = `${id}-last-modified`;
-  meta.delete(tag);
-  meta.delete(lastModifiedTag);
-  if (etag) {
-    meta.set(tag, etag);
-  } else if (lastModified) {
-    meta.set(lastModifiedTag, lastModified);
-  }
-}