@larkiny/astro-github-loader 0.11.3 → 0.12.0

package/dist/github.cleanup.js

@@ -1,18 +1,18 @@
 import { promises as fs } from "node:fs";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { generateId, generatePath, shouldIncludeFile } from "./github.content.js";
+import { generateId, generatePath, shouldIncludeFile, } from "./github.content.js";
 const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
 /**
  * Sleep utility for pacing file operations
  */
 function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
+    return new Promise((resolve) => setTimeout(resolve, ms));
 }
 /**
  * Gets all files that should exist locally based on remote repository state
  */
-async function getExpectedFiles(octokit, options, signal) {
+async function getExpectedFiles(octokit, options, logger, signal) {
     const { owner, repo, ref = "main" } = options;
     const expectedFiles = new Set();
     // Get all unique directory prefixes from include patterns to limit scanning
@@ -22,13 +22,15 @@ async function getExpectedFiles(octokit, options, signal) {
             // Extract directory part from pattern (before any glob wildcards)
             const pattern = includePattern.pattern;
             const beforeGlob = pattern.split(/[*?{]/)[0];
-            const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+            const dirPart = beforeGlob.includes("/")
+                ? beforeGlob.substring(0, beforeGlob.lastIndexOf("/"))
+                : "";
             directoriesToScan.add(dirPart);
         }
     }
     else {
         // If no includes specified, scan from root
-        directoriesToScan.add('');
+        directoriesToScan.add("");
     }
     async function processDirectory(dirPath) {
         try {
@@ -37,16 +39,19 @@ async function getExpectedFiles(octokit, options, signal) {
                 repo,
                 path: dirPath,
                 ref,
-                request: { signal }
+                request: { signal },
             });
             if (!Array.isArray(data)) {
                 // Single file
-                if (data.type === 'file' && shouldIncludeFile(data.path, options).included) {
+                if (data.type === "file" &&
+                    shouldIncludeFile(data.path, options).included) {
                     const id = generateId(data.path);
                     const includeResult = shouldIncludeFile(data.path, options);
                     const localPath = generatePath(data.path, includeResult.included ? includeResult.matchedPattern : null, options);
                     // Convert to absolute path for consistent comparison
-                    const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+                    const absolutePath = localPath.startsWith("/")
+                        ? localPath
+                        : join(process.cwd(), localPath);
                     expectedFiles.add(absolutePath);
                 }
                 return;
@@ -69,7 +74,9 @@ async function getExpectedFiles(octokit, options, signal) {
                     const includeResult = shouldIncludeFile(itemPath, options);
                     const localPath = generatePath(itemPath, includeResult.included ? includeResult.matchedPattern : null, options);
                     // Convert to absolute path for consistent comparison
-                    const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+                    const absolutePath = localPath.startsWith("/")
+                        ? localPath
+                        : join(process.cwd(), localPath);
                     expectedFiles.add(absolutePath);
                 }
             });
@@ -78,7 +85,7 @@ async function getExpectedFiles(octokit, options, signal) {
         catch (error) {
             if (signal?.aborted)
                 throw error;
-            console.warn(`Failed to process directory ${dirPath}:`, error);
+            logger.warn(`Failed to process directory ${dirPath}: ${error}`);
         }
     }
     // Process only the directories that match our include patterns
@@ -90,7 +97,7 @@ async function getExpectedFiles(octokit, options, signal) {
 /**
  * Gets all existing local files in the basePath as absolute paths
  */
-async function getExistingFiles(basePath) {
+async function getExistingFiles(basePath, logger) {
     const existingFiles = new Set();
     if (!existsSync(basePath)) {
         return existingFiles;
@@ -102,20 +109,20 @@ async function getExistingFiles(basePath) {
                 const fullPath = join(dirPath, entry.name);
                 if (entry.isDirectory()) {
                     // Skip manifest files and other system directories
-                    if (!entry.name.startsWith('.')) {
+                    if (!entry.name.startsWith(".")) {
                         await walkDirectory(fullPath);
                     }
                 }
                 else if (entry.isFile()) {
                     // Skip manifest and system files
-                    if (!entry.name.startsWith('.')) {
+                    if (!entry.name.startsWith(".")) {
                         existingFiles.add(fullPath);
                     }
                 }
             }
         }
         catch (error) {
-            console.warn(`Failed to read directory ${dirPath}:`, error);
+            logger.warn(`Failed to read directory ${dirPath}: ${error}`);
         }
     }
     await walkDirectory(basePath);
@@ -135,7 +142,7 @@ export async function performSelectiveCleanup(config, context, octokit, signal)
             updated: 0,
             deleted: 0,
             unchanged: 0,
-            duration: Date.now() - startTime
+            duration: Date.now() - startTime,
         };
     }
     logger.debug(`Starting selective cleanup for ${configName}`);
@@ -143,8 +150,8 @@ export async function performSelectiveCleanup(config, context, octokit, signal)
         // Get existing local files from all include pattern base paths
         const allExistingFiles = new Set();
         for (const includePattern of config.includes) {
-            const existingFiles = await getExistingFiles(includePattern.basePath);
-            existingFiles.forEach(file => allExistingFiles.add(file));
+            const existingFiles = await getExistingFiles(includePattern.basePath, logger);
+            existingFiles.forEach((file) => allExistingFiles.add(file));
         }
         // If no existing files, skip cleanup (fresh import)
         if (allExistingFiles.size === 0) {
@@ -154,11 +161,11 @@ export async function performSelectiveCleanup(config, context, octokit, signal)
                 updated: 0,
                 deleted: 0,
                 unchanged: 0,
-                duration: Date.now() - startTime
+                duration: Date.now() - startTime,
             };
         }
         // Get expected files from remote repository
-        const expectedFiles = await getExpectedFiles(octokit, config, signal);
+        const expectedFiles = await getExpectedFiles(octokit, config, logger, signal);
         // Find files to delete (exist locally but not in remote)
         const filesToDelete = [];
         for (const existingFile of allExistingFiles) {
@@ -184,10 +191,10 @@ export async function performSelectiveCleanup(config, context, octokit, signal)
         const duration = Date.now() - startTime;
         const stats = {
             added: 0, // Will be counted by main sync process
-            updated: 0, // Will be counted by main sync process  
+            updated: 0, // Will be counted by main sync process
             deleted: deletedCount,
             unchanged: 0, // Will be counted by main sync process
-            duration
+            duration,
         };
         if (deletedCount > 0) {
             logger.info(`Cleanup completed for ${configName}: ${deletedCount} obsolete files deleted (${duration}ms)`);
@@ -203,14 +210,14 @@ export async function performSelectiveCleanup(config, context, octokit, signal)
             throw error;
         }
         const duration = Date.now() - startTime;
-        logger.error(`Cleanup failed for ${configName} after ${duration}ms: ${error}`);
+        logger.error(`Cleanup failed for ${configName} after ${duration}ms: ${error instanceof Error ? error.message : String(error)}`);
         // Don't throw - let the main sync process continue
         return {
             added: 0,
             updated: 0,
             deleted: 0,
             unchanged: 0,
-            duration
+            duration,
         };
     }
 }

package/dist/github.constants.d.ts

@@ -6,19 +6,3 @@
  * @internal
  */
 export declare const INVALID_STRING_ERROR = "Invalid string";
-/**
- * Represents an error message indicating that a provided URL is invalid.
- * This constant is typically used for validation or error handling when a URL
- * does not conform to the expected format or requirements.
- *
- * @internal
- */
-export declare const INVALID_URL_ERROR = "Invalid url";
-/**
- * A constant that holds a default error message indicating that a service response is invalid.
- * This value is typically used to signify that the response from a service or API call
- * does not meet the expected format, structure, or criteria.
- *
- * @internal
- */
-export declare const INVALID_SERVICE_RESPONSE = "Invalid service response";

package/dist/github.constants.js

@@ -6,19 +6,3 @@
  * @internal
  */
 export const INVALID_STRING_ERROR = "Invalid string";
-/**
- * Represents an error message indicating that a provided URL is invalid.
- * This constant is typically used for validation or error handling when a URL
- * does not conform to the expected format or requirements.
- *
- * @internal
- */
-export const INVALID_URL_ERROR = "Invalid url";
-/**
- * A constant that holds a default error message indicating that a service response is invalid.
- * This value is typically used to signify that the response from a service or API call
- * does not meet the expected format, structure, or criteria.
- *
- * @internal
- */
-export const INVALID_SERVICE_RESPONSE = "Invalid service response";

package/dist/github.content.d.ts

@@ -1,110 +1,8 @@
-import type { LoaderContext, CollectionEntryOptions, ImportOptions, MatchedPattern } from "./github.types.js";
-export interface ImportStats {
-    processed: number;
-    updated: number;
-    unchanged: number;
-    assetsDownloaded?: number;
-    assetsCached?: number;
-}
-/**
- * Generates a unique identifier from a file path by removing the extension
- * @param filePath - The file path to generate ID from
- * @return {string} The generated identifier as a string with extension removed
- * @internal
- */
-export declare function generateId(filePath: string): string;
-/**
- * Applies path mapping logic to get the final filename for a file
- *
- * Supports two types of path mappings:
- * - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
- * - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
- *
- * @param filePath - Original source file path
- * @param matchedPattern - The pattern that matched this file
- * @param options - Import options containing path mappings
- * @returns Final filename after applying path mapping logic
- * @internal
- */
-export declare function applyRename(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
-/**
- * Generates a local file path based on the matched pattern and file path
- * @param filePath - The original file path from the repository
- * @param matchedPattern - The pattern that matched this file (or null if no includes specified)
- * @param options - Import options containing includes patterns for path mapping lookups
- * @return {string} The local file path where this content should be stored
- * @internal
- */
-export declare function generatePath(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
-/**
- * Synchronizes a file by ensuring the target directory exists and then writing the specified content to the file at the given path.
- *
- * @param {string} path - The path of the file to synchronize, including its directory and filename.
- * @param {string} content - The content to write into the file.
- * @return {Promise<void>} - A promise that resolves when the file has been successfully written.
- * @internal
- */
-export declare function syncFile(path: string, content: string): Promise<void>;
-/**
- * Checks if a file path should be included and returns the matching pattern
- * @param filePath - The file path to check (relative to the repository root)
- * @param options - Import options containing includes patterns
- * @returns Object with include status and matched pattern, or null if not included
- * @internal
- */
-export declare function shouldIncludeFile(filePath: string, options: ImportOptions): {
-    included: true;
-    matchedPattern: MatchedPattern | null;
-} | {
-    included: false;
-    matchedPattern: null;
-};
-/**
- * Detects asset references in markdown content using regex patterns
- * @param content - The markdown content to parse
- * @param assetPatterns - File extensions to treat as assets
- * @returns Array of detected asset paths
- * @internal
- */
-export declare function detectAssets(content: string, assetPatterns?: string[]): string[];
-/**
- * Downloads an asset from GitHub and saves it locally
- * @param octokit - GitHub API client
- * @param owner - Repository owner
- * @param repo - Repository name
- * @param ref - Git reference
- * @param assetPath - Path to the asset in the repository
- * @param localPath - Local path where the asset should be saved
- * @param signal - Abort signal for cancellation
- * @returns Promise that resolves when the asset is downloaded
- * @internal
- */
-export declare function downloadAsset(octokit: any, owner: string, repo: string, ref: string, assetPath: string, localPath: string, signal?: AbortSignal): Promise<void>;
-/**
- * Transforms asset references in markdown content to use local paths
- * @param content - The markdown content to transform
- * @param assetMap - Map of original asset paths to new local paths
- * @returns Transformed content with updated asset references
- * @internal
- */
-export declare function transformAssetReferences(content: string, assetMap: Map<string, string>): string;
-/**
- * Synchronizes an entry by fetching its contents, validating its metadata, and storing or rendering it as needed.
- *
- * @param {LoaderContext} context - The loader context containing the required utilities, metadata, and configuration.
- * @param {Object} urls - Object containing URL data.
- * @param {string | URL | null} urls.url - The URL of the entry to fetch. Throws an error if null or invalid.
- * @param {string} urls.editUrl - The URL for editing the entry.
- * @param {RootOptions} options - Configuration settings for processing the entry such as file paths and custom options.
- * @param {any} octokit - GitHub API client for downloading assets.
- * @param {RequestInit} [init] - Optional parameter for customizing the fetch request.
- * @return {Promise<void>} Resolves when the entry has been successfully processed and stored. Throws errors if invalid URL, missing configuration, or other issues occur.
- * @internal
- */
-export declare function syncEntry(context: LoaderContext, { url, editUrl }: {
-    url: string | URL | null;
-    editUrl: string;
-}, filePath: string, options: ImportOptions, octokit: any, init?: RequestInit): Promise<void>;
+import type { CollectionEntryOptions } from "./github.types.js";
+import { type ImportStats } from "./github.paths.js";
+export { type ImportStats, generateId, generatePath, shouldIncludeFile, applyRename, getHeaders, syncHeaders, } from "./github.paths.js";
+export { syncFile } from "./github.storage.js";
+export { resolveAssetConfig, detectAssets, downloadAsset, transformAssetReferences, } from "./github.assets.js";
 /**
  * Converts a given GitHub repository path into a collection entry by fetching the content
  * from the GitHub repository using the provided Octokit instance and options.
@@ -112,27 +10,3 @@ export declare function syncEntry(context: LoaderContext, { url, editUrl }: {
  * @internal
  */
 export declare function toCollectionEntry({ context, octokit, options, signal, force, clear, }: CollectionEntryOptions): Promise<ImportStats>;
-/**
- * Get the headers needed to make a conditional request.
- * Uses the etag and last-modified values from the meta store.
- * @internal
- */
-export declare function getHeaders({ init, meta, id, }: {
-    /** Initial headers to include */
-    init?: RequestInit["headers"];
-    /** Meta store to get etag and last-modified values from */
-    meta: LoaderContext["meta"];
-    id: string;
-}): Headers;
-/**
- * Store the etag or last-modified headers from a response in the meta store.
- * @internal
- */
-export declare function syncHeaders({ headers, meta, id, }: {
-    /** Headers from the response */
-    headers: Headers;
-    /** Meta store to store etag and last-modified values in */
-    meta: LoaderContext["meta"];
-    /** id string */
-    id: string;
-}): void;