@larkiny/astro-github-loader 0.9.0

package/dist/github.cleanup.js

@@ -0,0 +1,216 @@
+import { promises as fs } from "node:fs";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { generateId, generatePath, shouldIncludeFile } from "./github.content.js";
+const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
+/**
+ * Sleep utility for pacing file operations
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Gets all files that should exist locally based on remote repository state
+ */
+async function getExpectedFiles(octokit, options, signal) {
+    const { owner, repo, ref = "main" } = options;
+    const expectedFiles = new Set();
+    // Get all unique directory prefixes from include patterns to limit scanning
+    const directoriesToScan = new Set();
+    if (options.includes && options.includes.length > 0) {
+        for (const includePattern of options.includes) {
+            // Extract directory part from pattern (before any glob wildcards)
+            const pattern = includePattern.pattern;
+            const beforeGlob = pattern.split(/[*?{]/)[0];
+            const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+            directoriesToScan.add(dirPart);
+        }
+    }
+    else {
+        // If no includes specified, scan from root
+        directoriesToScan.add('');
+    }
+    async function processDirectory(dirPath) {
+        try {
+            const { data } = await octokit.rest.repos.getContent({
+                owner,
+                repo,
+                path: dirPath,
+                ref,
+                request: { signal }
+            });
+            if (!Array.isArray(data)) {
+                // Single file
+                if (data.type === 'file' && shouldIncludeFile(data.path, options).included) {
+                    const id = generateId(data.path);
+                    const includeResult = shouldIncludeFile(data.path, options);
+                    const localPath = generatePath(data.path, includeResult.included ? includeResult.matchedPattern : null, options);
+                    // Convert to absolute path for consistent comparison
+                    const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+                    expectedFiles.add(absolutePath);
+                }
+                return;
+            }
+            // Directory listing
+            const promises = data
+                .filter(({ type, path }) => {
+                if (type === "dir")
+                    return true;
+                if (type === "file")
+                    return shouldIncludeFile(path, options).included;
+                return false;
+            })
+                .map(async ({ type, path: itemPath }) => {
+                if (type === "dir") {
+                    await processDirectory(itemPath);
+                }
+                else if (type === "file") {
+                    const id = generateId(itemPath);
+                    const includeResult = shouldIncludeFile(itemPath, options);
+                    const localPath = generatePath(itemPath, includeResult.included ? includeResult.matchedPattern : null, options);
+                    // Convert to absolute path for consistent comparison
+                    const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+                    expectedFiles.add(absolutePath);
+                }
+            });
+            await Promise.all(promises);
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            console.warn(`Failed to process directory ${dirPath}:`, error);
+        }
+    }
+    // Process only the directories that match our include patterns
+    for (const dirPath of directoriesToScan) {
+        await processDirectory(dirPath);
+    }
+    return expectedFiles;
+}
+/**
+ * Gets all existing local files in the basePath as absolute paths
+ */
+async function getExistingFiles(basePath) {
+    const existingFiles = new Set();
+    if (!existsSync(basePath)) {
+        return existingFiles;
+    }
+    async function walkDirectory(dirPath) {
+        try {
+            const entries = await fs.readdir(dirPath, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = join(dirPath, entry.name);
+                if (entry.isDirectory()) {
+                    // Skip manifest files and other system directories
+                    if (!entry.name.startsWith('.')) {
+                        await walkDirectory(fullPath);
+                    }
+                }
+                else if (entry.isFile()) {
+                    // Skip manifest and system files
+                    if (!entry.name.startsWith('.')) {
+                        existingFiles.add(fullPath);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Failed to read directory ${dirPath}:`, error);
+        }
+    }
+    await walkDirectory(basePath);
+    return existingFiles;
+}
+/**
+ * Performs selective cleanup of obsolete files
+ */
+export async function performSelectiveCleanup(config, context, octokit, signal) {
+    const startTime = Date.now();
+    const { logger } = context;
+    const configName = config.name || `${config.owner}/${config.repo}`;
+    if (!config.includes || config.includes.length === 0) {
+        // No cleanup needed if no include patterns specified
+        return {
+            added: 0,
+            updated: 0,
+            deleted: 0,
+            unchanged: 0,
+            duration: Date.now() - startTime
+        };
+    }
+    logger.debug(`Starting selective cleanup for ${configName}`);
+    try {
+        // Get existing local files from all include pattern base paths
+        const allExistingFiles = new Set();
+        for (const includePattern of config.includes) {
+            const existingFiles = await getExistingFiles(includePattern.basePath);
+            existingFiles.forEach(file => allExistingFiles.add(file));
+        }
+        // If no existing files, skip cleanup (fresh import)
+        if (allExistingFiles.size === 0) {
+            logger.debug(`No existing files found in any base paths, skipping cleanup`);
+            return {
+                added: 0,
+                updated: 0,
+                deleted: 0,
+                unchanged: 0,
+                duration: Date.now() - startTime
+            };
+        }
+        // Get expected files from remote repository
+        const expectedFiles = await getExpectedFiles(octokit, config, signal);
+        // Find files to delete (exist locally but not in remote)
+        const filesToDelete = [];
+        for (const existingFile of allExistingFiles) {
+            if (!expectedFiles.has(existingFile)) {
+                filesToDelete.push(existingFile);
+            }
+        }
+        // Delete obsolete files with pacing
+        let deletedCount = 0;
+        for (const filePath of filesToDelete) {
+            try {
+                if (existsSync(filePath)) {
+                    await fs.unlink(filePath);
+                    logger.debug(`Deleted obsolete file: ${filePath}`);
+                    deletedCount++;
+                    await sleep(SLEEP_BETWEEN_DELETES);
+                }
+            }
+            catch (error) {
+                logger.warn(`Failed to delete ${filePath}: ${error}`);
+            }
+        }
+        const duration = Date.now() - startTime;
+        const stats = {
+            added: 0, // Will be counted by main sync process
+            updated: 0, // Will be counted by main sync process  
+            deleted: deletedCount,
+            unchanged: 0, // Will be counted by main sync process
+            duration
+        };
+        if (deletedCount > 0) {
+            logger.info(`Cleanup completed for ${configName}: ${deletedCount} obsolete files deleted (${duration}ms)`);
+        }
+        else {
+            logger.debug(`No cleanup needed for ${configName} (${duration}ms)`);
+        }
+        return stats;
+    }
+    catch (error) {
+        if (signal?.aborted) {
+            logger.info(`Cleanup cancelled for ${configName}`);
+            throw error;
+        }
+        const duration = Date.now() - startTime;
+        logger.error(`Cleanup failed for ${configName} after ${duration}ms: ${error}`);
+        // Don't throw - let the main sync process continue
+        return {
+            added: 0,
+            updated: 0,
+            deleted: 0,
+            unchanged: 0,
+            duration
+        };
+    }
+}

package/dist/github.constants.d.ts

@@ -0,0 +1,24 @@
+/**
+ * This variable represents an error message indicating that the provided string is invalid.
+ * It is typically used to flag inputs or data that do not meet the required string format or criteria.
+ * The value is a constant string: 'Invalid string'.
+ *
+ * @internal
+ */
+export declare const INVALID_STRING_ERROR = "Invalid string";
+/**
+ * Represents an error message indicating that a provided URL is invalid.
+ * This constant is typically used for validation or error handling when a URL
+ * does not conform to the expected format or requirements.
+ *
+ * @internal
+ */
+export declare const INVALID_URL_ERROR = "Invalid url";
+/**
+ * A constant that holds a default error message indicating that a service response is invalid.
+ * This value is typically used to signify that the response from a service or API call
+ * does not meet the expected format, structure, or criteria.
+ *
+ * @internal
+ */
+export declare const INVALID_SERVICE_RESPONSE = "Invalid service response";

package/dist/github.constants.js

@@ -0,0 +1,24 @@
+/**
+ * This variable represents an error message indicating that the provided string is invalid.
+ * It is typically used to flag inputs or data that do not meet the required string format or criteria.
+ * The value is a constant string: 'Invalid string'.
+ *
+ * @internal
+ */
+export const INVALID_STRING_ERROR = "Invalid string";
+/**
+ * Represents an error message indicating that a provided URL is invalid.
+ * This constant is typically used for validation or error handling when a URL
+ * does not conform to the expected format or requirements.
+ *
+ * @internal
+ */
+export const INVALID_URL_ERROR = "Invalid url";
+/**
+ * A constant that holds a default error message indicating that a service response is invalid.
+ * This value is typically used to signify that the response from a service or API call
+ * does not meet the expected format, structure, or criteria.
+ *
+ * @internal
+ */
+export const INVALID_SERVICE_RESPONSE = "Invalid service response";

package/dist/github.content.d.ts

@@ -0,0 +1,138 @@
+import type { LoaderContext, CollectionEntryOptions, ImportOptions, MatchedPattern } from "./github.types.js";
+export interface ImportStats {
+    processed: number;
+    updated: number;
+    unchanged: number;
+    assetsDownloaded?: number;
+    assetsCached?: number;
+}
+/**
+ * Generates a unique identifier from a file path by removing the extension
+ * @param filePath - The file path to generate ID from
+ * @return {string} The generated identifier as a string with extension removed
+ * @internal
+ */
+export declare function generateId(filePath: string): string;
+/**
+ * Applies path mapping logic to get the final filename for a file
+ *
+ * Supports two types of path mappings:
+ * - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
+ * - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
+ *
+ * @param filePath - Original source file path
+ * @param matchedPattern - The pattern that matched this file
+ * @param options - Import options containing path mappings
+ * @returns Final filename after applying path mapping logic
+ * @internal
+ */
+export declare function applyRename(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
+/**
+ * Generates a local file path based on the matched pattern and file path
+ * @param filePath - The original file path from the repository
+ * @param matchedPattern - The pattern that matched this file (or null if no includes specified)
+ * @param options - Import options containing includes patterns for path mapping lookups
+ * @return {string} The local file path where this content should be stored
+ * @internal
+ */
+export declare function generatePath(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
+/**
+ * Synchronizes a file by ensuring the target directory exists and then writing the specified content to the file at the given path.
+ *
+ * @param {string} path - The path of the file to synchronize, including its directory and filename.
+ * @param {string} content - The content to write into the file.
+ * @return {Promise<void>} - A promise that resolves when the file has been successfully written.
+ * @internal
+ */
+export declare function syncFile(path: string, content: string): Promise<void>;
+/**
+ * Checks if a file path should be included and returns the matching pattern
+ * @param filePath - The file path to check (relative to the repository root)
+ * @param options - Import options containing includes patterns
+ * @returns Object with include status and matched pattern, or null if not included
+ * @internal
+ */
+export declare function shouldIncludeFile(filePath: string, options: ImportOptions): {
+    included: true;
+    matchedPattern: MatchedPattern | null;
+} | {
+    included: false;
+    matchedPattern: null;
+};
+/**
+ * Detects asset references in markdown content using regex patterns
+ * @param content - The markdown content to parse
+ * @param assetPatterns - File extensions to treat as assets
+ * @returns Array of detected asset paths
+ * @internal
+ */
+export declare function detectAssets(content: string, assetPatterns?: string[]): string[];
+/**
+ * Downloads an asset from GitHub and saves it locally
+ * @param octokit - GitHub API client
+ * @param owner - Repository owner
+ * @param repo - Repository name
+ * @param ref - Git reference
+ * @param assetPath - Path to the asset in the repository
+ * @param localPath - Local path where the asset should be saved
+ * @param signal - Abort signal for cancellation
+ * @returns Promise that resolves when the asset is downloaded
+ * @internal
+ */
+export declare function downloadAsset(octokit: any, owner: string, repo: string, ref: string, assetPath: string, localPath: string, signal?: AbortSignal): Promise<void>;
+/**
+ * Transforms asset references in markdown content to use local paths
+ * @param content - The markdown content to transform
+ * @param assetMap - Map of original asset paths to new local paths
+ * @returns Transformed content with updated asset references
+ * @internal
+ */
+export declare function transformAssetReferences(content: string, assetMap: Map<string, string>): string;
+/**
+ * Synchronizes an entry by fetching its contents, validating its metadata, and storing or rendering it as needed.
+ *
+ * @param {LoaderContext} context - The loader context containing the required utilities, metadata, and configuration.
+ * @param {Object} urls - Object containing URL data.
+ * @param {string | URL | null} urls.url - The URL of the entry to fetch. Throws an error if null or invalid.
+ * @param {string} urls.editUrl - The URL for editing the entry.
+ * @param {RootOptions} options - Configuration settings for processing the entry such as file paths and custom options.
+ * @param {any} octokit - GitHub API client for downloading assets.
+ * @param {RequestInit} [init] - Optional parameter for customizing the fetch request.
+ * @return {Promise<void>} Resolves when the entry has been successfully processed and stored. Throws errors if invalid URL, missing configuration, or other issues occur.
+ * @internal
+ */
+export declare function syncEntry(context: LoaderContext, { url, editUrl }: {
+    url: string | URL | null;
+    editUrl: string;
+}, filePath: string, options: ImportOptions, octokit: any, init?: RequestInit): Promise<void>;
+/**
+ * Converts a given GitHub repository path into a collection entry by fetching the content
+ * from the GitHub repository using the provided Octokit instance and options.
+ * Handles both files and directories, recursively processing directories if needed.
+ * @internal
+ */
+export declare function toCollectionEntry({ context, octokit, options, signal, force, }: CollectionEntryOptions): Promise<ImportStats>;
+/**
+ * Get the headers needed to make a conditional request.
+ * Uses the etag and last-modified values from the meta store.
+ * @internal
+ */
+export declare function getHeaders({ init, meta, id, }: {
+    /** Initial headers to include */
+    init?: RequestInit["headers"];
+    /** Meta store to get etag and last-modified values from */
+    meta: LoaderContext["meta"];
+    id: string;
+}): Headers;
+/**
+ * Store the etag or last-modified headers from a response in the meta store.
+ * @internal
+ */
+export declare function syncHeaders({ headers, meta, id, }: {
+    /** Headers from the response */
+    headers: Headers;
+    /** Meta store to store etag and last-modified values in */
+    meta: LoaderContext["meta"];
+    /** id string */
+    id: string;
+}): void;