@larkiny/astro-github-loader 0.9.0

package/dist/github.logger.d.ts

@@ -0,0 +1,132 @@
+/**
+ * Multi-level logging system for astro-github-loader
+ */
+export type LogLevel = 'silent' | 'default' | 'verbose' | 'debug';
+export interface LoggerOptions {
+    level: LogLevel;
+    prefix?: string;
+}
+export interface ImportSummary {
+    configName: string;
+    repository: string;
+    ref?: string;
+    filesProcessed: number;
+    filesUpdated: number;
+    filesUnchanged: number;
+    assetsDownloaded?: number;
+    assetsCached?: number;
+    duration: number;
+    status: 'success' | 'error' | 'cancelled';
+    error?: string;
+}
+export interface SyncSummary {
+    added: number;
+    updated: number;
+    deleted: number;
+    unchanged: number;
+    duration: number;
+}
+export interface CleanupSummary {
+    deleted: number;
+    duration: number;
+}
+/**
+ * Centralized logger with configurable verbosity levels and spinner support for long-running operations
+ */
+export declare class Logger {
+    private level;
+    private prefix;
+    private spinnerInterval?;
+    private spinnerChars;
+    private spinnerIndex;
+    private spinnerStartTime?;
+    constructor(options: LoggerOptions);
+    /**
+     * Set the logging level
+     */
+    setLevel(level: LogLevel): void;
+    /**
+     * Get the current logging level
+     */
+    getLevel(): LogLevel;
+    /**
+     * Check if a specific level should be logged
+     */
+    private shouldLog;
+    /**
+     * Format message with prefix
+     */
+    private formatMessage;
+    /**
+     * Silent level - no output
+     */
+    silent(): void;
+    /**
+     * Default level - summary information only
+     */
+    info(message: string): void;
+    /**
+     * Verbose level - detailed operation information
+     */
+    verbose(message: string): void;
+    /**
+     * Debug level - all information including diagnostics
+     */
+    debug(message: string): void;
+    /**
+     * Error - always shown unless silent
+     */
+    error(message: string): void;
+    /**
+     * Warning - shown at default level and above
+     */
+    warn(message: string): void;
+    /**
+     * Log structured import summary (default level)
+     */
+    logImportSummary(summary: ImportSummary): void;
+    /**
+     * Log sync operation summary (default level)
+     */
+    logSyncSummary(configName: string, summary: SyncSummary): void;
+    /**
+     * Log cleanup operation summary (default level)
+     */
+    logCleanupSummary(configName: string, summary: CleanupSummary): void;
+    /**
+     * Log file-level processing (verbose level)
+     */
+    logFileProcessing(action: string, filePath: string, details?: string): void;
+    /**
+     * Log asset processing (verbose level)
+     */
+    logAssetProcessing(action: string, assetPath: string, details?: string): void;
+    /**
+     * Create a child logger with additional prefix
+     */
+    child(prefix: string): Logger;
+    /**
+     * Time a function execution and log the result
+     */
+    time<T>(label: string, fn: () => Promise<T>): Promise<T>;
+    /**
+     * Format duration in human-readable format
+     */
+    private formatDuration;
+    /**
+     * Start a spinner with duration timer for long-running operations
+     */
+    startSpinner(message?: string): void;
+    /**
+     * Stop the spinner and optionally show a final message
+     */
+    stopSpinner(finalMessage?: string): void;
+    /**
+     * Execute a function with spinner feedback
+     */
+    withSpinner<T>(message: string, fn: () => Promise<T>, successMessage?: string, errorMessage?: string): Promise<T>;
+}
+/**
+ * Create a logger instance with the specified level
+ */
+export declare function createLogger(level?: LogLevel, prefix?: string): Logger;

package/dist/github.logger.js

@@ -0,0 +1,260 @@
+/**
+ * Multi-level logging system for astro-github-loader
+ */
+/**
+ * Centralized logger with configurable verbosity levels and spinner support for long-running operations
+ */
+export class Logger {
+    constructor(options) {
+        this.spinnerChars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+        this.spinnerIndex = 0;
+        this.level = options.level;
+        this.prefix = options.prefix || '';
+    }
+    /**
+     * Set the logging level
+     */
+    setLevel(level) {
+        this.level = level;
+    }
+    /**
+     * Get the current logging level
+     */
+    getLevel() {
+        return this.level;
+    }
+    /**
+     * Check if a specific level should be logged
+     */
+    shouldLog(level) {
+        const levels = {
+            silent: 0,
+            default: 1,
+            verbose: 2,
+            debug: 3,
+        };
+        return levels[this.level] >= levels[level];
+    }
+    /**
+     * Format message with prefix
+     */
+    formatMessage(message) {
+        return this.prefix ? `${this.prefix} ${message}` : message;
+    }
+    /**
+     * Silent level - no output
+     */
+    silent() {
+        // Intentionally empty
+    }
+    /**
+     * Default level - summary information only
+     */
+    info(message) {
+        if (this.shouldLog('default')) {
+            console.log(this.formatMessage(message));
+        }
+    }
+    /**
+     * Verbose level - detailed operation information
+     */
+    verbose(message) {
+        if (this.shouldLog('verbose')) {
+            console.log(this.formatMessage(message));
+        }
+    }
+    /**
+     * Debug level - all information including diagnostics
+     */
+    debug(message) {
+        if (this.shouldLog('debug')) {
+            console.log(this.formatMessage(message));
+        }
+    }
+    /**
+     * Error - always shown unless silent
+     */
+    error(message) {
+        if (this.shouldLog('default')) {
+            console.error(this.formatMessage(message));
+        }
+    }
+    /**
+     * Warning - shown at default level and above
+     */
+    warn(message) {
+        if (this.shouldLog('default')) {
+            console.warn(this.formatMessage(message));
+        }
+    }
+    /**
+     * Log structured import summary (default level)
+     */
+    logImportSummary(summary) {
+        if (!this.shouldLog('default'))
+            return;
+        const statusIcon = summary.status === 'success' ? '✅' : summary.status === 'error' ? '❌' : '🚫';
+        this.info('');
+        this.info(`📊 Import Summary: ${summary.configName}`);
+        this.info(`├─ Repository: ${summary.repository}${summary.ref ? `@${summary.ref}` : ''}`);
+        this.info(`├─ Files: ${summary.filesProcessed} processed, ${summary.filesUpdated} updated, ${summary.filesUnchanged} unchanged`);
+        if (summary.assetsDownloaded !== undefined || summary.assetsCached !== undefined) {
+            const downloaded = summary.assetsDownloaded || 0;
+            const cached = summary.assetsCached || 0;
+            this.info(`├─ Assets: ${downloaded} downloaded, ${cached} cached`);
+        }
+        this.info(`├─ Duration: ${(summary.duration / 1000).toFixed(1)}s`);
+        this.info(`└─ Status: ${statusIcon} ${summary.status === 'success' ? 'Success' : summary.status === 'error' ? `Error: ${summary.error}` : 'Cancelled'}`);
+        this.info('');
+    }
+    /**
+     * Log sync operation summary (default level)
+     */
+    logSyncSummary(configName, summary) {
+        if (!this.shouldLog('default'))
+            return;
+        if (summary.added > 0 || summary.updated > 0 || summary.deleted > 0) {
+            this.info(`Sync completed for ${configName}: ${summary.added} added, ${summary.updated} updated, ${summary.deleted} deleted (${summary.duration}ms)`);
+        }
+        else {
+            this.info(`No changes needed for ${configName} (${summary.duration}ms)`);
+        }
+    }
+    /**
+     * Log cleanup operation summary (default level)
+     */
+    logCleanupSummary(configName, summary) {
+        if (!this.shouldLog('default'))
+            return;
+        if (summary.deleted > 0) {
+            this.info(`Cleanup completed for ${configName}: ${summary.deleted} obsolete files deleted (${summary.duration}ms)`);
+        }
+        else {
+            this.debug(`No cleanup needed for ${configName} (${summary.duration}ms)`);
+        }
+    }
+    /**
+     * Log file-level processing (verbose level)
+     */
+    logFileProcessing(action, filePath, details) {
+        const message = details ? `${action}: ${filePath} - ${details}` : `${action}: ${filePath}`;
+        this.verbose(message);
+    }
+    /**
+     * Log asset processing (verbose level)
+     */
+    logAssetProcessing(action, assetPath, details) {
+        const message = details ? `Asset ${action}: ${assetPath} - ${details}` : `Asset ${action}: ${assetPath}`;
+        this.verbose(message);
+    }
+    /**
+     * Create a child logger with additional prefix
+     */
+    child(prefix) {
+        return new Logger({
+            level: this.level,
+            prefix: this.prefix ? `${this.prefix}${prefix}` : prefix,
+        });
+    }
+    /**
+     * Time a function execution and log the result
+     */
+    async time(label, fn) {
+        const startTime = Date.now();
+        this.debug(`⏱️  Starting: ${label}`);
+        try {
+            const result = await fn();
+            const duration = Date.now() - startTime;
+            this.verbose(`✅ Completed: ${label} (${duration}ms)`);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            this.error(`❌ Failed: ${label} (${duration}ms): ${error}`);
+            throw error;
+        }
+    }
+    /**
+     * Format duration in human-readable format
+     */
+    formatDuration(seconds) {
+        if (seconds < 60) {
+            return `${seconds}s`;
+        }
+        else if (seconds < 3600) {
+            const mins = Math.floor(seconds / 60);
+            const secs = seconds % 60;
+            return `${mins}m ${secs}s`;
+        }
+        else {
+            const hours = Math.floor(seconds / 3600);
+            const mins = Math.floor((seconds % 3600) / 60);
+            return `${hours}h ${mins}m`;
+        }
+    }
+    /**
+     * Start a spinner with duration timer for long-running operations
+     */
+    startSpinner(message = 'Processing...') {
+        if (this.level === 'silent')
+            return;
+        this.spinnerStartTime = Date.now();
+        this.spinnerIndex = 0;
+        const updateSpinner = () => {
+            const elapsed = Math.floor((Date.now() - this.spinnerStartTime) / 1000);
+            const spinner = this.spinnerChars[this.spinnerIndex];
+            const duration = this.formatDuration(elapsed);
+            const formattedMessage = this.formatMessage(`${message} ${spinner} (${duration})`);
+            process.stdout.write(`\r${formattedMessage}`);
+            this.spinnerIndex = (this.spinnerIndex + 1) % this.spinnerChars.length;
+        };
+        // Initial display
+        updateSpinner();
+        // Update every 100ms
+        this.spinnerInterval = setInterval(updateSpinner, 100);
+    }
+    /**
+     * Stop the spinner and optionally show a final message
+     */
+    stopSpinner(finalMessage) {
+        if (this.spinnerInterval) {
+            clearInterval(this.spinnerInterval);
+            this.spinnerInterval = undefined;
+        }
+        if (finalMessage && this.spinnerStartTime) {
+            const totalTime = Math.floor((Date.now() - this.spinnerStartTime) / 1000);
+            const duration = this.formatDuration(totalTime);
+            const formattedMessage = this.formatMessage(`${finalMessage} (${duration})`);
+            process.stdout.write(`\r${formattedMessage}\n`);
+        }
+        else if (finalMessage) {
+            const formattedMessage = this.formatMessage(finalMessage);
+            process.stdout.write(`\r${formattedMessage}\n`);
+        }
+        else {
+            process.stdout.write('\r\x1b[K'); // Clear the line
+        }
+        this.spinnerStartTime = undefined;
+    }
+    /**
+     * Execute a function with spinner feedback
+     */
+    async withSpinner(message, fn, successMessage, errorMessage) {
+        this.startSpinner(message);
+        try {
+            const result = await fn();
+            this.stopSpinner(successMessage || `✅ ${message.replace(/^[🔄⏳]?\s*/, '')} completed`);
+            return result;
+        }
+        catch (error) {
+            this.stopSpinner(errorMessage || `❌ ${message.replace(/^[🔄⏳]?\s*/, '')} failed`);
+            throw error;
+        }
+    }
+}
+/**
+ * Create a logger instance with the specified level
+ */
+export function createLogger(level = 'default', prefix) {
+    return new Logger({ level, prefix });
+}

package/dist/github.sync.d.ts

@@ -0,0 +1,5 @@
+import type { ImportOptions, SyncStats, LoaderContext } from "./github.types.js";
+/**
+ * Performs incremental sync for a single import configuration
+ */
+export declare function performIncrementalSync(config: ImportOptions, context: LoaderContext, octokit: any, signal?: AbortSignal): Promise<SyncStats>;

package/dist/github.sync.js

@@ -0,0 +1,292 @@
+import { promises as fs } from "node:fs";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import { generateId, generatePath, shouldIncludeFile, syncEntry } from "./github.content.js";
+const MANIFEST_FILENAME = '.astro-github-manifest.json';
+const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
+/**
+ * Creates a hash of the configuration to detect changes
+ */
+function createConfigHash(options) {
+    const configForHashing = {
+        owner: options.owner,
+        repo: options.repo,
+        ref: options.ref,
+        includes: options.includes
+    };
+    return createHash('md5').update(JSON.stringify(configForHashing)).digest('hex');
+}
+/**
+ * Loads the sync manifest from disk
+ */
+async function loadManifest(basePath) {
+    const manifestPath = join(basePath, MANIFEST_FILENAME);
+    if (!existsSync(manifestPath)) {
+        return {
+            files: {},
+            lastSync: new Date().toISOString()
+        };
+    }
+    try {
+        const content = await fs.readFile(manifestPath, 'utf-8');
+        return JSON.parse(content);
+    }
+    catch (error) {
+        console.warn(`Failed to load manifest from ${manifestPath}, starting fresh:`, error);
+        return {
+            files: {},
+            lastSync: new Date().toISOString()
+        };
+    }
+}
+/**
+ * Saves the sync manifest to disk
+ */
+async function saveManifest(basePath, manifest) {
+    const manifestPath = join(basePath, MANIFEST_FILENAME);
+    // Ensure directory exists
+    if (!existsSync(basePath)) {
+        await fs.mkdir(basePath, { recursive: true });
+    }
+    try {
+        await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+    }
+    catch (error) {
+        console.warn(`Failed to save manifest to ${manifestPath}:`, error);
+    }
+}
+/**
+ * Discovers all files in a GitHub repository directory
+ */
+async function discoverRemoteFiles(octokit, options, signal) {
+    const { owner, repo, ref = "main" } = options;
+    const files = new Map();
+    // Get all unique directory prefixes from include patterns to limit scanning
+    const directoriesToScan = new Set();
+    if (options.includes && options.includes.length > 0) {
+        for (const includePattern of options.includes) {
+            // Extract directory part from pattern (before any glob wildcards)
+            const pattern = includePattern.pattern;
+            const beforeGlob = pattern.split(/[*?{]/)[0];
+            const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+            directoriesToScan.add(dirPart);
+        }
+    }
+    else {
+        // If no includes specified, scan from root
+        directoriesToScan.add('');
+    }
+    async function processDirectory(dirPath) {
+        try {
+            const { data } = await octokit.rest.repos.getContent({
+                owner,
+                repo,
+                path: dirPath,
+                ref,
+                request: { signal }
+            });
+            if (!Array.isArray(data)) {
+                // Single file
+                if (data.type === 'file' && shouldIncludeFile(data.path, options).included) {
+                    const id = generateId(data.path);
+                    const includeResult = shouldIncludeFile(data.path, options);
+                    const localPath = generatePath(data.path, includeResult.included ? includeResult.matchedPattern : null, options);
+                    files.set(id, {
+                        path: data.path,
+                        localPath,
+                        lastModified: data.last_modified || undefined,
+                        etag: data.sha // Use SHA as ETag equivalent
+                    });
+                }
+                return;
+            }
+            // Directory listing
+            const promises = data
+                .filter(({ type, path }) => {
+                if (type === "dir")
+                    return true;
+                if (type === "file")
+                    return shouldIncludeFile(path, options).included;
+                return false;
+            })
+                .map(async ({ type, path: itemPath }) => {
+                if (type === "dir") {
+                    await processDirectory(itemPath);
+                }
+                else if (type === "file") {
+                    const id = generateId(itemPath);
+                    const includeResult = shouldIncludeFile(itemPath, options);
+                    const localPath = generatePath(itemPath, includeResult.included ? includeResult.matchedPattern : null, options);
+                    files.set(id, {
+                        path: itemPath,
+                        localPath,
+                        etag: data.find(item => item.path === itemPath)?.sha
+                    });
+                }
+            });
+            await Promise.all(promises);
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            console.warn(`Failed to process directory ${dirPath}:`, error);
+        }
+    }
+    // Process only the directories that match our include patterns
+    for (const dirPath of directoriesToScan) {
+        await processDirectory(dirPath);
+    }
+    return files;
+}
+/**
+ * Creates a sync plan by comparing remote files with the local manifest
+ */
+async function createSyncPlan(remoteFiles, manifest, options) {
+    const plan = {
+        toAdd: [],
+        toUpdate: [],
+        toDelete: [],
+        unchanged: []
+    };
+    // Check remote files against local manifest
+    for (const [id, remoteEntry] of remoteFiles) {
+        const localEntry = manifest.files[id];
+        if (!localEntry) {
+            // New file
+            plan.toAdd.push(remoteEntry);
+        }
+        else if (remoteEntry.etag !== localEntry.etag ||
+            remoteEntry.lastModified !== localEntry.lastModified ||
+            !existsSync(localEntry.localPath)) {
+            // Changed or missing local file
+            plan.toUpdate.push({ ...remoteEntry, localPath: localEntry.localPath });
+        }
+        else {
+            // Unchanged
+            plan.unchanged.push(localEntry);
+        }
+    }
+    // Check for files to delete (in local manifest but not in remote)
+    for (const [id, localEntry] of Object.entries(manifest.files)) {
+        if (!remoteFiles.has(id)) {
+            plan.toDelete.push(localEntry);
+        }
+    }
+    return plan;
+}
+/**
+ * Sleep utility for pacing file operations
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Executes the sync plan with proper pacing to avoid Astro issues
+ */
+async function executeSyncPlan(plan, options, context, octokit, signal) {
+    const { logger } = context;
+    // Delete obsolete files first (with pacing)
+    for (const entry of plan.toDelete) {
+        try {
+            if (existsSync(entry.localPath)) {
+                await fs.unlink(entry.localPath);
+                logger.debug(`Deleted ${entry.localPath}`);
+            }
+            await sleep(SLEEP_BETWEEN_DELETES);
+        }
+        catch (error) {
+            logger.warn(`Failed to delete ${entry.localPath}: ${error}`);
+        }
+    }
+    // Process additions and updates (can be done in parallel)
+    const processFile = async (entry) => {
+        try {
+            const { owner, repo, ref = "main" } = options;
+            const { data } = await octokit.rest.repos.getContent({
+                owner,
+                repo,
+                path: entry.path,
+                ref,
+                request: { signal }
+            });
+            if (Array.isArray(data) || data.type !== 'file' || !data.download_url) {
+                throw new Error(`${entry.path} is not a valid file`);
+            }
+            await syncEntry(context, { url: data.download_url, editUrl: data.url }, entry.path, options, octokit, { signal });
+            logger.debug(`Synced ${entry.path} -> ${entry.localPath}`);
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            logger.error(`Failed to sync ${entry.path}: ${error}`);
+        }
+    };
+    // Process additions and updates with controlled concurrency
+    const allFilesToProcess = [...plan.toAdd, ...plan.toUpdate];
+    const concurrency = 5; // Limit concurrent operations
+    for (let i = 0; i < allFilesToProcess.length; i += concurrency) {
+        const batch = allFilesToProcess.slice(i, i + concurrency);
+        await Promise.all(batch.map(processFile));
+    }
+}
+/**
+ * Performs incremental sync for a single import configuration
+ */
+export async function performIncrementalSync(config, context, octokit, signal) {
+    const startTime = Date.now();
+    const { logger } = context;
+    const configName = config.name || `${config.owner}/${config.repo}`;
+    if (!config.includes || config.includes.length === 0) {
+        throw new Error(`includes patterns are required for incremental sync in config: ${configName}`);
+    }
+    logger.debug(`Starting incremental sync for ${configName}`);
+    try {
+        // Load existing manifest (using first include pattern's base path)
+        const manifestPath = config.includes[0].basePath;
+        const manifest = await loadManifest(manifestPath);
+        // Check if config changed (force full sync if it did)
+        const currentConfigHash = createConfigHash(config);
+        const configChanged = manifest.configHash && manifest.configHash !== currentConfigHash;
+        if (configChanged) {
+            logger.info(`Configuration changed for ${configName}, performing full sync`);
+            manifest.files = {}; // Clear manifest to force full re-sync
+        }
+        // Discover remote files
+        const remoteFiles = await discoverRemoteFiles(octokit, config, signal);
+        // Create sync plan
+        const plan = await createSyncPlan(remoteFiles, manifest, config);
+        // Execute the sync plan
+        await executeSyncPlan(plan, config, context, octokit, signal);
+        // Update manifest
+        const newManifest = {
+            files: Object.fromEntries(remoteFiles),
+            lastSync: new Date().toISOString(),
+            configHash: currentConfigHash
+        };
+        await saveManifest(manifestPath, newManifest);
+        const duration = Date.now() - startTime;
+        const stats = {
+            added: plan.toAdd.length,
+            updated: plan.toUpdate.length,
+            deleted: plan.toDelete.length,
+            unchanged: plan.unchanged.length,
+            duration
+        };
+        // Log summary
+        logger.info(`Sync completed for ${configName}: ` +
+            `${stats.added} added, ${stats.updated} updated, ` +
+            `${stats.deleted} deleted, ${stats.unchanged} unchanged ` +
+            `(${duration}ms)`);
+        return stats;
+    }
+    catch (error) {
+        if (signal?.aborted) {
+            logger.info(`Sync cancelled for ${configName}`);
+            throw error;
+        }
+        const duration = Date.now() - startTime;
+        logger.error(`Sync failed for ${configName} after ${duration}ms: ${error}`);
+        throw error;
+    }
+}