@larkiny/astro-github-loader 0.11.2 → 0.12.0

package/src/github.loader.ts

@@ -1,48 +1,26 @@
 import { toCollectionEntry } from "./github.content.js";
 import { performSelectiveCleanup } from "./github.cleanup.js";
-import { performDryRun, displayDryRunResults, updateImportState, loadImportState, createConfigId, getLatestCommitInfo } from "./github.dryrun.js";
-import { createLogger, type Logger, type ImportSummary } from "./github.logger.js";
+import {
+  performDryRun,
+  displayDryRunResults,
+  updateImportState,
+  loadImportState,
+  createConfigId,
+  getLatestCommitInfo,
+} from "./github.dryrun.js";
+import {
+  createLogger,
+  type Logger,
+  type ImportSummary,
+} from "./github.logger.js";
 
 import type {
   Loader,
   GithubLoaderOptions,
+  ExtendedLoaderContext,
   ImportOptions,
-  SyncStats,
 } from "./github.types.js";
 
-/**
- * Performs selective cleanup for configurations with basePath
- * @param configs - Array of configuration objects
- * @param context - Loader context  
- * @param octokit - GitHub API client
- * @internal
- */
-async function performSelectiveCleanups(
-  configs: ImportOptions[],
-  context: any,
-  octokit: any
-): Promise<SyncStats[]> {
-  const results: SyncStats[] = [];
-  
-  // Process each config sequentially to avoid overwhelming Astro's file watcher
-  for (const config of configs) {
-    if (config.enabled === false) {
-      context.logger.debug(`Skipping disabled config: ${config.name || `${config.owner}/${config.repo}`}`);
-      continue;
-    }
-
-    try {
-      const stats = await performSelectiveCleanup(config, context, octokit);
-      results.push(stats);
-    } catch (error: any) {
-      context.logger.error(`Selective cleanup failed for ${config.name || `${config.owner}/${config.repo}`}: ${error}`);
-      // Continue with other configs even if one fails
-    }
-  }
-  
-  return results;
-}
-
 /**
  * Loads data from GitHub repositories based on the provided configurations and options.
  *
@@ -67,13 +45,13 @@ export function githubLoader({
   return {
     name: "github-loader",
     load: async (context) => {
-      const { store } = context;
-
       // Create global logger with specified level or default
-      const globalLogger = createLogger(logLevel || 'default');
+      const globalLogger = createLogger(logLevel || "default");
 
       if (dryRun) {
-        globalLogger.info("🔍 Dry run mode enabled - checking for changes only");
+        globalLogger.info(
+          "🔍 Dry run mode enabled - checking for changes only",
+        );
 
         try {
           const results = await performDryRun(configs, context, octokit);
@@ -83,40 +61,48 @@ export function githubLoader({
           globalLogger.info("💡 Set dryRun: false to perform actual imports");
 
           return; // Exit without importing
-        } catch (error: any) {
-          globalLogger.error(`Dry run failed: ${error.message}`);
+        } catch (error: unknown) {
+          globalLogger.error(
+            `Dry run failed: ${error instanceof Error ? error.message : String(error)}`,
+          );
           throw error;
         }
       }
 
       globalLogger.debug(`Loading data from ${configs.length} sources`);
 
-      // Always use standard processing - no file deletions to avoid Astro issues
-      globalLogger.info(clear ? "Processing with content store clear" : "Processing without content store clear");
-
-      if (clear) {
-        store.clear();
-      }
+      // Log clear mode status - actual clearing happens per-entry in toCollectionEntry
+      // to avoid breaking Astro's content collection by emptying the store all at once
+      globalLogger.info(
+        clear
+          ? "Processing with selective entry replacement"
+          : "Processing without entry replacement",
+      );
 
       // Process each config sequentially to avoid overwhelming GitHub API/CDN
       for (let i = 0; i < configs.length; i++) {
         const config = configs[i];
 
         if (config.enabled === false) {
-          globalLogger.debug(`Skipping disabled config: ${config.name || `${config.owner}/${config.repo}`}`);
+          globalLogger.debug(
+            `Skipping disabled config: ${config.name || `${config.owner}/${config.repo}`}`,
+          );
           continue;
         }
 
         // Add small delay between configs to be gentler on GitHub's CDN
         if (i > 0) {
-          await new Promise(resolve => setTimeout(resolve, 1000));
+          await new Promise((resolve) => setTimeout(resolve, 1000));
         }
 
         // Determine the effective log level for this config
-        const effectiveLogLevel = logLevel || config.logLevel || 'default';
+        const effectiveLogLevel = logLevel || config.logLevel || "default";
         const configLogger = createLogger(effectiveLogLevel);
 
-        const configName = config.name || `${config.owner}/${config.repo}`;
+        const langSuffix = config.language ? ` (${config.language})` : "";
+        const configName = config.name
+          ? `${config.name}${langSuffix}`
+          : `${config.owner}/${config.repo}${langSuffix}`;
         const repository = `${config.owner}/${config.repo}`;
 
         let summary: ImportSummary = {
@@ -127,7 +113,7 @@ export function githubLoader({
           filesUpdated: 0,
           filesUnchanged: 0,
           duration: 0,
-          status: 'error',
+          status: "error",
         };
 
         const startTime = Date.now();
@@ -138,51 +124,92 @@ export function githubLoader({
 
           if (!force) {
             try {
-              const state = await loadImportState(process.cwd());
+              const state = await loadImportState(process.cwd(), configLogger);
               const currentState = state.imports[configId];
 
               if (currentState && currentState.lastCommitSha) {
-                configLogger.debug(`🔍 Checking repository changes for ${configName}...`);
+                configLogger.debug(
+                  `🔍 Checking repository changes for ${configName}...`,
+                );
                 const latestCommit = await getLatestCommitInfo(octokit, config);
 
-                if (latestCommit && currentState.lastCommitSha === latestCommit.sha) {
-                  configLogger.info(`✅ Repository ${configName} unchanged (${latestCommit.sha.slice(0, 7)}) - skipping import`);
+                if (
+                  latestCommit &&
+                  currentState.lastCommitSha === latestCommit.sha
+                ) {
+                  configLogger.info(
+                    `✅ Repository ${configName} unchanged (${latestCommit.sha.slice(0, 7)}) - skipping import`,
+                  );
 
                   // Update summary for unchanged repository
                   summary.duration = Date.now() - startTime;
                   summary.filesProcessed = 0;
                   summary.filesUpdated = 0;
                   summary.filesUnchanged = 0;
-                  summary.status = 'success';
+                  summary.status = "success";
 
                   configLogger.logImportSummary(summary);
                   continue; // Skip to next config
                 } else if (latestCommit) {
-                  configLogger.info(`🔄 Repository ${configName} changed (${currentState.lastCommitSha?.slice(0, 7) || 'unknown'} -> ${latestCommit.sha.slice(0, 7)}) - proceeding with import`);
+                  configLogger.info(
+                    `🔄 Repository ${configName} changed (${currentState.lastCommitSha?.slice(0, 7) || "unknown"} -> ${latestCommit.sha.slice(0, 7)}) - proceeding with import`,
+                  );
                 }
               } else {
-                configLogger.debug(`📥 First time importing ${configName} - no previous state found`);
+                configLogger.debug(
+                  `📥 First time importing ${configName} - no previous state found`,
+                );
               }
             } catch (error) {
-              configLogger.warn(`Failed to check repository state for ${configName}: ${error instanceof Error ? error.message : String(error)}`);
+              configLogger.warn(
+                `Failed to check repository state for ${configName}: ${error instanceof Error ? error.message : String(error)}`,
+              );
               // Continue with import if state check fails
             }
           } else {
-            configLogger.info(`🔄 Force mode enabled for ${configName} - proceeding with full import`);
+            configLogger.info(
+              `🔄 Force mode enabled for ${configName} - proceeding with full import`,
+            );
+          }
+
+          // Determine effective clear setting: per-config takes precedence over global
+          const effectiveClear = config.clear ?? clear;
+
+          // Perform selective cleanup before importing if clear is enabled
+          if (effectiveClear) {
+            configLogger.info(
+              `🧹 Clearing obsolete files for ${configName}...`,
+            );
+            try {
+              await performSelectiveCleanup(
+                config,
+                { ...context, logger: configLogger } as ExtendedLoaderContext,
+                octokit,
+              );
+            } catch (error) {
+              configLogger.warn(
+                `Cleanup failed for ${configName}, continuing with import: ${error}`,
+              );
+            }
           }
 
           // Perform the import with spinner
           const stats = await globalLogger.withSpinner(
             `🔄 Importing ${configName}...`,
-            () => toCollectionEntry({
-              context: { ...context, logger: configLogger as any },
-              octokit,
-              options: config,
-              fetchOptions,
-              force,
-            }),
+            () =>
+              toCollectionEntry({
+                context: {
+                  ...context,
+                  logger: configLogger,
+                } as ExtendedLoaderContext,
+                octokit,
+                options: config,
+                fetchOptions,
+                force,
+                clear: effectiveClear,
+              }),
             `✅ ${configName} imported successfully`,
-            `❌ ${configName} import failed`
+            `❌ ${configName} import failed`,
           );
 
           summary.duration = Date.now() - startTime;
@@ -191,7 +218,7 @@ export function githubLoader({
           summary.filesUnchanged = stats?.unchanged || 0;
           summary.assetsDownloaded = stats?.assetsDownloaded || 0;
           summary.assetsCached = stats?.assetsCached || 0;
-          summary.status = 'success';
+          summary.status = "success";
 
           // Log structured summary
           configLogger.logImportSummary(summary);
@@ -202,21 +229,29 @@ export function githubLoader({
             const { data } = await octokit.rest.repos.listCommits({
               owner: config.owner,
               repo: config.repo,
-              sha: config.ref || 'main',
-              per_page: 1
+              sha: config.ref || "main",
+              per_page: 1,
             });
 
             if (data.length > 0) {
-              await updateImportState(process.cwd(), config, data[0].sha);
+              await updateImportState(
+                process.cwd(),
+                config,
+                data[0].sha,
+                configLogger,
+              );
             }
           } catch (error) {
             // Don't fail the import if state tracking fails
-            configLogger.debug(`Failed to update import state for ${configName}: ${error}`);
+            configLogger.debug(
+              `Failed to update import state for ${configName}: ${error}`,
+            );
           }
-        } catch (error: any) {
+        } catch (error: unknown) {
           summary.duration = Date.now() - startTime;
-          summary.status = 'error';
-          summary.error = error.message;
+          summary.status = "error";
+          summary.error =
+            error instanceof Error ? error.message : String(error);
 
           configLogger.logImportSummary(summary);
           // Continue with other configs even if one fails