@larkiny/astro-github-loader 0.11.3 → 0.13.0

package/src/github.cleanup.spec.ts

@@ -0,0 +1,380 @@
+import { beforeEach, describe, it, expect, vi, afterEach } from "vitest";
+import { performSelectiveCleanup } from "./github.cleanup.js";
+import type { ImportOptions, SyncStats } from "./github.types.js";
+import { createMockContext, createMockOctokit } from "./test-helpers.js";
+import { promises as fs } from "node:fs";
+import { existsSync } from "node:fs";
+
+// Mock the filesystem modules
+vi.mock("node:fs/promises");
+vi.mock("node:fs");
+
+describe("github.cleanup", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe("performSelectiveCleanup", () => {
+    it("should return zero stats when repository has no include patterns", async () => {
+      const { octokit } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [],
+      };
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      expect(stats.deleted).toBe(0);
+      expect(stats.added).toBe(0);
+      expect(stats.updated).toBe(0);
+      expect(stats.unchanged).toBe(0);
+      expect(stats.duration).toBeGreaterThanOrEqual(0);
+    });
+
+    it("should detect orphaned files that are not in the remote repository tree", async () => {
+      const { octokit, spies } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync to return true for basePath and orphaned file
+      vi.mocked(existsSync).mockImplementation((path) => {
+        return path === "/test-base" || path === "/test-base/orphaned-file.md";
+      });
+
+      // Mock readdir to return local files including an orphaned file
+      vi.mocked(fs.readdir).mockResolvedValue([
+        {
+          name: "orphaned-file.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+      ]);
+
+      // Mock getContent to return only files that exist in remote
+      spies.getContentSpy.mockResolvedValueOnce({
+        data: [
+          {
+            type: "file",
+            path: "docs/valid-file.md",
+            name: "valid-file.md",
+            sha: "abc123",
+          },
+        ],
+        status: 200,
+        url: "",
+        headers: {},
+      } as any);
+
+      // Mock unlink for file deletion
+      vi.mocked(fs.unlink).mockResolvedValue(undefined);
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      // The orphaned file should be detected and deleted
+      expect(stats.deleted).toBe(1);
+      expect(vi.mocked(fs.unlink)).toHaveBeenCalledWith(
+        "/test-base/orphaned-file.md",
+      );
+    });
+
+    it("should track deletion stats correctly", async () => {
+      const { octokit, spies } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync
+      vi.mocked(existsSync).mockImplementation((path) => {
+        return (
+          path === "/test-base" ||
+          path === "/test-base/orphan1.md" ||
+          path === "/test-base/orphan2.md" ||
+          path === "/test-base/orphan3.md"
+        );
+      });
+
+      // Mock readdir to return multiple orphaned files
+      vi.mocked(fs.readdir).mockResolvedValue([
+        {
+          name: "orphan1.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+        {
+          name: "orphan2.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+        {
+          name: "orphan3.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+      ]);
+
+      // Mock getContent to return empty (no remote files match)
+      spies.getContentSpy.mockResolvedValue({
+        data: [],
+        status: 200,
+        url: "",
+        headers: {},
+      } as any);
+
+      // Mock unlink
+      vi.mocked(fs.unlink).mockResolvedValue(undefined);
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      expect(stats.deleted).toBe(3);
+      expect(vi.mocked(fs.unlink)).toHaveBeenCalledTimes(3);
+      expect(stats.duration).toBeGreaterThanOrEqual(0);
+    });
+
+    it("should handle Octokit API failures gracefully", async () => {
+      const { octokit, spies } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync
+      vi.mocked(existsSync).mockReturnValue(true);
+
+      // Mock readdir to return local files
+      vi.mocked(fs.readdir).mockResolvedValue([
+        {
+          name: "some-file.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+      ]);
+
+      // Mock unlink for file deletion
+      vi.mocked(fs.unlink).mockResolvedValue(undefined);
+
+      // Mock getContent to throw an error (this happens when fetching expected files)
+      // NOTE: Current behavior - when API fails, processDirectory catches the error and logs a warning,
+      // but getExpectedFiles returns an empty Set. This causes all local files to be treated as orphans.
+      spies.getContentSpy.mockRejectedValue(new Error("API rate limit exceeded"));
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      // Current behavior: When API fails, expectedFiles is empty, so all local files are deleted
+      expect(stats.deleted).toBe(1);
+      expect(stats.added).toBe(0);
+      expect(stats.updated).toBe(0);
+      expect(stats.unchanged).toBe(0);
+      expect(stats.duration).toBeGreaterThanOrEqual(0);
+
+      // Should log a warning from processDirectory
+      expect(ctx.logger.warn).toHaveBeenCalled();
+
+      // Files are deleted because empty expectedFiles makes all local files orphans
+      expect(vi.mocked(fs.unlink)).toHaveBeenCalledWith("/test-base/some-file.md");
+    });
+
+    it("should skip cleanup when no local files exist (fresh import)", async () => {
+      const { octokit } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync to return false (basePath doesn't exist)
+      vi.mocked(existsSync).mockReturnValue(false);
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      expect(stats.deleted).toBe(0);
+      expect(vi.mocked(fs.readdir)).not.toHaveBeenCalled();
+      expect(vi.mocked(fs.unlink)).not.toHaveBeenCalled();
+    });
+
+    it("should handle file deletion errors without failing the entire cleanup", async () => {
+      const { octokit, spies } = createMockOctokit();
+      const ctx = createMockContext();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync
+      vi.mocked(existsSync).mockImplementation((path) => {
+        return (
+          path === "/test-base" ||
+          path === "/test-base/file1.md" ||
+          path === "/test-base/file2.md"
+        );
+      });
+
+      // Mock readdir
+      vi.mocked(fs.readdir).mockResolvedValue([
+        {
+          name: "file1.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+        {
+          name: "file2.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+      ]);
+
+      // Mock getContent to return empty
+      spies.getContentSpy.mockResolvedValue({
+        data: [],
+        status: 200,
+        url: "",
+        headers: {},
+      } as any);
+
+      // Mock unlink to fail on first file, succeed on second
+      vi.mocked(fs.unlink)
+        .mockRejectedValueOnce(new Error("Permission denied"))
+        .mockResolvedValueOnce(undefined);
+
+      const stats: SyncStats = await performSelectiveCleanup(
+        config,
+        ctx as any,
+        octokit,
+      );
+
+      // Should still delete the second file despite first failure
+      expect(stats.deleted).toBe(1);
+      expect(ctx.logger.warn).toHaveBeenCalled();
+    });
+
+    it("should handle AbortSignal cancellation", async () => {
+      const { octokit, spies } = createMockOctokit();
+      const ctx = createMockContext();
+      const abortController = new AbortController();
+
+      const config: ImportOptions = {
+        name: "Test Repo",
+        owner: "test-owner",
+        repo: "test-repo",
+        ref: "main",
+        includes: [
+          {
+            pattern: "docs/**/*.md",
+            basePath: "/test-base",
+          },
+        ],
+      };
+
+      // Mock existsSync
+      vi.mocked(existsSync).mockReturnValue(true);
+
+      // Mock readdir
+      vi.mocked(fs.readdir).mockResolvedValue([
+        {
+          name: "file.md",
+          isFile: () => true,
+          isDirectory: () => false,
+        } as any,
+      ]);
+
+      // Mock getContent to throw abort error
+      const abortError = new Error("Aborted");
+      spies.getContentSpy.mockRejectedValue(abortError);
+
+      // Abort the operation
+      abortController.abort();
+
+      await expect(
+        performSelectiveCleanup(
+          config,
+          ctx as any,
+          octokit,
+          abortController.signal,
+        ),
+      ).rejects.toThrow();
+
+      expect(ctx.logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("cancelled"),
+      );
+    });
+  });
+});

package/src/github.cleanup.ts

@@ -1,8 +1,18 @@
 import { promises as fs } from "node:fs";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { generateId, generatePath, shouldIncludeFile } from "./github.content.js";
-import type { ImportOptions, LoaderContext, SyncStats } from "./github.types.js";
+import {
+  generateId,
+  generatePath,
+  shouldIncludeFile,
+} from "./github.content.js";
+import { Octokit } from "octokit";
+import type { Logger } from "./github.logger.js";
+import type {
+  ExtendedLoaderContext,
+  ImportOptions,
+  SyncStats,
+} from "./github.types.js";
 
 const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
 
@@ -10,16 +20,17 @@ const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
  * Sleep utility for pacing file operations
  */
 function sleep(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
 /**
  * Gets all files that should exist locally based on remote repository state
  */
 async function getExpectedFiles(
-  octokit: any,
+  octokit: Octokit,
   options: ImportOptions,
-  signal?: AbortSignal
+  logger: Logger,
+  signal?: AbortSignal,
 ): Promise<Set<string>> {
   const { owner, repo, ref = "main" } = options;
   const expectedFiles = new Set<string>();
@@ -31,12 +42,14 @@ async function getExpectedFiles(
       // Extract directory part from pattern (before any glob wildcards)
       const pattern = includePattern.pattern;
       const beforeGlob = pattern.split(/[*?{]/)[0];
-      const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+      const dirPart = beforeGlob.includes("/")
+        ? beforeGlob.substring(0, beforeGlob.lastIndexOf("/"))
+        : "";
       directoriesToScan.add(dirPart);
     }
   } else {
     // If no includes specified, scan from root
-    directoriesToScan.add('');
+    directoriesToScan.add("");
   }
 
   async function processDirectory(dirPath: string) {
@@ -46,17 +59,26 @@ async function getExpectedFiles(
         repo,
         path: dirPath,
         ref,
-        request: { signal }
+        request: { signal },
       });
 
       if (!Array.isArray(data)) {
         // Single file
-        if (data.type === 'file' && shouldIncludeFile(data.path, options).included) {
+        if (
+          data.type === "file" &&
+          shouldIncludeFile(data.path, options).included
+        ) {
           const id = generateId(data.path);
           const includeResult = shouldIncludeFile(data.path, options);
-          const localPath = generatePath(data.path, includeResult.included ? includeResult.matchedPattern : null, options);
+          const localPath = generatePath(
+            data.path,
+            includeResult.included ? includeResult.matchedPattern : null,
+            options,
+          );
           // Convert to absolute path for consistent comparison
-          const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+          const absolutePath = localPath.startsWith("/")
+            ? localPath
+            : join(process.cwd(), localPath);
           expectedFiles.add(absolutePath);
         }
         return;
@@ -75,17 +97,23 @@ async function getExpectedFiles(
           } else if (type === "file") {
             const id = generateId(itemPath);
             const includeResult = shouldIncludeFile(itemPath, options);
-            const localPath = generatePath(itemPath, includeResult.included ? includeResult.matchedPattern : null, options);
+            const localPath = generatePath(
+              itemPath,
+              includeResult.included ? includeResult.matchedPattern : null,
+              options,
+            );
             // Convert to absolute path for consistent comparison
-            const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+            const absolutePath = localPath.startsWith("/")
+              ? localPath
+              : join(process.cwd(), localPath);
             expectedFiles.add(absolutePath);
           }
         });
 
       await Promise.all(promises);
-    } catch (error: any) {
+    } catch (error: unknown) {
       if (signal?.aborted) throw error;
-      console.warn(`Failed to process directory ${dirPath}:`, error);
+      logger.warn(`Failed to process directory ${dirPath}: ${error}`);
     }
   }
 
@@ -99,9 +127,12 @@ async function getExpectedFiles(
 /**
  * Gets all existing local files in the basePath as absolute paths
  */
-async function getExistingFiles(basePath: string): Promise<Set<string>> {
+async function getExistingFiles(
+  basePath: string,
+  logger: Logger,
+): Promise<Set<string>> {
   const existingFiles = new Set<string>();
-  
+
   if (!existsSync(basePath)) {
     return existingFiles;
   }
@@ -109,24 +140,24 @@ async function getExistingFiles(basePath: string): Promise<Set<string>> {
   async function walkDirectory(dirPath: string) {
     try {
       const entries = await fs.readdir(dirPath, { withFileTypes: true });
-      
+
       for (const entry of entries) {
         const fullPath = join(dirPath, entry.name);
-        
+
         if (entry.isDirectory()) {
           // Skip manifest files and other system directories
-          if (!entry.name.startsWith('.')) {
+          if (!entry.name.startsWith(".")) {
             await walkDirectory(fullPath);
           }
         } else if (entry.isFile()) {
           // Skip manifest and system files
-          if (!entry.name.startsWith('.')) {
+          if (!entry.name.startsWith(".")) {
             existingFiles.add(fullPath);
           }
         }
       }
     } catch (error) {
-      console.warn(`Failed to read directory ${dirPath}:`, error);
+      logger.warn(`Failed to read directory ${dirPath}: ${error}`);
     }
   }
 
@@ -139,14 +170,14 @@ async function getExistingFiles(basePath: string): Promise<Set<string>> {
  */
 export async function performSelectiveCleanup(
   config: ImportOptions,
-  context: LoaderContext,
-  octokit: any,
-  signal?: AbortSignal
+  context: ExtendedLoaderContext,
+  octokit: Octokit,
+  signal?: AbortSignal,
 ): Promise<SyncStats> {
   const startTime = Date.now();
   const { logger } = context;
   const configName = config.name || `${config.owner}/${config.repo}`;
-  
+
   if (!config.includes || config.includes.length === 0) {
     // No cleanup needed if no include patterns specified
     return {
@@ -154,7 +185,7 @@ export async function performSelectiveCleanup(
       updated: 0,
       deleted: 0,
       unchanged: 0,
-      duration: Date.now() - startTime
+      duration: Date.now() - startTime,
     };
   }
 
@@ -164,25 +195,35 @@ export async function performSelectiveCleanup(
     // Get existing local files from all include pattern base paths
     const allExistingFiles = new Set<string>();
     for (const includePattern of config.includes) {
-      const existingFiles = await getExistingFiles(includePattern.basePath);
-      existingFiles.forEach(file => allExistingFiles.add(file));
+      const existingFiles = await getExistingFiles(
+        includePattern.basePath,
+        logger,
+      );
+      existingFiles.forEach((file) => allExistingFiles.add(file));
     }
-    
+
     // If no existing files, skip cleanup (fresh import)
     if (allExistingFiles.size === 0) {
-      logger.debug(`No existing files found in any base paths, skipping cleanup`);
+      logger.debug(
+        `No existing files found in any base paths, skipping cleanup`,
+      );
       return {
         added: 0,
         updated: 0,
         deleted: 0,
         unchanged: 0,
-        duration: Date.now() - startTime
+        duration: Date.now() - startTime,
       };
     }
-    
+
     // Get expected files from remote repository
-    const expectedFiles = await getExpectedFiles(octokit, config, signal);
-    
+    const expectedFiles = await getExpectedFiles(
+      octokit,
+      config,
+      logger,
+      signal,
+    );
+
     // Find files to delete (exist locally but not in remote)
     const filesToDelete: string[] = [];
     for (const existingFile of allExistingFiles) {
@@ -190,7 +231,7 @@ export async function performSelectiveCleanup(
         filesToDelete.push(existingFile);
       }
     }
-    
+
     // Delete obsolete files with pacing
     let deletedCount = 0;
     for (const filePath of filesToDelete) {
@@ -205,39 +246,42 @@ export async function performSelectiveCleanup(
         logger.warn(`Failed to delete ${filePath}: ${error}`);
       }
     }
-    
+
     const duration = Date.now() - startTime;
     const stats: SyncStats = {
       added: 0, // Will be counted by main sync process
-      updated: 0, // Will be counted by main sync process  
+      updated: 0, // Will be counted by main sync process
       deleted: deletedCount,
       unchanged: 0, // Will be counted by main sync process
-      duration
+      duration,
     };
 
     if (deletedCount > 0) {
-      logger.info(`Cleanup completed for ${configName}: ${deletedCount} obsolete files deleted (${duration}ms)`);
+      logger.info(
+        `Cleanup completed for ${configName}: ${deletedCount} obsolete files deleted (${duration}ms)`,
+      );
     } else {
       logger.debug(`No cleanup needed for ${configName} (${duration}ms)`);
     }
 
     return stats;
-    
-  } catch (error: any) {
+  } catch (error: unknown) {
     if (signal?.aborted) {
       logger.info(`Cleanup cancelled for ${configName}`);
       throw error;
     }
-    
+
     const duration = Date.now() - startTime;
-    logger.error(`Cleanup failed for ${configName} after ${duration}ms: ${error}`);
+    logger.error(
+      `Cleanup failed for ${configName} after ${duration}ms: ${error instanceof Error ? error.message : String(error)}`,
+    );
     // Don't throw - let the main sync process continue
     return {
       added: 0,
-      updated: 0, 
+      updated: 0,
       deleted: 0,
       unchanged: 0,
-      duration
+      duration,
     };
   }
-}
+}

package/src/github.constants.ts

@@ -6,20 +6,3 @@
  * @internal
  */
 export const INVALID_STRING_ERROR = "Invalid string";
-/**
- * Represents an error message indicating that a provided URL is invalid.
- * This constant is typically used for validation or error handling when a URL
- * does not conform to the expected format or requirements.
- *
- * @internal
- */
-export const INVALID_URL_ERROR = "Invalid url";
-
-/**
- * A constant that holds a default error message indicating that a service response is invalid.
- * This value is typically used to signify that the response from a service or API call
- * does not meet the expected format, structure, or criteria.
- *
- * @internal
- */
-export const INVALID_SERVICE_RESPONSE = "Invalid service response";