@aigne/doc-smith 0.5.1 → 0.7.0

package/tests/load-sources.test.mjs

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { mkdir, rm, writeFile } from "node:fs/promises";
 import path, { dirname } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -95,8 +95,56 @@ describe("loadSources", () => {
   });
 
   afterEach(async () => {
-    // Clean up test directory
-    await rm(testDir, { recursive: true, force: true });
+    // Clean up test directory with comprehensive handling
+    try {
+      await rm(testDir, { recursive: true, force: true });
+    } catch {
+      // Try multiple cleanup strategies
+      try {
+        const { chmod, readdir, unlink } = await import("node:fs/promises");
+
+        // Reset permissions recursively
+        const resetPermissions = async (dir) => {
+          try {
+            await chmod(dir, 0o755);
+            const entries = await readdir(dir, { withFileTypes: true });
+            for (const entry of entries) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                await resetPermissions(fullPath);
+              } else {
+                try {
+                  await chmod(fullPath, 0o644);
+                } catch {
+                  // Try to remove symbolic links directly
+                  try {
+                    await unlink(fullPath);
+                  } catch {
+                    // Ignore individual file cleanup failures
+                  }
+                }
+              }
+            }
+          } catch {
+            // Ignore permission reset failures
+          }
+        };
+
+        await resetPermissions(testDir);
+        await rm(testDir, { recursive: true, force: true });
+      } catch {
+        // Final fallback - try to remove with system command
+        try {
+          const { exec } = await import("node:child_process");
+          const { promisify } = await import("node:util");
+          const execAsync = promisify(exec);
+          await execAsync(`rm -rf "${testDir}"`);
+        } catch {
+          // Ignore final cleanup failures - test isolation is more important than perfect cleanup
+          console.warn(`Warning: Could not fully clean up test directory: ${testDir}`);
+        }
+      }
+    }
   });
 
   test("should load files with default patterns", async () => {
@@ -355,4 +403,580 @@ describe("loadSources", () => {
     expect(filePaths.some((element) => element.includes("temp/"))).toBe(false);
     expect(filePaths.some((element) => element.includes("test/test.js"))).toBe(false);
   });
+
+  test("should handle glob patterns in sourcesPath", async () => {
+    const result = await loadSources({
+      sourcesPath: [`${testDir}/src/**/*.js`],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+    expect(result.datasourcesList.length).toBeGreaterThan(0);
+
+    const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+    // Should include JS files from src directory and subdirectories
+    expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/utils.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/components/ui/Modal.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/utils/helpers/format.js"))).toBe(true);
+
+    // Should exclude non-JS files and files outside src
+    expect(filePaths.some((element) => element.includes("package.json"))).toBe(false);
+    expect(filePaths.some((element) => element.includes("README.md"))).toBe(false);
+    expect(filePaths.some((element) => element.includes("styles.css"))).toBe(false);
+  });
+
+  test("should handle multiple glob patterns in sourcesPath", async () => {
+    const result = await loadSources({
+      sourcesPath: [`${testDir}/src/**/*.js`, `${testDir}/*.json`, `${testDir}/*.md`],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+    expect(result.datasourcesList.length).toBeGreaterThan(0);
+
+    const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+    // Should include JS files from src directory
+    expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(true);
+
+    // Should include JSON files from root
+    expect(filePaths.some((element) => element.includes("package.json"))).toBe(true);
+
+    // Should include Markdown files from root
+    expect(filePaths.some((element) => element.includes("README.md"))).toBe(true);
+
+    // Should exclude CSS files
+    expect(filePaths.some((element) => element.includes("styles.css"))).toBe(false);
+  });
+
+  test("should handle glob pattern with specific file extensions", async () => {
+    const result = await loadSources({
+      sourcesPath: [`${testDir}/src/**/*.{js,json,yaml}`],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+
+    const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+    // Should include JS files
+    expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/utils.js"))).toBe(true);
+
+    // Should include JSON files
+    expect(filePaths.some((element) => element.includes("src/config/settings.json"))).toBe(true);
+
+    // Should include YAML files
+    expect(filePaths.some((element) => element.includes("src/utils/helpers/data.yaml"))).toBe(true);
+
+    // Should exclude CSS files
+    expect(filePaths.some((element) => element.includes("styles.css"))).toBe(false);
+  });
+
+  test("should handle glob pattern that matches no files", async () => {
+    const result = await loadSources({
+      sourcesPath: [`${testDir}/nonexistent/**/*.xyz`],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+    expect(result.datasourcesList.length).toBe(0);
+  });
+
+  test("should handle mixed regular paths and glob patterns", async () => {
+    const result = await loadSources({
+      sourcesPath: [testDir, `${testDir}/src/**/*.js`],
+      includePatterns: ["*.md"],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+    expect(result.datasourcesList.length).toBeGreaterThan(0);
+
+    const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+    // Should include markdown files from directory scan
+    expect(filePaths.some((element) => element.includes("README.md"))).toBe(true);
+
+    // Should include JS files from glob pattern
+    expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+    expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(true);
+  });
+
+  test("should handle glob patterns with wildcards and character classes", async () => {
+    const result = await loadSources({
+      sourcesPath: [`${testDir}/src/**/[Bb]utton.js`, `${testDir}/src/**/?odal.js`],
+      useDefaultPatterns: false,
+      outputDir: tempDir,
+      docsDir: path.join(testDir, "docs"),
+    });
+
+    expect(result.datasourcesList).toBeDefined();
+
+    const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+    // Should match Button.js with character class
+    expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(true);
+
+    // Should match Modal.js with single character wildcard
+    expect(filePaths.some((element) => element.includes("src/components/ui/Modal.js"))).toBe(true);
+
+    // Should not match other files
+    expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(false);
+    expect(filePaths.some((element) => element.includes("src/utils.js"))).toBe(false);
+  });
+
+  describe("Configuration integration tests", () => {
+    test("should handle YAML config-like sourcesPath input", async () => {
+      const yamlConfigInput = {
+        sourcesPath: ["./src", "./lib", "**/*.{js,ts,jsx,tsx}"],
+        docsDir: "./docs",
+        useDefaultPatterns: true,
+        outputDir: tempDir,
+      };
+
+      const result = await loadSources(yamlConfigInput);
+
+      expect(result.datasourcesList).toBeDefined();
+      expect(result.datasourcesList.length).toBeGreaterThan(0);
+
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+      expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+    });
+
+    test("should handle config with empty sourcesPath array - ACTUALLY WORKS", async () => {
+      // This test shows that empty arrays are actually handled correctly
+      const result = await loadSources({
+        sourcesPath: [],
+        useDefaultPatterns: true,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      expect(result.datasourcesList.length).toBe(0);
+      expect(result.files.length).toBe(0);
+
+      // Empty arrays are handled gracefully - this is correct behavior
+    });
+
+    test("should filter invalid path elements gracefully - EXPOSES ROBUSTNESS BUG", async () => {
+      // ROBUSTNESS BUG TEST: Function should handle invalid elements gracefully
+      const mixedArray = ["./src", null, undefined, "./lib", ""];
+
+      // This test expects the function to filter out invalid elements and work with valid ones
+      const result = await loadSources({
+        sourcesPath: mixedArray,
+        useDefaultPatterns: true,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      // Should successfully process valid paths and ignore invalid ones
+      expect(result.datasourcesList).toBeDefined();
+      // This test will FAIL because the function crashes instead of filtering
+    });
+
+    test("should handle config with undefined sourcesPath - ACTUALLY WORKS", async () => {
+      // This test shows undefined sourcesPath is actually handled correctly
+      const result = await loadSources({
+        sourcesPath: undefined,
+        useDefaultPatterns: true,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      expect(Array.isArray(result.datasourcesList)).toBe(true);
+      expect(result.datasourcesList.length).toBe(0);
+
+      // undefined is handled gracefully by skipping sourcesPath processing
+    });
+  });
+
+  describe("Function robustness issues - EXPOSES INPUT HANDLING PROBLEMS", () => {
+    test("should handle invalid inputs gracefully - EXPOSES POOR ERROR HANDLING", async () => {
+      // ROBUSTNESS TEST: Function should validate inputs and provide helpful errors
+      const invalidInputs = [
+        { value: [null], description: "array with null element", shouldFail: true },
+        { value: [undefined], description: "array with undefined element", shouldFail: true },
+        { value: ["", null], description: "mixed array with null", shouldFail: true },
+      ];
+
+      for (const { value: invalidInput, shouldFail } of invalidInputs) {
+        if (shouldFail) {
+          // These should provide helpful error messages, not crash unexpectedly
+          const result = await loadSources({
+            sourcesPath: invalidInput,
+            useDefaultPatterns: true,
+            outputDir: tempDir,
+            docsDir: path.join(testDir, "docs"),
+          });
+          expect(result.datasourcesList).toBeDefined();
+          expect(Array.isArray(result.datasourcesList)).toBe(true);
+          expect(result.datasourcesList.length).toBe(0);
+        }
+      }
+
+      // Test inputs that should work but be handled gracefully
+      const shouldWorkInputs = [
+        null, // Should be treated like undefined
+        "", // Should be treated like undefined
+        123, // Should provide helpful error
+        {}, // Should provide helpful error
+        [""], // Should filter out empty strings
+      ];
+
+      for (const validInput of shouldWorkInputs) {
+        // These tests expect graceful handling but may FAIL due to poor input validation
+        const result = await loadSources({
+          sourcesPath: validInput,
+          useDefaultPatterns: true,
+          outputDir: tempDir,
+          docsDir: path.join(testDir, "docs"),
+        });
+
+        expect(result.datasourcesList).toBeDefined();
+        // Some of these tests will FAIL, exposing input validation issues
+      }
+    });
+  });
+
+  describe("File system edge cases", () => {
+    test("should handle symbolic links", async () => {
+      // Skip on systems that don't support symlinks
+      try {
+        const { symlink } = await import("node:fs/promises");
+        const symlinkPath = path.join(testDir, "symlink-to-src");
+        await symlink(path.join(testDir, "src"), symlinkPath);
+
+        const result = await loadSources({
+          sourcesPath: symlinkPath,
+          includePatterns: ["*.js"],
+          useDefaultPatterns: false,
+          outputDir: tempDir,
+          docsDir: path.join(testDir, "docs"),
+        });
+
+        expect(result.datasourcesList).toBeDefined();
+        const filePaths = result.datasourcesList.map((f) => f.sourceId);
+        expect(filePaths.some((element) => element.includes("index.js"))).toBe(true);
+      } catch {
+        // Skip test on systems that don't support symlinks
+        expect(true).toBe(true);
+      }
+    });
+
+    test("should handle very deep directory structures", async () => {
+      // Create deep nested structure
+      const deepPath = path.join(testDir, "a/b/c/d/e/f/g/h");
+      await mkdir(deepPath, { recursive: true });
+      await writeFile(path.join(deepPath, "deep.js"), "console.log('deep');");
+
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["**/*.js"],
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+      expect(filePaths.some((element) => element.includes("a/b/c/d/e/f/g/h/deep.js"))).toBe(true);
+    });
+
+    test("should handle files with unusual names", async () => {
+      // Create files with special characters in names
+      await writeFile(path.join(testDir, "file with spaces.js"), "console.log('spaces');");
+      await writeFile(path.join(testDir, "file-with-dashes.js"), "console.log('dashes');");
+      await writeFile(
+        path.join(testDir, "file_with_underscores.js"),
+        "console.log('underscores');",
+      );
+      await writeFile(path.join(testDir, "file.with.dots.js"), "console.log('dots');");
+      await writeFile(path.join(testDir, "文件中文名.js"), "console.log('chinese');");
+
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["*.js"],
+        excludePatterns: ["**/test/**", "*_test*"],
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+      expect(filePaths.some((element) => element.includes("file with spaces.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("file-with-dashes.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("file_with_underscores.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("file.with.dots.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("文件中文名.js"))).toBe(true);
+    });
+
+    test("should handle empty directories", async () => {
+      const emptyDir = path.join(testDir, "empty");
+      await mkdir(emptyDir, { recursive: true });
+
+      const result = await loadSources({
+        sourcesPath: emptyDir,
+        useDefaultPatterns: true,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      expect(result.datasourcesList.length).toBe(0);
+    });
+  });
+
+  describe("Pattern matching edge cases", () => {
+    test("should handle negation patterns correctly", async () => {
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["**/*.js"],
+        excludePatterns: ["!src/components/**"], // Negation should include
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+      // Should still exclude test files by default gitignore/patterns
+      expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(true);
+    });
+
+    test("should handle conflicting include/exclude patterns", async () => {
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["src/**/*.js"], // Include all JS in src
+        excludePatterns: ["src/**/*.js"], // But exclude all JS in src
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+      // Exclude should win over include
+      expect(filePaths.some((element) => element.includes("src/index.js"))).toBe(false);
+      expect(filePaths.some((element) => element.includes("src/components/Button.js"))).toBe(false);
+    });
+
+    test("should handle case sensitivity in patterns", async () => {
+      // Create files with different cases
+      await writeFile(path.join(testDir, "CamelCase.js"), "console.log('CamelCase');");
+      await writeFile(path.join(testDir, "lowercase.js"), "console.log('lowercase');");
+      await writeFile(path.join(testDir, "UPPERCASE.JS"), "console.log('UPPERCASE');");
+
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["*.[jJ][sS]"], // Case insensitive for extension
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+
+      expect(filePaths.some((element) => element.includes("CamelCase.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("lowercase.js"))).toBe(true);
+      expect(filePaths.some((element) => element.includes("UPPERCASE.JS"))).toBe(true);
+    });
+
+    test("should handle extremely complex glob patterns", async () => {
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: [
+          "src/**/!(test|spec)*{.js,.ts,.jsx,.tsx}", // Complex negation with alternatives
+          "**/@(components|utils|services)/**/*.{js,ts}", // At-patterns with alternatives
+        ],
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      // Just verify it doesn't crash and returns some results
+      expect(Array.isArray(result.datasourcesList)).toBe(true);
+    });
+  });
+
+  describe("Performance and scale tests", () => {
+    test("should handle large number of patterns efficiently", async () => {
+      const manyPatterns = Array(100)
+        .fill(0)
+        .map((_, i) => `**/*${i}*.js`);
+
+      const startTime = Date.now();
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: manyPatterns,
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+      const endTime = Date.now();
+
+      expect(result.datasourcesList).toBeDefined();
+      expect(endTime - startTime).toBeLessThan(5000); // Should complete within 5 seconds
+    });
+
+    test("should handle very long paths", async () => {
+      const longDirName = "a".repeat(100);
+      const longPath = path.join(testDir, longDirName);
+      await mkdir(longPath, { recursive: true });
+      await writeFile(path.join(longPath, "file.js"), "console.log('long path');");
+
+      const result = await loadSources({
+        sourcesPath: testDir,
+        includePatterns: ["**/*.js"],
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      const filePaths = result.datasourcesList.map((f) => f.sourceId);
+      expect(filePaths.some((element) => element.includes(longDirName))).toBe(true);
+    });
+  });
+
+  describe("Error handling and resilience", () => {
+    test("should handle permission denied gracefully", async () => {
+      // This test may not work on all systems, so we wrap in try-catch
+      try {
+        const restrictedDir = path.join(testDir, "restricted");
+        await mkdir(restrictedDir, { recursive: true, mode: 0o000 });
+
+        const result = await loadSources({
+          sourcesPath: restrictedDir,
+          useDefaultPatterns: true,
+          outputDir: tempDir,
+          docsDir: path.join(testDir, "docs"),
+        });
+
+        expect(result.datasourcesList).toBeDefined();
+        // Should handle gracefully without crashing
+      } catch {
+        // Skip on systems where permission tests don't work
+        expect(true).toBe(true);
+      }
+    });
+
+    test("should handle malformed glob patterns", async () => {
+      const result = await loadSources({
+        sourcesPath: [`${testDir}/[unclosed`, `${testDir}/**/{unclosed`],
+        useDefaultPatterns: false,
+        outputDir: tempDir,
+        docsDir: path.join(testDir, "docs"),
+      });
+
+      expect(result.datasourcesList).toBeDefined();
+      // Should not crash, even with malformed patterns
+      expect(Array.isArray(result.datasourcesList)).toBe(true);
+    });
+
+    test("should handle circular symbolic links", async () => {
+      try {
+        const { symlink, unlink } = await import("node:fs/promises");
+        const link1 = path.join(testDir, "link1");
+        const link2 = path.join(testDir, "link2");
+
+        // Create circular links
+        await symlink(link2, link1);
+        await symlink(link1, link2);
+
+        const result = await loadSources({
+          sourcesPath: testDir,
+          includePatterns: ["**/*"],
+          useDefaultPatterns: false,
+          outputDir: tempDir,
+          docsDir: path.join(testDir, "docs"),
+        });
+
+        expect(result.datasourcesList).toBeDefined();
+        // Should handle circular links without infinite loop
+
+        // Clean up circular links immediately to prevent interference with other tests
+        try {
+          await unlink(link1);
+        } catch {
+          // Ignore cleanup errors
+        }
+        try {
+          await unlink(link2);
+        } catch {
+          // Ignore cleanup errors
+        }
+      } catch {
+        // Skip on systems that don't support symlinks
+        expect(true).toBe(true);
+      }
+    });
+  });
+
+  describe("Integration with generateYAML output", () => {
+    test("should process typical generateYAML sourcesPath configurations", async () => {
+      const typicalConfigs = [
+        ["./src", "./lib"],
+        ["**/*.{js,ts,jsx,tsx}"],
+        ["src/**/*.js", "lib/**/*.ts", "docs/**/*.md"],
+        ["./", "!node_modules", "!dist"],
+      ];
+
+      for (const sourcesPathConfig of typicalConfigs) {
+        const result = await loadSources({
+          sourcesPath: sourcesPathConfig,
+          useDefaultPatterns: true,
+          outputDir: tempDir,
+          docsDir: path.join(testDir, "docs"),
+        });
+
+        expect(result.datasourcesList).toBeDefined();
+        expect(Array.isArray(result.datasourcesList)).toBe(true);
+        // Should not crash with any typical config
+      }
+    });
+  });
+
+  // Global cleanup to ensure test directories are fully removed
+  afterAll(async () => {
+    const testDirBase = path.join(__dirname, "test-content-generator");
+    try {
+      await rm(testDirBase, { recursive: true, force: true });
+    } catch {
+      // Try with system command as final fallback
+      try {
+        const { exec } = await import("node:child_process");
+        const { promisify } = await import("node:util");
+        const execAsync = promisify(exec);
+        await execAsync(`rm -rf "${testDirBase}"`);
+      } catch {
+        // If we still can't clean up, warn but don't fail
+        console.warn(`Warning: Could not fully clean up test directory: ${testDirBase}`);
+      }
+    }
+  });
 });