@larkiny/astro-github-loader 0.11.2 → 0.12.0

package/src/github.paths.ts

@@ -0,0 +1,259 @@
+import path, { join, basename } from "node:path";
+import picomatch from "picomatch";
+import type {
+  ExtendedLoaderContext,
+  ImportOptions,
+  MatchedPattern,
+} from "./github.types.js";
+
+export interface ImportStats {
+  processed: number;
+  updated: number;
+  unchanged: number;
+  assetsDownloaded?: number;
+  assetsCached?: number;
+}
+
+/**
+ * Generates a unique identifier from a file path by removing the extension
+ * @param filePath - The file path to generate ID from
+ * @return {string} The generated identifier as a string with extension removed
+ * @internal
+ */
+export function generateId(filePath: string): string {
+  let id = filePath;
+
+  // Remove file extension for ID generation
+  const lastDotIndex = id.lastIndexOf(".");
+  if (lastDotIndex > 0) {
+    id = id.substring(0, lastDotIndex);
+  }
+  return id;
+}
+
+/**
+ * Applies path mapping logic to get the final filename for a file
+ *
+ * Supports two types of path mappings:
+ * - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
+ * - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
+ *
+ * @param filePath - Original source file path
+ * @param matchedPattern - The pattern that matched this file
+ * @param options - Import options containing path mappings
+ * @returns Final filename after applying path mapping logic
+ * @internal
+ */
+export function applyRename(
+  filePath: string,
+  matchedPattern?: MatchedPattern | null,
+  options?: ImportOptions,
+): string {
+  if (
+    options?.includes &&
+    matchedPattern &&
+    matchedPattern.index < options.includes.length
+  ) {
+    const includePattern = options.includes[matchedPattern.index];
+
+    if (includePattern.pathMappings) {
+      // First check for exact file match (current behavior - backwards compatible)
+      if (includePattern.pathMappings[filePath]) {
+        const mappingValue = includePattern.pathMappings[filePath];
+        return typeof mappingValue === "string"
+          ? mappingValue
+          : mappingValue.target;
+      }
+
+      // Then check for folder-to-folder mappings
+      for (const [sourceFolder, mappingValue] of Object.entries(
+        includePattern.pathMappings,
+      )) {
+        // Check if this is a folder mapping (ends with /) and file is within it
+        if (sourceFolder.endsWith("/") && filePath.startsWith(sourceFolder)) {
+          // Replace the source folder path with target folder path
+          const targetFolder =
+            typeof mappingValue === "string"
+              ? mappingValue
+              : mappingValue.target;
+          const relativePath = filePath.slice(sourceFolder.length);
+          return path.posix.join(targetFolder, relativePath);
+        }
+      }
+    }
+  }
+
+  // Return original filename if no path mapping found
+  return basename(filePath);
+}
+
+/**
+ * Generates a local file path based on the matched pattern and file path
+ * @param filePath - The original file path from the repository
+ * @param matchedPattern - The pattern that matched this file (or null if no includes specified)
+ * @param options - Import options containing includes patterns for path mapping lookups
+ * @return {string} The local file path where this content should be stored
+ * @internal
+ */
+export function generatePath(
+  filePath: string,
+  matchedPattern?: MatchedPattern | null,
+  options?: ImportOptions,
+): string {
+  if (matchedPattern) {
+    // Extract the directory part from the pattern (before any glob wildcards)
+    const pattern = matchedPattern.pattern;
+    const beforeGlob = pattern.split(/[*?{]/)[0];
+
+    // Remove the pattern prefix from the file path to get the relative path
+    let relativePath = filePath;
+    if (beforeGlob && filePath.startsWith(beforeGlob)) {
+      relativePath = filePath.substring(beforeGlob.length);
+      // Remove leading slash if present
+      if (relativePath.startsWith("/")) {
+        relativePath = relativePath.substring(1);
+      }
+    }
+
+    // If no relative path remains, use just the filename
+    if (!relativePath) {
+      relativePath = basename(filePath);
+    }
+
+    // Apply path mapping logic
+    const finalFilename = applyRename(filePath, matchedPattern, options);
+    // Always apply path mapping if applyRename returned something different from the original basename
+    // OR if there are pathMappings configured (since empty string mappings might return same basename)
+    const hasPathMappings =
+      options?.includes?.[matchedPattern.index]?.pathMappings &&
+      Object.keys(options.includes[matchedPattern.index].pathMappings!).length >
+        0;
+    if (finalFilename !== basename(filePath) || hasPathMappings) {
+      // Check if applyRename returned a full path (contains path separators) or just a filename
+      if (finalFilename.includes("/") || finalFilename.includes("\\")) {
+        // applyRename returned a full relative path - need to extract relative part
+        // Remove the pattern prefix to get the relative path within the pattern context
+        const beforeGlob = pattern.split(/[*?{]/)[0];
+        if (beforeGlob && finalFilename.startsWith(beforeGlob)) {
+          relativePath = finalFilename.substring(beforeGlob.length);
+          // Remove leading slash if present
+          if (relativePath.startsWith("/")) {
+            relativePath = relativePath.substring(1);
+          }
+        } else {
+          relativePath = finalFilename;
+        }
+      } else {
+        // applyRename returned just a filename
+        // If the filename is different due to pathMapping, use it directly
+        // This handles cases where pathMappings flatten directory structures
+        relativePath = finalFilename;
+      }
+    }
+
+    return join(matchedPattern.basePath, relativePath);
+  }
+
+  // Should not happen since we always use includes
+  throw new Error("No matched pattern provided - includes are required");
+}
+
+/**
+ * Checks if a file path should be included and returns the matching pattern
+ * @param filePath - The file path to check (relative to the repository root)
+ * @param options - Import options containing includes patterns
+ * @returns Object with include status and matched pattern, or null if not included
+ * @internal
+ */
+export function shouldIncludeFile(
+  filePath: string,
+  options: ImportOptions,
+):
+  | { included: true; matchedPattern: MatchedPattern | null }
+  | { included: false; matchedPattern: null } {
+  const { includes } = options;
+
+  // If no include patterns specified, include all files
+  if (!includes || includes.length === 0) {
+    return { included: true, matchedPattern: null };
+  }
+
+  // Check each include pattern to find a match
+  for (let i = 0; i < includes.length; i++) {
+    const includePattern = includes[i];
+    const matcher = picomatch(includePattern.pattern);
+
+    if (matcher(filePath)) {
+      return {
+        included: true,
+        matchedPattern: {
+          pattern: includePattern.pattern,
+          basePath: includePattern.basePath,
+          index: i,
+        },
+      };
+    }
+  }
+
+  // No patterns matched
+  return { included: false, matchedPattern: null };
+}
+
+/**
+ * Get the headers needed to make a conditional request.
+ * Uses the etag and last-modified values from the meta store.
+ * @internal
+ */
+export function getHeaders({
+  init,
+  meta,
+  id,
+}: {
+  /** Initial headers to include */
+  init?: RequestInit["headers"];
+  /** Meta store to get etag and last-modified values from */
+  meta: ExtendedLoaderContext["meta"];
+  id: string;
+}): Headers {
+  const tag = `${id}-etag`;
+  const lastModifiedTag = `${id}-last-modified`;
+  const etag = meta.get(tag);
+  const lastModified = meta.get(lastModifiedTag);
+  const headers = new Headers(init);
+
+  if (etag) {
+    headers.set("If-None-Match", etag);
+  } else if (lastModified) {
+    headers.set("If-Modified-Since", lastModified);
+  }
+  return headers;
+}
+
+/**
+ * Store the etag or last-modified headers from a response in the meta store.
+ * @internal
+ */
+export function syncHeaders({
+  headers,
+  meta,
+  id,
+}: {
+  /** Headers from the response */
+  headers: Headers;
+  /** Meta store to store etag and last-modified values in */
+  meta: ExtendedLoaderContext["meta"];
+  /** id string */
+  id: string;
+}) {
+  const etag = headers.get("etag");
+  const lastModified = headers.get("last-modified");
+  const tag = `${id}-etag`;
+  const lastModifiedTag = `${id}-last-modified`;
+  meta.delete(tag);
+  meta.delete(lastModifiedTag);
+  if (etag) {
+    meta.set(tag, etag);
+  } else if (lastModified) {
+    meta.set(lastModifiedTag, lastModified);
+  }
+}

package/src/github.storage.spec.ts

@@ -0,0 +1,367 @@
+import { beforeEach, describe, it, expect, vi } from "vitest";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { syncFile, storeProcessedFile } from "./github.storage.js";
+import { createMockContext } from "./test-helpers.js";
+
+vi.mock("node:fs", async () => {
+  const actual = await vi.importActual("node:fs");
+  return {
+    ...actual,
+    existsSync: vi.fn(() => false),
+    promises: {
+      mkdir: vi.fn(),
+      writeFile: vi.fn(),
+    },
+  };
+});
+
+// Import the mocked fs after vi.mock so we can control return values per test
+import { existsSync, promises as fs } from "node:fs";
+
+const mockedExistsSync = existsSync as ReturnType<typeof vi.fn>;
+const mockedMkdir = fs.mkdir as ReturnType<typeof vi.fn>;
+const mockedWriteFile = fs.writeFile as ReturnType<typeof vi.fn>;
+
+const mockFile = {
+  sourcePath: "docs/guide.md",
+  targetPath: "src/content/docs/guide.md",
+  content: "# Guide\nContent here",
+  id: "docs/guide",
+};
+
+describe("syncFile", () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+    mockedExistsSync.mockReturnValue(false);
+    mockedMkdir.mockResolvedValue(undefined);
+    mockedWriteFile.mockResolvedValue(undefined);
+  });
+
+  it("creates directory and writes file when directory does not exist", async () => {
+    await syncFile("some/nested/dir/file.md", "content");
+
+    expect(mockedExistsSync).toHaveBeenCalledWith("some/nested/dir");
+    expect(mockedMkdir).toHaveBeenCalledWith("some/nested/dir", {
+      recursive: true,
+    });
+    expect(mockedWriteFile).toHaveBeenCalledWith(
+      "some/nested/dir/file.md",
+      "content",
+      "utf-8",
+    );
+  });
+
+  it("skips mkdir when directory already exists", async () => {
+    mockedExistsSync.mockReturnValue(true);
+
+    await syncFile("existing/dir/file.md", "content");
+
+    expect(mockedExistsSync).toHaveBeenCalledWith("existing/dir");
+    expect(mockedMkdir).not.toHaveBeenCalled();
+    expect(mockedWriteFile).toHaveBeenCalledWith(
+      "existing/dir/file.md",
+      "content",
+      "utf-8",
+    );
+  });
+
+  it("skips mkdir when path has no directory component", async () => {
+    await syncFile("file.md", "content");
+
+    // dir is "" which is falsy, so existsSync should not be called for dir check
+    expect(mockedMkdir).not.toHaveBeenCalled();
+    expect(mockedWriteFile).toHaveBeenCalledWith(
+      "file.md",
+      "content",
+      "utf-8",
+    );
+  });
+
+  it("writes content to the specified path", async () => {
+    const longContent = "# Title\n\nParagraph with **bold** and _italic_.";
+    await syncFile("output/test.md", longContent);
+
+    expect(mockedWriteFile).toHaveBeenCalledWith(
+      "output/test.md",
+      longContent,
+      "utf-8",
+    );
+  });
+});
+
+describe("storeProcessedFile", () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+    mockedExistsSync.mockReturnValue(false);
+    mockedMkdir.mockResolvedValue(undefined);
+    mockedWriteFile.mockResolvedValue(undefined);
+  });
+
+  it("stores a basic entry without getRenderFunction or contentModuleTypes", async () => {
+    const ctx = createMockContext();
+
+    const result = await storeProcessedFile(mockFile, ctx as any, false);
+
+    const stored = ctx._store.get("docs/guide");
+    expect(stored).toBeDefined();
+    expect(stored.id).toBe("docs/guide");
+    expect(stored.body).toBe(mockFile.content);
+    expect(stored.filePath).toBe(mockFile.targetPath);
+    expect(stored.digest).toBe(String(mockFile.content.length));
+    expect(stored.rendered).toBeUndefined();
+    expect(stored.deferredRender).toBeUndefined();
+    expect(result).toEqual({
+      id: "docs/guide",
+      filePath: "src/content/docs/guide.md",
+    });
+  });
+
+  it("stores entry with rendered content when getRenderFunction is present", async () => {
+    const ctx = createMockContext();
+    ctx.entryTypes.set(".md", {
+      getEntryInfo: async ({ contents }: any) => ({
+        body: contents,
+        data: {},
+      }),
+      getRenderFunction: async () => async () => ({
+        html: "<p>rendered</p>",
+      }),
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    const stored = ctx._store.get("docs/guide");
+    expect(stored.rendered).toEqual({ html: "<p>rendered</p>" });
+  });
+
+  it("logs error and stores undefined rendered when getRenderFunction throws", async () => {
+    const ctx = createMockContext();
+    ctx.entryTypes.set(".md", {
+      getEntryInfo: async ({ contents }: any) => ({
+        body: contents,
+        data: {},
+      }),
+      getRenderFunction: async () => async () => {
+        throw new Error("render failed");
+      },
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(ctx.logger.error).toHaveBeenCalledWith(
+      expect.stringContaining("render failed"),
+    );
+    const stored = ctx._store.get("docs/guide");
+    expect(stored).toBeDefined();
+    expect(stored.rendered).toBeUndefined();
+  });
+
+  it("sets deferredRender when entry type has contentModuleTypes", async () => {
+    const ctx = createMockContext();
+    ctx.entryTypes.set(".md", {
+      getEntryInfo: async ({ contents }: any) => ({
+        body: contents,
+        data: {},
+      }),
+      contentModuleTypes: "some types",
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    const stored = ctx._store.get("docs/guide");
+    expect(stored.deferredRender).toBe(true);
+    expect(stored.rendered).toBeUndefined();
+  });
+
+  it("deletes existing entry before setting when clear is true", async () => {
+    const ctx = createMockContext();
+    // Pre-populate the store with an existing entry
+    ctx.store.set({
+      id: "docs/guide",
+      data: {},
+      body: "old content",
+      filePath: "old/path.md",
+      digest: "0",
+    });
+
+    const deleteSpy = vi.spyOn(ctx.store, "delete");
+
+    await storeProcessedFile(mockFile, ctx as any, true);
+
+    expect(deleteSpy).toHaveBeenCalledWith("docs/guide");
+    const stored = ctx._store.get("docs/guide");
+    expect(stored.body).toBe(mockFile.content);
+  });
+
+  it("does not call delete when clear is true but entry does not exist", async () => {
+    const ctx = createMockContext();
+    const deleteSpy = vi.spyOn(ctx.store, "delete");
+
+    await storeProcessedFile(mockFile, ctx as any, true);
+
+    expect(deleteSpy).not.toHaveBeenCalled();
+    const stored = ctx._store.get("docs/guide");
+    expect(stored).toBeDefined();
+  });
+
+  it("skips writing file to disk when file already exists", async () => {
+    const ctx = createMockContext();
+    const expectedFileUrl = pathToFileURL(mockFile.targetPath);
+    mockedExistsSync.mockImplementation((path: string) => {
+      return path === fileURLToPath(expectedFileUrl);
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(mockedWriteFile).not.toHaveBeenCalled();
+    // Entry should still be stored
+    expect(ctx._store.get("docs/guide")).toBeDefined();
+  });
+
+  it("writes file to disk when file does not exist", async () => {
+    const ctx = createMockContext();
+    mockedExistsSync.mockReturnValue(false);
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(mockedWriteFile).toHaveBeenCalled();
+    expect(ctx.logger.verbose).toHaveBeenCalledWith(
+      expect.stringContaining("Writing docs/guide"),
+    );
+  });
+
+  it("throws when no entry type is found for the file extension", async () => {
+    const ctx = createMockContext();
+    const unknownFile = {
+      ...mockFile,
+      sourcePath: "docs/data.yaml",
+    };
+
+    await expect(
+      storeProcessedFile(unknownFile, ctx as any, false),
+    ).rejects.toThrow("No entry type found");
+  });
+
+  it("warns and throws when source path has no extension", async () => {
+    const ctx = createMockContext();
+    const noExtFile = {
+      ...mockFile,
+      sourcePath: "Makefile",
+    };
+
+    // "Makefile" split by "." yields ["Makefile"], .at(-1) = "Makefile"
+    // entryTypes won't have ".Makefile", so it throws
+    await expect(
+      storeProcessedFile(noExtFile, ctx as any, false),
+    ).rejects.toThrow("No entry type found");
+  });
+
+  it("returns id and filePath on success", async () => {
+    const ctx = createMockContext();
+
+    const result = await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(result).toEqual({
+      id: "docs/guide",
+      filePath: "src/content/docs/guide.md",
+    });
+  });
+
+  it("falls back to tmp.md when sourcePath is empty", async () => {
+    const ctx = createMockContext();
+    const fileWithoutSource = {
+      ...mockFile,
+      sourcePath: "",
+    };
+
+    // Empty sourcePath falls back to "tmp.md", which resolves to .md entry type
+    const result = await storeProcessedFile(
+      fileWithoutSource,
+      ctx as any,
+      false,
+    );
+
+    expect(result).toEqual({
+      id: "docs/guide",
+      filePath: "src/content/docs/guide.md",
+    });
+  });
+
+  it("uses file content for digest generation", async () => {
+    const ctx = createMockContext();
+    const digestSpy = vi.fn((content: string) => `digest-${content.length}`);
+    ctx.generateDigest = digestSpy;
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(digestSpy).toHaveBeenCalledWith(mockFile.content);
+    const stored = ctx._store.get("docs/guide");
+    expect(stored.digest).toBe(`digest-${mockFile.content.length}`);
+  });
+
+  it("logs debug message for existing entry update", async () => {
+    const ctx = createMockContext();
+    // Pre-populate so it's an update
+    ctx.store.set({
+      id: "docs/guide",
+      data: {},
+      body: "old",
+      filePath: "old.md",
+      digest: "0",
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(ctx.logger.debug).toHaveBeenCalledWith(
+      expect.stringContaining("updating"),
+    );
+  });
+
+  it("logs debug message for new entry addition", async () => {
+    const ctx = createMockContext();
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(ctx.logger.debug).toHaveBeenCalledWith(
+      expect.stringContaining("adding"),
+    );
+  });
+
+  it("passes file content and fileUrl to getEntryInfo", async () => {
+    const ctx = createMockContext();
+    const getEntryInfoSpy = vi.fn(async ({ contents }: any) => ({
+      body: contents,
+      data: { title: "Test" },
+    }));
+    ctx.entryTypes.set(".md", { getEntryInfo: getEntryInfoSpy });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(getEntryInfoSpy).toHaveBeenCalledWith({
+      contents: mockFile.content,
+      fileUrl: pathToFileURL(mockFile.targetPath),
+    });
+    const stored = ctx._store.get("docs/guide");
+    expect(stored.body).toBe(mockFile.content);
+  });
+
+  it("passes config to getRenderFunction", async () => {
+    const ctx = createMockContext();
+    const mockConfig = { root: "/project" };
+    ctx.config = mockConfig;
+    const getRenderFunctionSpy = vi
+      .fn()
+      .mockResolvedValue(async () => ({ html: "<p>ok</p>" }));
+    ctx.entryTypes.set(".md", {
+      getEntryInfo: async ({ contents }: any) => ({
+        body: contents,
+        data: {},
+      }),
+      getRenderFunction: getRenderFunctionSpy,
+    });
+
+    await storeProcessedFile(mockFile, ctx as any, false);
+
+    expect(getRenderFunctionSpy).toHaveBeenCalledWith(mockConfig);
+  });
+});