@larkiny/astro-github-loader 0.11.2 → 0.12.0

package/src/github.storage.ts

@@ -0,0 +1,127 @@
+import { existsSync, promises as fs } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import type { ImportedFile } from "./github.link-transform.js";
+import type { ExtendedLoaderContext } from "./github.types.js";
+
+/**
+ * Ensures directory exists and writes file to disk.
+ * @internal
+ */
+export async function syncFile(path: string, content: string) {
+  const dir = path.substring(0, path.lastIndexOf("/"));
+  if (dir && !existsSync(dir)) {
+    await fs.mkdir(dir, { recursive: true });
+  }
+  await fs.writeFile(path, content, "utf-8");
+}
+
+/**
+ * Stores a processed file in Astro's content store
+ * @internal
+ */
+export async function storeProcessedFile(
+  file: ImportedFile,
+  context: ExtendedLoaderContext,
+  clear: boolean,
+) {
+  const { store, generateDigest, entryTypes, logger, parseData, config } =
+    context;
+
+  function configForFile(filePath: string) {
+    const ext = filePath.split(".").at(-1);
+    if (!ext) {
+      logger.warn(`No extension found for ${filePath}`);
+      return;
+    }
+    return entryTypes?.get(`.${ext}`);
+  }
+
+  const entryType = configForFile(file.sourcePath || "tmp.md");
+  if (!entryType) throw new Error("No entry type found");
+
+  const fileUrl = pathToFileURL(file.targetPath);
+  const { body, data } = await entryType.getEntryInfo({
+    contents: file.content,
+    fileUrl: fileUrl,
+  });
+
+  // Generate digest for storage (repository-level caching handles change detection)
+  const digest = generateDigest(file.content);
+  const existingEntry = store.get(file.id);
+
+  if (existingEntry) {
+    logger.debug(`🔄 File ${file.id} - updating`);
+  } else {
+    logger.debug(`📄 File ${file.id} - adding`);
+  }
+
+  // Write file to disk
+  if (!existsSync(fileURLToPath(fileUrl))) {
+    logger.verbose(`Writing ${file.id} to ${fileUrl}`);
+    await syncFile(fileURLToPath(fileUrl), file.content);
+  }
+
+  const parsedData = await parseData({
+    id: file.id,
+    data,
+    filePath: fileUrl.toString(),
+  });
+
+  // When clear mode is enabled, delete the existing entry before setting the new one.
+  // This provides atomic replacement without breaking Astro's content collection,
+  // as opposed to calling store.clear() which empties everything at once.
+  if (clear && existingEntry) {
+    logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
+    store.delete(file.id);
+  }
+
+  // Store in content store
+  if (entryType.getRenderFunction) {
+    logger.verbose(`Rendering ${file.id}`);
+    const render = await entryType.getRenderFunction(config);
+    let rendered = undefined;
+    try {
+      rendered = await render?.({
+        id: file.id,
+        data,
+        body,
+        filePath: fileUrl.toString(),
+        digest,
+      });
+    } catch (error: unknown) {
+      logger.error(
+        `Error rendering ${file.id}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+    logger.debug(
+      `🔍 Storing collection entry: ${file.id} (${file.sourcePath} -> ${file.targetPath})`,
+    );
+    store.set({
+      id: file.id,
+      data: parsedData,
+      body,
+      filePath: file.targetPath,
+      digest,
+      rendered,
+    });
+  } else if ("contentModuleTypes" in entryType) {
+    store.set({
+      id: file.id,
+      data: parsedData,
+      body,
+      filePath: file.targetPath,
+      digest,
+      deferredRender: true,
+    });
+  } else {
+    store.set({
+      id: file.id,
+      data: parsedData,
+      body,
+      filePath: file.targetPath,
+      digest,
+    });
+  }
+
+  return { id: file.id, filePath: file.targetPath };
+}

package/src/github.types.ts

@@ -3,12 +3,12 @@ import type {
   LoaderContext as AstroLoaderContext,
 } from "astro/loaders";
 import type { ContentEntryType } from "astro";
-import type {MarkdownHeading} from "@astrojs/markdown-remark";
-import {Octokit} from "octokit";
+import type { MarkdownHeading } from "@astrojs/markdown-remark";
+import { Octokit } from "octokit";
 
 // Import link transformation types from the dedicated module
 import type { LinkHandler } from "./github.link-transform.js";
-import type { LogLevel } from "./github.logger.js";
+import type { LogLevel, Logger } from "./github.logger.js";
 
 /**
  * Context information for link transformations
@@ -33,7 +33,13 @@ export interface LinkMapping {
   /** Pattern to match (string or regex) */
   pattern: string | RegExp;
   /** Replacement string or function */
-  replacement: string | ((match: string, anchor: string, context: any) => string);
+  replacement:
+    | string
+    | ((
+        match: string,
+        anchor: string,
+        context: LinkTransformContext,
+      ) => string);
   /** Apply to all links, not just unresolved internal links (default: false) */
   global?: boolean;
   /** Function to determine if this mapping should apply to the current file context */
@@ -86,7 +92,10 @@ export interface TransformContext {
  * @param context - Context information about the file being processed
  * @returns The transformed content
  */
-export type TransformFunction = (content: string, context: TransformContext) => string;
+export type TransformFunction = (
+  content: string,
+  context: TransformContext,
+) => string;
 
 /**
  * Enhanced path mapping configuration that supports cross-section linking
@@ -139,7 +148,6 @@ export interface IncludePattern {
   pathMappings?: Record<string, PathMappingValue>;
 }
 
-
 export type GithubLoaderOptions = {
   octokit: Octokit;
   configs: Array<ImportOptions>;
@@ -179,7 +187,7 @@ export type CollectionEntryOptions = {
    * The LoaderContext may contain properties and methods that offer
    * control or inspection over the loading behavior.
    */
-  context: LoaderContext;
+  context: ExtendedLoaderContext;
   /**
    * An instance of the Octokit library, which provides a way to interact
    * with GitHub's REST API. This variable allows you to access and perform
@@ -216,6 +224,13 @@ export type CollectionEntryOptions = {
    * @default false
    */
   force?: boolean;
+  /**
+   * When true, deletes existing store entries before setting new ones.
+   * This enables atomic replacement of entries without breaking the content collection.
+   * Passed from GithubLoaderOptions.clear
+   * @internal
+   */
+  clear?: boolean;
 };
 
 /**
@@ -237,6 +252,17 @@ export interface RenderedContent {
   };
 }
 
+/**
+ * Represents a version of a library variant to display in the devportal's version picker.
+ * Versions are manually curated in the import config — no auto-discovery.
+ */
+export interface VersionConfig {
+  /** URL segment for this version (e.g., "latest", "v8.0.0") */
+  slug: string;
+  /** Display name for this version (e.g., "Latest", "v8.0.0") */
+  label: string;
+}
+
 /**
  * Represents configuration options for importing content from GitHub repositories.
  */
@@ -298,6 +324,17 @@ export type ImportOptions = {
    * @default 'default'
    */
   logLevel?: LogLevel;
+  /**
+   * Language for this import variant (e.g., "TypeScript", "Python", "Go").
+   * Used for logging and passed through to the devportal for UI display.
+   */
+  language?: string;
+  /**
+   * Versions to display in the devportal's version picker.
+   * Informational — tells the loader which version folders exist in the source content.
+   * The loader imports content as-is; the version folder structure carries through from source to destination.
+   */
+  versions?: VersionConfig[];
 };
 
 export type FetchOptions = RequestInit & {
@@ -306,13 +343,23 @@ export type FetchOptions = RequestInit & {
 };
 
 /**
- * @internal
+ * Astro loader context extended with optional entry type support.
+ * Use this type when calling `.load(context as LoaderContext)` in multi-loader patterns.
  */
 export interface LoaderContext extends AstroLoaderContext {
   /** @internal */
   entryTypes?: Map<string, ContentEntryType>;
 }
 
+/**
+ * LoaderContext with Astro's logger replaced by our Logger class.
+ * Used by internal functions that need verbose/logFileProcessing/etc.
+ * @internal
+ */
+export type ExtendedLoaderContext = Omit<LoaderContext, "logger"> & {
+  logger: Logger;
+};
+
 /**
  * @internal
  */
@@ -321,7 +368,6 @@ export interface Loader extends AstroLoader {
   load: (context: LoaderContext) => Promise<void>;
 }
 
-
 /**
  * Statistics for a sync operation
  */

package/src/index.ts

@@ -1,6 +1,43 @@
-export * from './github.auth.js'
-export * from './github.constants.js'
-export * from './github.content.js'
-export * from './github.loader.js'
-export * from './github.types.js'
-export * from './github.link-transform.js'
+// Public API — functions
+export { githubLoader } from "./github.loader.js";
+export {
+  createAuthenticatedOctokit,
+  createOctokitFromEnv,
+} from "./github.auth.js";
+
+// Public API — types: loader config
+export type {
+  GithubLoaderOptions,
+  ImportOptions,
+  FetchOptions,
+  IncludePattern,
+  PathMappingValue,
+  EnhancedPathMapping,
+  VersionConfig,
+  LoaderContext,
+} from "./github.types.js";
+
+// Public API — types: transforms
+export type {
+  TransformFunction,
+  TransformContext,
+  MatchedPattern,
+} from "./github.types.js";
+
+// Public API — types: link transforms
+export type {
+  LinkMapping,
+  LinkTransformContext,
+  ImportLinkTransformOptions,
+} from "./github.types.js";
+export type { LinkHandler } from "./github.link-transform.js";
+
+// Public API — types: auth
+export type {
+  GitHubAuthConfig,
+  GitHubAppAuthConfig,
+  GitHubPATAuthConfig,
+} from "./github.auth.js";
+
+// Public API — types: logging
+export type { LogLevel } from "./github.logger.js";

package/src/test-helpers.ts

@@ -0,0 +1,215 @@
+/**
+ * Shared test helpers for astro-github-loader test suite.
+ * Provides factory functions for creating mock Astro loader contexts,
+ * Octokit instances with pre-configured spies, and common fixtures.
+ */
+import { vi } from "vitest";
+import { Octokit } from "octokit";
+import type { ImportOptions } from "./github.types.js";
+
+/**
+ * Creates a mock Astro LoaderContext with all required properties.
+ * The returned store is a real Map wrapped in the store interface,
+ * so tests can inspect stored entries directly.
+ */
+export function createMockContext() {
+  const store = new Map<string, any>();
+  const meta = new Map<string, string>();
+
+  return {
+    store: {
+      set: (entry: any) => {
+        store.set(entry.id, entry);
+        return entry;
+      },
+      get: (id: string) => store.get(id),
+      delete: (id: string) => store.delete(id),
+      clear: () => store.clear(),
+      entries: () => store.entries(),
+      keys: () => store.keys(),
+      values: () => store.values(),
+    },
+    meta,
+    logger: {
+      info: vi.fn(),
+      warn: vi.fn(),
+      error: vi.fn(),
+      debug: vi.fn(),
+      verbose: vi.fn(),
+      logFileProcessing: vi.fn(),
+      logImportSummary: vi.fn(),
+      logAssetProcessing: vi.fn(),
+      withSpinner: async (_msg: string, fn: () => Promise<any>) => await fn(),
+      getLevel: () => "default" as const,
+    },
+    config: {},
+    entryTypes: new Map([
+      [
+        ".md",
+        {
+          getEntryInfo: async ({
+            contents,
+          }: {
+            contents: string;
+            fileUrl: URL;
+          }) => ({
+            body: contents,
+            data: {},
+          }),
+        },
+      ],
+    ]),
+    generateDigest: (content: string) => String(content.length),
+    parseData: async (data: any) => data,
+    /** Direct access to the underlying store Map for assertions */
+    _store: store,
+    /** Direct access to the underlying meta Map for assertions */
+    _meta: meta,
+  };
+}
+
+/** Standard mock commit used across tests */
+export const MOCK_COMMIT = {
+  sha: "abc123def456",
+  commit: {
+    tree: { sha: "tree123abc456" },
+    message: "Test commit",
+    author: {
+      name: "Test Author",
+      email: "test@example.com",
+      date: "2024-01-01T00:00:00Z",
+    },
+    committer: {
+      name: "Test Committer",
+      email: "test@example.com",
+      date: "2024-01-01T00:00:00Z",
+    },
+  },
+};
+
+/** Mock tree data representing a typical repository structure */
+export const MOCK_TREE_DATA = {
+  sha: "tree123abc456",
+  url: "https://api.github.com/repos/test/repo/git/trees/tree123abc456",
+  tree: [
+    {
+      path: "docs/algokit.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file1sha",
+      size: 1234,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file1sha",
+    },
+    {
+      path: "docs/features",
+      mode: "040000",
+      type: "tree",
+      sha: "dir1sha",
+      url: "https://api.github.com/repos/test/repo/git/trees/dir1sha",
+    },
+    {
+      path: "docs/features/accounts.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file2sha",
+      size: 2345,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file2sha",
+    },
+    {
+      path: "docs/features/tasks.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file3sha",
+      size: 3456,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file3sha",
+    },
+    {
+      path: "docs/features/generate.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file4sha",
+      size: 4567,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file4sha",
+    },
+    {
+      path: "docs/cli/index.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file5sha",
+      size: 5678,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file5sha",
+    },
+    {
+      path: "README.md",
+      mode: "100644",
+      type: "blob",
+      sha: "file6sha",
+      size: 678,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file6sha",
+    },
+    {
+      path: "package.json",
+      mode: "100644",
+      type: "blob",
+      sha: "file7sha",
+      size: 789,
+      url: "https://api.github.com/repos/test/repo/git/blobs/file7sha",
+    },
+  ],
+  truncated: false,
+};
+
+/**
+ * Creates an Octokit instance with mocked API methods for listCommits and getTree.
+ * Returns both the instance and the spies for assertions.
+ */
+export function createMockOctokit(options?: {
+  treeData?: typeof MOCK_TREE_DATA;
+  commitData?: typeof MOCK_COMMIT;
+}) {
+  const octokit = new Octokit({ auth: "mock-token" });
+  const commit = options?.commitData ?? MOCK_COMMIT;
+  const tree = options?.treeData ?? MOCK_TREE_DATA;
+
+  const listCommitsSpy = vi
+    .spyOn(octokit.rest.repos, "listCommits")
+    .mockResolvedValue({
+      data: [commit],
+      status: 200,
+      url: "",
+      headers: {},
+    } as any);
+
+  const getTreeSpy = vi.spyOn(octokit.rest.git, "getTree").mockResolvedValue({
+    data: tree,
+    status: 200,
+    url: "",
+    headers: {},
+  } as any);
+
+  const getContentSpy = vi
+    .spyOn(octokit.rest.repos, "getContent")
+    .mockResolvedValue({ data: [], status: 200, url: "", headers: {} } as any);
+
+  return {
+    octokit,
+    spies: { listCommitsSpy, getTreeSpy, getContentSpy },
+  };
+}
+
+/**
+ * Sets up a global fetch mock that returns markdown content.
+ * Returns the mock for assertions.
+ */
+export function mockFetch(
+  content: string = "# Test Content\n\nThis is test markdown content.",
+) {
+  const fetchMock = vi.fn().mockResolvedValue({
+    ok: true,
+    status: 200,
+    headers: new Headers(),
+    text: async () => content,
+  } as any);
+  global.fetch = fetchMock;
+  return fetchMock;
+}