@larkiny/astro-github-loader 0.11.3 → 0.13.0

package/dist/github.storage.js

@@ -0,0 +1,115 @@
+import { existsSync, promises as fs } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+/**
+ * Ensures directory exists and writes file to disk.
+ * Validates that the resolved path stays within the project root.
+ * @internal
+ */
+export async function syncFile(filePath, content) {
+    const resolved = resolve(filePath);
+    if (!resolved.startsWith(process.cwd())) {
+        throw new Error(`syncFile: path "${filePath}" resolves outside project root`);
+    }
+    const dir = resolved.substring(0, resolved.lastIndexOf("/"));
+    if (dir && !existsSync(dir)) {
+        await fs.mkdir(dir, { recursive: true });
+    }
+    await fs.writeFile(resolved, content, "utf-8");
+}
+/**
+ * Stores a processed file in Astro's content store
+ * @internal
+ */
+export async function storeProcessedFile(file, context, clear) {
+    const { store, generateDigest, entryTypes, logger, parseData, config } = context;
+    function configForFile(filePath) {
+        const ext = filePath.split(".").at(-1);
+        if (!ext) {
+            logger.warn(`No extension found for ${filePath}`);
+            return;
+        }
+        return entryTypes?.get(`.${ext}`);
+    }
+    const entryType = configForFile(file.sourcePath || "tmp.md");
+    if (!entryType)
+        throw new Error("No entry type found");
+    const fileUrl = pathToFileURL(file.targetPath);
+    const { body, data } = await entryType.getEntryInfo({
+        contents: file.content,
+        fileUrl: fileUrl,
+    });
+    // Generate digest for storage (repository-level caching handles change detection)
+    const digest = generateDigest(file.content);
+    const existingEntry = store.get(file.id);
+    if (existingEntry) {
+        logger.debug(`🔄 File ${file.id} - updating`);
+    }
+    else {
+        logger.debug(`📄 File ${file.id} - adding`);
+    }
+    // Write file to disk
+    if (!existsSync(fileURLToPath(fileUrl))) {
+        logger.verbose(`Writing ${file.id} to ${fileUrl}`);
+        await syncFile(fileURLToPath(fileUrl), file.content);
+    }
+    const parsedData = await parseData({
+        id: file.id,
+        data,
+        filePath: fileUrl.toString(),
+    });
+    // When clear mode is enabled, delete the existing entry before setting the new one.
+    // This provides atomic replacement without breaking Astro's content collection,
+    // as opposed to calling store.clear() which empties everything at once.
+    if (clear && existingEntry) {
+        logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
+        store.delete(file.id);
+    }
+    // Store in content store
+    if (entryType.getRenderFunction) {
+        logger.verbose(`Rendering ${file.id}`);
+        const render = await entryType.getRenderFunction(config);
+        let rendered = undefined;
+        try {
+            rendered = await render?.({
+                id: file.id,
+                data,
+                body,
+                filePath: fileUrl.toString(),
+                digest,
+            });
+        }
+        catch (error) {
+            logger.error(`Error rendering ${file.id}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        logger.debug(`🔍 Storing collection entry: ${file.id} (${file.sourcePath} -> ${file.targetPath})`);
+        store.set({
+            id: file.id,
+            data: parsedData,
+            body,
+            filePath: file.targetPath,
+            digest,
+            rendered,
+        });
+    }
+    else if ("contentModuleTypes" in entryType) {
+        store.set({
+            id: file.id,
+            data: parsedData,
+            body,
+            filePath: file.targetPath,
+            digest,
+            deferredRender: true,
+        });
+    }
+    else {
+        store.set({
+            id: file.id,
+            data: parsedData,
+            body,
+            filePath: file.targetPath,
+            digest,
+        });
+    }
+    return { id: file.id, filePath: file.targetPath };
+}

package/dist/github.types.d.ts

@@ -3,7 +3,7 @@ import type { ContentEntryType } from "astro";
 import type { MarkdownHeading } from "@astrojs/markdown-remark";
 import { Octokit } from "octokit";
 import type { LinkHandler } from "./github.link-transform.js";
-import type { LogLevel } from "./github.logger.js";
+import type { LogLevel, Logger } from "./github.logger.js";
 /**
  * Context information for link transformations
  */
@@ -26,7 +26,7 @@ export interface LinkMapping {
     /** Pattern to match (string or regex) */
     pattern: string | RegExp;
     /** Replacement string or function */
-    replacement: string | ((match: string, anchor: string, context: any) => string);
+    replacement: string | ((match: string, anchor: string, context: LinkTransformContext) => string);
     /** Apply to all links, not just unresolved internal links (default: false) */
     global?: boolean;
     /** Function to determine if this mapping should apply to the current file context */
@@ -162,7 +162,7 @@ export type CollectionEntryOptions = {
      * The LoaderContext may contain properties and methods that offer
      * control or inspection over the loading behavior.
      */
-    context: LoaderContext;
+    context: ExtendedLoaderContext;
     /**
      * An instance of the Octokit library, which provides a way to interact
      * with GitHub's REST API. This variable allows you to access and perform
@@ -225,6 +225,16 @@ export interface RenderedContent {
         [key: string]: unknown;
     };
 }
+/**
+ * Represents a version of a library variant to display in the devportal's version picker.
+ * Versions are manually curated in the import config — no auto-discovery.
+ */
+export interface VersionConfig {
+    /** URL segment for this version (e.g., "latest", "v8.0.0") */
+    slug: string;
+    /** Display name for this version (e.g., "Latest", "v8.0.0") */
+    label: string;
+}
 /**
  * Represents configuration options for importing content from GitHub repositories.
  */
@@ -233,6 +243,12 @@ export type ImportOptions = {
      * Display name for this configuration (used in logging)
      */
     name?: string;
+    /**
+     * Custom state key for import tracking. When provided, overrides the default
+     * `owner/repo@ref` key used to track import state. This allows the same repo
+     * to be imported independently to multiple locations.
+     */
+    stateKey?: string;
     /**
      * Repository owner
      */
@@ -286,18 +302,38 @@ export type ImportOptions = {
      * @default 'default'
      */
     logLevel?: LogLevel;
+    /**
+     * Language for this import variant (e.g., "TypeScript", "Python", "Go").
+     * Used for logging and passed through to the devportal for UI display.
+     */
+    language?: string;
+    /**
+     * Versions to display in the devportal's version picker.
+     * Informational — tells the loader which version folders exist in the source content.
+     * The loader imports content as-is; the version folder structure carries through from source to destination.
+     */
+    versions?: VersionConfig[];
 };
 export type FetchOptions = RequestInit & {
     signal?: AbortSignal;
     concurrency?: number;
 };
 /**
- * @internal
+ * Astro loader context extended with optional entry type support.
+ * Use this type when calling `.load(context as LoaderContext)` in multi-loader patterns.
  */
 export interface LoaderContext extends AstroLoaderContext {
     /** @internal */
     entryTypes?: Map<string, ContentEntryType>;
 }
+/**
+ * LoaderContext with Astro's logger replaced by our Logger class.
+ * Used by internal functions that need verbose/logFileProcessing/etc.
+ * @internal
+ */
+export type ExtendedLoaderContext = Omit<LoaderContext, "logger"> & {
+    logger: Logger;
+};
 /**
  * @internal
  */

package/dist/index.d.ts

@@ -1,6 +1,8 @@
-export * from './github.auth.js';
-export * from './github.constants.js';
-export * from './github.content.js';
-export * from './github.loader.js';
-export * from './github.types.js';
-export * from './github.link-transform.js';
+export { githubLoader } from "./github.loader.js";
+export { createAuthenticatedOctokit, createOctokitFromEnv, } from "./github.auth.js";
+export type { GithubLoaderOptions, ImportOptions, FetchOptions, IncludePattern, PathMappingValue, EnhancedPathMapping, VersionConfig, LoaderContext, } from "./github.types.js";
+export type { TransformFunction, TransformContext, MatchedPattern, } from "./github.types.js";
+export type { LinkMapping, LinkTransformContext, ImportLinkTransformOptions, } from "./github.types.js";
+export type { LinkHandler } from "./github.link-transform.js";
+export type { GitHubAuthConfig, GitHubAppAuthConfig, GitHubPATAuthConfig, } from "./github.auth.js";
+export type { LogLevel } from "./github.logger.js";

package/dist/index.js

@@ -1,6 +1,3 @@
-export * from './github.auth.js';
-export * from './github.constants.js';
-export * from './github.content.js';
-export * from './github.loader.js';
-export * from './github.types.js';
-export * from './github.link-transform.js';
+// Public API — functions
+export { githubLoader } from "./github.loader.js";
+export { createAuthenticatedOctokit, createOctokitFromEnv, } from "./github.auth.js";

package/dist/test-helpers.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Creates a mock Astro LoaderContext with all required properties.
+ * The returned store is a real Map wrapped in the store interface,
+ * so tests can inspect stored entries directly.
+ */
+export declare function createMockContext(): {
+    store: {
+        set: (entry: any) => any;
+        get: (id: string) => any;
+        delete: (id: string) => boolean;
+        clear: () => void;
+        entries: () => MapIterator<[string, any]>;
+        keys: () => MapIterator<string>;
+        values: () => MapIterator<any>;
+    };
+    meta: Map<string, string>;
+    logger: {
+        info: import("vitest").Mock<(...args: any[]) => any>;
+        warn: import("vitest").Mock<(...args: any[]) => any>;
+        error: import("vitest").Mock<(...args: any[]) => any>;
+        debug: import("vitest").Mock<(...args: any[]) => any>;
+        verbose: import("vitest").Mock<(...args: any[]) => any>;
+        logFileProcessing: import("vitest").Mock<(...args: any[]) => any>;
+        logImportSummary: import("vitest").Mock<(...args: any[]) => any>;
+        logAssetProcessing: import("vitest").Mock<(...args: any[]) => any>;
+        withSpinner: (_msg: string, fn: () => Promise<any>) => Promise<any>;
+        getLevel: () => "default";
+    };
+    config: {};
+    entryTypes: Map<string, {
+        getEntryInfo: ({ contents, }: {
+            contents: string;
+            fileUrl: URL;
+        }) => Promise<{
+            body: string;
+            data: {};
+        }>;
+    }>;
+    generateDigest: (content: string) => string;
+    parseData: (data: any) => Promise<any>;
+    /** Direct access to the underlying store Map for assertions */
+    _store: Map<string, any>;
+    /** Direct access to the underlying meta Map for assertions */
+    _meta: Map<string, string>;
+};
+/** Standard mock commit used across tests */
+export declare const MOCK_COMMIT: {
+    sha: string;
+    commit: {
+        tree: {
+            sha: string;
+        };
+        message: string;
+        author: {
+            name: string;
+            email: string;
+            date: string;
+        };
+        committer: {
+            name: string;
+            email: string;
+            date: string;
+        };
+    };
+};
+/** Mock tree data representing a typical repository structure */
+export declare const MOCK_TREE_DATA: {
+    sha: string;
+    url: string;
+    tree: ({
+        path: string;
+        mode: string;
+        type: string;
+        sha: string;
+        size: number;
+        url: string;
+    } | {
+        path: string;
+        mode: string;
+        type: string;
+        sha: string;
+        url: string;
+        size?: undefined;
+    })[];
+    truncated: boolean;
+};
+/**
+ * Creates an Octokit instance with mocked API methods for listCommits and getTree.
+ * Returns both the instance and the spies for assertions.
+ */
+export declare function createMockOctokit(options?: {
+    treeData?: typeof MOCK_TREE_DATA;
+    commitData?: typeof MOCK_COMMIT;
+}): {
+    octokit: import("@octokit/core").Octokit & {
+        paginate: import("@octokit/plugin-paginate-rest").PaginateInterface;
+    } & import("@octokit/plugin-paginate-graphql").paginateGraphQLInterface & import("@octokit/plugin-rest-endpoint-methods").Api & {
+        retry: {
+            retryRequest: (error: import("octokit").RequestError, retries: number, retryAfter: number) => import("octokit").RequestError;
+        };
+    };
+    spies: {
+        listCommitsSpy: import("vitest").MockInstance<{
+            (params?: import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["repos"]["listCommits"]["parameters"]): Promise<import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["repos"]["listCommits"]["response"]>;
+            defaults: import("@octokit/types").RequestInterface["defaults"];
+            endpoint: import("@octokit/types").EndpointInterface<{
+                url: string;
+            }>;
+        }>;
+        getTreeSpy: import("vitest").MockInstance<{
+            (params?: import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["git"]["getTree"]["parameters"]): Promise<import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["git"]["getTree"]["response"]>;
+            defaults: import("@octokit/types").RequestInterface["defaults"];
+            endpoint: import("@octokit/types").EndpointInterface<{
+                url: string;
+            }>;
+        }>;
+        getContentSpy: import("vitest").MockInstance<{
+            (params?: import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["repos"]["getContent"]["parameters"]): Promise<import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["repos"]["getContent"]["response"]>;
+            defaults: import("@octokit/types").RequestInterface["defaults"];
+            endpoint: import("@octokit/types").EndpointInterface<{
+                url: string;
+            }>;
+        }>;
+    };
+};
+/**
+ * Sets up a global fetch mock that returns markdown content.
+ * Returns the mock for assertions.
+ */
+export declare function mockFetch(content?: string): import("vitest").Mock<(...args: any[]) => any>;

package/dist/test-helpers.js

@@ -0,0 +1,194 @@
+/**
+ * Shared test helpers for astro-github-loader test suite.
+ * Provides factory functions for creating mock Astro loader contexts,
+ * Octokit instances with pre-configured spies, and common fixtures.
+ */
+import { vi } from "vitest";
+import { Octokit } from "octokit";
+/**
+ * Creates a mock Astro LoaderContext with all required properties.
+ * The returned store is a real Map wrapped in the store interface,
+ * so tests can inspect stored entries directly.
+ */
+export function createMockContext() {
+    const store = new Map();
+    const meta = new Map();
+    return {
+        store: {
+            set: (entry) => {
+                store.set(entry.id, entry);
+                return entry;
+            },
+            get: (id) => store.get(id),
+            delete: (id) => store.delete(id),
+            clear: () => store.clear(),
+            entries: () => store.entries(),
+            keys: () => store.keys(),
+            values: () => store.values(),
+        },
+        meta,
+        logger: {
+            info: vi.fn(),
+            warn: vi.fn(),
+            error: vi.fn(),
+            debug: vi.fn(),
+            verbose: vi.fn(),
+            logFileProcessing: vi.fn(),
+            logImportSummary: vi.fn(),
+            logAssetProcessing: vi.fn(),
+            withSpinner: async (_msg, fn) => await fn(),
+            getLevel: () => "default",
+        },
+        config: {},
+        entryTypes: new Map([
+            [
+                ".md",
+                {
+                    getEntryInfo: async ({ contents, }) => ({
+                        body: contents,
+                        data: {},
+                    }),
+                },
+            ],
+        ]),
+        generateDigest: (content) => String(content.length),
+        parseData: async (data) => data,
+        /** Direct access to the underlying store Map for assertions */
+        _store: store,
+        /** Direct access to the underlying meta Map for assertions */
+        _meta: meta,
+    };
+}
+/** Standard mock commit used across tests */
+export const MOCK_COMMIT = {
+    sha: "abc123def456",
+    commit: {
+        tree: { sha: "tree123abc456" },
+        message: "Test commit",
+        author: {
+            name: "Test Author",
+            email: "test@example.com",
+            date: "2024-01-01T00:00:00Z",
+        },
+        committer: {
+            name: "Test Committer",
+            email: "test@example.com",
+            date: "2024-01-01T00:00:00Z",
+        },
+    },
+};
+/** Mock tree data representing a typical repository structure */
+export const MOCK_TREE_DATA = {
+    sha: "tree123abc456",
+    url: "https://api.github.com/repos/test/repo/git/trees/tree123abc456",
+    tree: [
+        {
+            path: "docs/algokit.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file1sha",
+            size: 1234,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file1sha",
+        },
+        {
+            path: "docs/features",
+            mode: "040000",
+            type: "tree",
+            sha: "dir1sha",
+            url: "https://api.github.com/repos/test/repo/git/trees/dir1sha",
+        },
+        {
+            path: "docs/features/accounts.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file2sha",
+            size: 2345,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file2sha",
+        },
+        {
+            path: "docs/features/tasks.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file3sha",
+            size: 3456,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file3sha",
+        },
+        {
+            path: "docs/features/generate.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file4sha",
+            size: 4567,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file4sha",
+        },
+        {
+            path: "docs/cli/index.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file5sha",
+            size: 5678,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file5sha",
+        },
+        {
+            path: "README.md",
+            mode: "100644",
+            type: "blob",
+            sha: "file6sha",
+            size: 678,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file6sha",
+        },
+        {
+            path: "package.json",
+            mode: "100644",
+            type: "blob",
+            sha: "file7sha",
+            size: 789,
+            url: "https://api.github.com/repos/test/repo/git/blobs/file7sha",
+        },
+    ],
+    truncated: false,
+};
+/**
+ * Creates an Octokit instance with mocked API methods for listCommits and getTree.
+ * Returns both the instance and the spies for assertions.
+ */
+export function createMockOctokit(options) {
+    const octokit = new Octokit({ auth: "mock-token" });
+    const commit = options?.commitData ?? MOCK_COMMIT;
+    const tree = options?.treeData ?? MOCK_TREE_DATA;
+    const listCommitsSpy = vi
+        .spyOn(octokit.rest.repos, "listCommits")
+        .mockResolvedValue({
+        data: [commit],
+        status: 200,
+        url: "",
+        headers: {},
+    });
+    const getTreeSpy = vi.spyOn(octokit.rest.git, "getTree").mockResolvedValue({
+        data: tree,
+        status: 200,
+        url: "",
+        headers: {},
+    });
+    const getContentSpy = vi
+        .spyOn(octokit.rest.repos, "getContent")
+        .mockResolvedValue({ data: [], status: 200, url: "", headers: {} });
+    return {
+        octokit,
+        spies: { listCommitsSpy, getTreeSpy, getContentSpy },
+    };
+}
+/**
+ * Sets up a global fetch mock that returns markdown content.
+ * Returns the mock for assertions.
+ */
+export function mockFetch(content = "# Test Content\n\nThis is test markdown content.") {
+    const fetchMock = vi.fn().mockResolvedValue({
+        ok: true,
+        status: 200,
+        headers: new Headers(),
+        text: async () => content,
+    });
+    global.fetch = fetchMock;
+    return fetchMock;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@larkiny/astro-github-loader",
   "type": "module",
-  "version": "0.11.3",
+  "version": "0.13.0",
   "description": "Load content from GitHub repositories into Astro content collections with asset management and content transformations",
   "keywords": [
     "astro",
@@ -38,6 +38,7 @@
     "build": "tsc",
     "test": "vitest run",
     "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
     "lint": "eslint .",
     "prettier": "prettier --check .",
     "preview": "astro preview",
@@ -57,6 +58,7 @@
     "@types/node": "^22.14.0",
     "@types/picomatch": "^4.0.0",
     "@typescript-eslint/parser": "^8.29.0",
+    "@vitest/coverage-v8": "^3.2.4",
     "eslint": "^9.24.0",
     "eslint-plugin-astro": "^1.3.1",
     "prettier": "3.5.3",