@larkiny/astro-github-loader 0.9.0

package/dist/github.link-transform.js

@@ -0,0 +1,321 @@
+import { slug } from 'github-slugger';
+import path from 'node:path';
+/**
+ * Extract anchor fragment from a link
+ */
+function extractAnchor(link) {
+    const anchorMatch = link.match(/#.*$/);
+    const anchor = anchorMatch ? anchorMatch[0] : '';
+    const path = link.replace(/#.*$/, '');
+    return { path, anchor };
+}
+/**
+ * Check if a link is external (should not be transformed)
+ * External links are left completely unchanged by all transformations
+ */
+function isExternalLink(link) {
+    return (
+    // Common protocols
+    /^https?:\/\//.test(link) ||
+        /^mailto:/.test(link) ||
+        /^tel:/.test(link) ||
+        /^ftp:/.test(link) ||
+        /^ftps:\/\//.test(link) ||
+        // Any protocol with ://
+        link.includes('://') ||
+        // Anchor-only links (same page)
+        link.startsWith('#') ||
+        // Data URLs
+        /^data:/.test(link) ||
+        // File protocol
+        /^file:\/\//.test(link));
+}
+/**
+ * Normalize path separators and resolve relative paths
+ */
+function normalizePath(linkPath, currentFilePath, logger) {
+    logger?.debug(`[normalizePath] BEFORE: linkPath="${linkPath}", currentFilePath="${currentFilePath}"`);
+    // Handle relative paths
+    if (linkPath.startsWith('./') || linkPath.includes('../')) {
+        const currentDir = path.dirname(currentFilePath);
+        const resolved = path.posix.normalize(path.posix.join(currentDir, linkPath));
+        logger?.debug(`[normalizePath] RELATIVE PATH RESOLVED: "${linkPath}" -> "${resolved}" (currentDir: "${currentDir}")`);
+        return resolved;
+    }
+    // Remove leading './'
+    if (linkPath.startsWith('./')) {
+        return linkPath.slice(2);
+    }
+    logger?.debug(`[normalizePath] AFTER: "${linkPath}" (no changes)`);
+    return linkPath;
+}
+/**
+ * Apply link mappings to transform a URL
+ */
+function applyLinkMappings(linkUrl, linkMappings, context) {
+    const { path: linkPath, anchor } = extractAnchor(linkUrl);
+    let transformedPath = linkPath;
+    for (const mapping of linkMappings) {
+        // Check if contextFilter allows this mapping to be applied
+        if (mapping.contextFilter && context.currentFile.linkContext) {
+            if (!mapping.contextFilter(context.currentFile.linkContext)) {
+                continue; // Skip this mapping
+            }
+        }
+        // Handle relative links automatically if enabled
+        if (mapping.relativeLinks && context.currentFile.linkContext) {
+            // Check if this is a relative link (doesn't start with /, http, etc.)
+            if (!linkPath.startsWith('/') && !isExternalLink(linkPath)) {
+                // Check if the link points to a known directory structure
+                const knownPaths = ['modules/', 'classes/', 'interfaces/', 'enums/'];
+                const isKnownPath = knownPaths.some(p => linkPath.startsWith(p));
+                if (isKnownPath) {
+                    // Strip .md extension from the link path
+                    const cleanLinkPath = linkPath.replace(/\.md$/, '');
+                    // Convert relative path to absolute path using the target base
+                    const targetBase = generateSiteUrl(context.currentFile.linkContext.basePath, context.global.stripPrefixes);
+                    // Construct final URL with proper Starlight formatting
+                    let finalUrl = targetBase.replace(/\/$/, '') + '/' + cleanLinkPath;
+                    // Add trailing slash if it doesn't end with one and isn't empty
+                    if (finalUrl && !finalUrl.endsWith('/')) {
+                        finalUrl += '/';
+                    }
+                    transformedPath = finalUrl;
+                    return transformedPath + anchor;
+                }
+            }
+        }
+        let matched = false;
+        let replacement = '';
+        if (typeof mapping.pattern === 'string') {
+            // String pattern - exact match or contains
+            if (transformedPath.includes(mapping.pattern)) {
+                matched = true;
+                if (typeof mapping.replacement === 'string') {
+                    replacement = transformedPath.replace(mapping.pattern, mapping.replacement);
+                }
+                else {
+                    replacement = mapping.replacement(transformedPath, anchor, context);
+                }
+            }
+        }
+        else {
+            // RegExp pattern
+            const match = transformedPath.match(mapping.pattern);
+            if (match) {
+                matched = true;
+                if (typeof mapping.replacement === 'string') {
+                    replacement = transformedPath.replace(mapping.pattern, mapping.replacement);
+                }
+                else {
+                    replacement = mapping.replacement(transformedPath, anchor, context);
+                }
+            }
+        }
+        if (matched) {
+            // Apply the transformation and continue with next mapping
+            transformedPath = replacement;
+            // Note: We continue applying other mappings to allow chaining
+        }
+    }
+    return transformedPath + anchor;
+}
+/**
+ * Convert a target path to a site-compatible URL
+ */
+function generateSiteUrl(targetPath, stripPrefixes) {
+    let url = targetPath;
+    // Strip configured prefixes
+    for (const prefix of stripPrefixes) {
+        if (url.startsWith(prefix)) {
+            url = url.slice(prefix.length);
+            break;
+        }
+    }
+    // Remove leading slash if present
+    url = url.replace(/^\//, '');
+    // Remove file extension
+    url = url.replace(/\.(md|mdx)$/i, '');
+    // Handle index files - they should resolve to parent directory
+    if (url.endsWith('/index')) {
+        url = url.replace('/index', '');
+    }
+    else if (url === 'index') {
+        url = '';
+    }
+    // Split path into segments and slugify each
+    const segments = url.split('/').map(segment => segment ? slug(segment) : '');
+    // Reconstruct URL
+    url = segments.filter(s => s).join('/');
+    // Ensure leading slash
+    if (url && !url.startsWith('/')) {
+        url = '/' + url;
+    }
+    // Add trailing slash for non-empty paths
+    if (url && !url.endsWith('/')) {
+        url = url + '/';
+    }
+    return url || '/';
+}
+/**
+ * Transform a single markdown link
+ *
+ * Processing order:
+ * 1. Skip external links (no transformation)
+ * 2. Normalize path relative to current file
+ * 3. Apply global path mappings to normalized path
+ * 4. Check if link targets imported file in sourceToTargetMap
+ * 5. Apply non-global path mappings if unresolved
+ * 6. Check custom handlers
+ */
+function transformLink(linkText, linkUrl, context) {
+    // Skip external links FIRST - no transformations should ever be applied to them
+    if (isExternalLink(linkUrl)) {
+        return `[${linkText}](${linkUrl})`;
+    }
+    const { path: linkPath, anchor } = extractAnchor(linkUrl);
+    // Normalize the link path relative to current file FIRST
+    const normalizedPath = normalizePath(linkPath, context.currentFile.sourcePath, context.global.logger);
+    // Apply global path mappings to the normalized path
+    let processedNormalizedPath = normalizedPath;
+    if (context.global.linkMappings) {
+        const globalMappings = context.global.linkMappings.filter(m => m.global);
+        if (globalMappings.length > 0) {
+            processedNormalizedPath = applyLinkMappings(normalizedPath + anchor, globalMappings, context);
+            // Extract path again after global mappings
+            const { path: newPath } = extractAnchor(processedNormalizedPath);
+            processedNormalizedPath = newPath;
+        }
+    }
+    // Check if this links to an imported file
+    const targetPath = context.global.sourceToTargetMap.get(normalizedPath);
+    if (targetPath) {
+        // This is an internal link to an imported file
+        const siteUrl = generateSiteUrl(targetPath, context.global.stripPrefixes);
+        return `[${linkText}](${siteUrl}${anchor})`;
+    }
+    // Apply non-global path mappings to unresolved links
+    if (context.global.linkMappings) {
+        const nonGlobalMappings = context.global.linkMappings.filter(m => !m.global);
+        if (nonGlobalMappings.length > 0) {
+            const mappedUrl = applyLinkMappings(processedNormalizedPath + anchor, nonGlobalMappings, context);
+            if (mappedUrl !== (processedNormalizedPath + anchor)) {
+                return `[${linkText}](${mappedUrl})`;
+            }
+        }
+    }
+    // Check custom handlers
+    if (context.global.customHandlers) {
+        for (const handler of context.global.customHandlers) {
+            const currentUrl = processedNormalizedPath + anchor;
+            if (handler.test(currentUrl, context)) {
+                const transformedUrl = handler.transform(currentUrl, context);
+                return `[${linkText}](${transformedUrl})`;
+            }
+        }
+    }
+    // No transformation needed - return processed URL
+    return `[${linkText}](${processedNormalizedPath + anchor})`;
+}
+/**
+ * Global link transformation function
+ * Processes all imported files and resolves internal links
+ */
+export function globalLinkTransform(importedFiles, options) {
+    // Build global context
+    const sourceToTargetMap = new Map();
+    const sourceToIdMap = new Map();
+    for (const file of importedFiles) {
+        sourceToTargetMap.set(file.sourcePath, file.targetPath);
+        sourceToIdMap.set(file.sourcePath, file.id);
+    }
+    const globalContext = {
+        sourceToTargetMap,
+        sourceToIdMap,
+        stripPrefixes: options.stripPrefixes,
+        customHandlers: options.customHandlers,
+        linkMappings: options.linkMappings,
+        logger: options.logger,
+    };
+    // Transform links in all files
+    const markdownLinkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
+    return importedFiles.map(file => ({
+        ...file,
+        content: file.content.replace(markdownLinkRegex, (match, linkText, linkUrl) => {
+            const linkContext = {
+                currentFile: file,
+                originalLink: linkUrl,
+                anchor: extractAnchor(linkUrl).anchor,
+                global: globalContext,
+            };
+            return transformLink(linkText, linkUrl, linkContext);
+        }),
+    }));
+}
+/**
+ * Infer cross-section path from basePath
+ * @param basePath - The base path from include pattern (e.g., 'src/content/docs/reference/api')
+ * @returns Inferred cross-section path (e.g., '/reference/api')
+ */
+function inferCrossSectionPath(basePath) {
+    return basePath
+        .replace(/^src\/content\/docs/, '')
+        .replace(/\/$/, '') || '/';
+}
+/**
+ * Generate link mappings automatically from pathMappings in include patterns
+ * @param includes - Array of include patterns with pathMappings
+ * @param stripPrefixes - Prefixes to strip when generating URLs
+ * @returns Array of generated link mappings
+ */
+export function generateAutoLinkMappings(includes, stripPrefixes = []) {
+    const linkMappings = [];
+    for (const includePattern of includes) {
+        if (!includePattern.pathMappings)
+            continue;
+        const inferredCrossSection = inferCrossSectionPath(includePattern.basePath);
+        for (const [sourcePath, mappingValue] of Object.entries(includePattern.pathMappings)) {
+            // Handle both string and enhanced object formats
+            const targetPath = typeof mappingValue === 'string' ? mappingValue : mappingValue.target;
+            const crossSectionPath = typeof mappingValue === 'object' && mappingValue.crossSectionPath
+                ? mappingValue.crossSectionPath
+                : inferredCrossSection;
+            if (sourcePath.endsWith('/')) {
+                // Folder mapping - use regex with capture group
+                const sourcePattern = sourcePath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                linkMappings.push({
+                    pattern: new RegExp(`^${sourcePattern}(.+)$`),
+                    replacement: (transformedPath, anchor, context) => {
+                        const relativePath = transformedPath.replace(new RegExp(`^${sourcePattern}`), '');
+                        let finalPath;
+                        if (crossSectionPath && crossSectionPath !== '/') {
+                            finalPath = targetPath === ''
+                                ? `${crossSectionPath}/${relativePath}`
+                                : `${crossSectionPath}/${targetPath}${relativePath}`;
+                        }
+                        else {
+                            finalPath = targetPath === '' ? relativePath : `${targetPath}${relativePath}`;
+                        }
+                        return generateSiteUrl(finalPath, stripPrefixes);
+                    },
+                    global: true,
+                });
+            }
+            else {
+                // File mapping - exact string match
+                const sourcePattern = sourcePath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                linkMappings.push({
+                    pattern: new RegExp(`^${sourcePattern}$`),
+                    replacement: (transformedPath, anchor, context) => {
+                        const finalPath = crossSectionPath && crossSectionPath !== '/'
+                            ? `${crossSectionPath}/${targetPath}`
+                            : targetPath;
+                        return generateSiteUrl(finalPath, stripPrefixes);
+                    },
+                    global: true,
+                });
+            }
+        }
+    }
+    return linkMappings;
+}

package/dist/github.loader.d.ts

@@ -0,0 +1,14 @@
+import type { Loader, GithubLoaderOptions } from "./github.types.js";
+/**
+ * Loads data from GitHub repositories based on the provided configurations and options.
+ *
+ * Features:
+ * - Sequential processing with spinner feedback for long-running operations
+ * - Dry run mode for change detection without actual imports
+ * - Configurable logging levels per configuration
+ * - Import state tracking for incremental updates
+ * - Content store management with optional clearing
+ *
+ * @return A loader object responsible for managing the data loading process.
+ */
+export declare function githubLoader({ octokit, configs, fetchOptions, clear, dryRun, logLevel, force, }: GithubLoaderOptions): Loader;

package/dist/github.loader.js

@@ -0,0 +1,143 @@
+import { toCollectionEntry } from "./github.content.js";
+import { performSelectiveCleanup } from "./github.cleanup.js";
+import { performDryRun, displayDryRunResults, updateImportState } from "./github.dryrun.js";
+import { createLogger } from "./github.logger.js";
+/**
+ * Performs selective cleanup for configurations with basePath
+ * @param configs - Array of configuration objects
+ * @param context - Loader context
+ * @param octokit - GitHub API client
+ * @internal
+ */
+async function performSelectiveCleanups(configs, context, octokit) {
+    const results = [];
+    // Process each config sequentially to avoid overwhelming Astro's file watcher
+    for (const config of configs) {
+        if (config.enabled === false) {
+            context.logger.debug(`Skipping disabled config: ${config.name || `${config.owner}/${config.repo}`}`);
+            continue;
+        }
+        try {
+            const stats = await performSelectiveCleanup(config, context, octokit);
+            results.push(stats);
+        }
+        catch (error) {
+            context.logger.error(`Selective cleanup failed for ${config.name || `${config.owner}/${config.repo}`}: ${error}`);
+            // Continue with other configs even if one fails
+        }
+    }
+    return results;
+}
+/**
+ * Loads data from GitHub repositories based on the provided configurations and options.
+ *
+ * Features:
+ * - Sequential processing with spinner feedback for long-running operations
+ * - Dry run mode for change detection without actual imports
+ * - Configurable logging levels per configuration
+ * - Import state tracking for incremental updates
+ * - Content store management with optional clearing
+ *
+ * @return A loader object responsible for managing the data loading process.
+ */
+export function githubLoader({ octokit, configs, fetchOptions = {}, clear = false, dryRun = false, logLevel, force = false, }) {
+    return {
+        name: "github-loader",
+        load: async (context) => {
+            const { store } = context;
+            // Create global logger with specified level or default
+            const globalLogger = createLogger(logLevel || 'default');
+            if (dryRun) {
+                globalLogger.info("🔍 Dry run mode enabled - checking for changes only");
+                try {
+                    const results = await performDryRun(configs, context, octokit);
+                    displayDryRunResults(results, context.logger);
+                    globalLogger.info("\n🚫 Dry run complete - no imports performed");
+                    globalLogger.info("💡 Set dryRun: false to perform actual imports");
+                    return; // Exit without importing
+                }
+                catch (error) {
+                    globalLogger.error(`Dry run failed: ${error.message}`);
+                    throw error;
+                }
+            }
+            globalLogger.debug(`Loading data from ${configs.length} sources`);
+            // Always use standard processing - no file deletions to avoid Astro issues
+            globalLogger.info(clear ? "Processing with content store clear" : "Processing without content store clear");
+            if (clear) {
+                store.clear();
+            }
+            // Process each config sequentially to avoid overwhelming GitHub API/CDN
+            for (let i = 0; i < configs.length; i++) {
+                const config = configs[i];
+                if (config.enabled === false) {
+                    globalLogger.debug(`Skipping disabled config: ${config.name || `${config.owner}/${config.repo}`}`);
+                    continue;
+                }
+                // Add small delay between configs to be gentler on GitHub's CDN
+                if (i > 0) {
+                    await new Promise(resolve => setTimeout(resolve, 1000));
+                }
+                // Determine the effective log level for this config
+                const effectiveLogLevel = logLevel || config.logLevel || 'default';
+                const configLogger = createLogger(effectiveLogLevel);
+                const configName = config.name || `${config.owner}/${config.repo}`;
+                const repository = `${config.owner}/${config.repo}`;
+                let summary = {
+                    configName,
+                    repository,
+                    ref: config.ref,
+                    filesProcessed: 0,
+                    filesUpdated: 0,
+                    filesUnchanged: 0,
+                    duration: 0,
+                    status: 'error',
+                };
+                const startTime = Date.now();
+                try {
+                    // Perform the import with spinner
+                    const stats = await globalLogger.withSpinner(`🔄 Importing ${configName}...`, () => toCollectionEntry({
+                        context: { ...context, logger: configLogger },
+                        octokit,
+                        options: config,
+                        fetchOptions,
+                        force,
+                    }), `✅ ${configName} imported successfully`, `❌ ${configName} import failed`);
+                    summary.duration = Date.now() - startTime;
+                    summary.filesProcessed = stats?.processed || 0;
+                    summary.filesUpdated = stats?.updated || 0;
+                    summary.filesUnchanged = stats?.unchanged || 0;
+                    summary.assetsDownloaded = stats?.assetsDownloaded || 0;
+                    summary.assetsCached = stats?.assetsCached || 0;
+                    summary.status = 'success';
+                    // Log structured summary
+                    configLogger.logImportSummary(summary);
+                    // Update state tracking for future dry runs
+                    try {
+                        // Get the latest commit info to track state
+                        const { data } = await octokit.rest.repos.listCommits({
+                            owner: config.owner,
+                            repo: config.repo,
+                            sha: config.ref || 'main',
+                            per_page: 1
+                        });
+                        if (data.length > 0) {
+                            await updateImportState(process.cwd(), config, data[0].sha);
+                        }
+                    }
+                    catch (error) {
+                        // Don't fail the import if state tracking fails
+                        configLogger.debug(`Failed to update import state for ${configName}: ${error}`);
+                    }
+                }
+                catch (error) {
+                    summary.duration = Date.now() - startTime;
+                    summary.status = 'error';
+                    summary.error = error.message;
+                    configLogger.logImportSummary(summary);
+                    // Continue with other configs even if one fails
+                }
+            }
+        },
+    };
+}

package/dist/github.loader.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/github.loader.spec.js

@@ -0,0 +1,96 @@
+import { beforeEach, describe, it, expect } from "vitest";
+import { githubLoader } from "./github.loader.js";
+import { globalLinkTransform } from "./github.link-transform.js";
+import { createLogger } from "./github.logger.js";
+import { Octokit } from "octokit";
+const FIXTURES = [
+    {
+        owner: "awesome-algorand",
+        repo: "algokit-cli",
+        ref: "docs/starlight-preview",
+        path: ".devportal/starlight",
+    },
+];
+describe("githubLoader", () => {
+    let octokit;
+    beforeEach(() => {
+        octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    });
+    it("should work", async () => {
+        const result = githubLoader({ octokit, configs: FIXTURES });
+        console.log(result);
+    });
+    describe("context-aware link transformations", () => {
+        it("should handle relative links from API files with contextFilter", () => {
+            const testFiles = [
+                {
+                    id: "api-readme",
+                    sourcePath: "docs/code/README.md",
+                    targetPath: "src/content/docs/reference/algokit-utils-ts/api/README.md",
+                    content: 'Check out the [modules](modules/) for more info.',
+                    linkContext: {
+                        sourcePath: "docs/code/README.md",
+                        targetPath: "src/content/docs/reference/algokit-utils-ts/api/README.md",
+                        basePath: "src/content/docs/reference/algokit-utils-ts/api",
+                        pathMappings: { "docs/code/": "" }
+                    }
+                },
+                {
+                    id: "modules-index",
+                    sourcePath: "docs/code/modules/index.md",
+                    targetPath: "src/content/docs/reference/algokit-utils-ts/api/modules/index.md",
+                    content: 'This is the modules index.',
+                    linkContext: {
+                        sourcePath: "docs/code/modules/index.md",
+                        targetPath: "src/content/docs/reference/algokit-utils-ts/api/modules/index.md",
+                        basePath: "src/content/docs/reference/algokit-utils-ts/api",
+                        pathMappings: { "docs/code/": "" }
+                    }
+                }
+            ];
+            const result = globalLinkTransform(testFiles, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [
+                    {
+                        contextFilter: (context) => context.sourcePath.startsWith('docs/code/'),
+                        relativeLinks: true,
+                        pattern: /.*/,
+                        replacement: '',
+                        global: false
+                    }
+                ]
+            });
+            // The relative link `modules/` should be transformed to `/reference/algokit-utils-ts/api/modules/`
+            expect(result[0].content).toContain('[modules](/reference/algokit-utils-ts/api/modules/)');
+        });
+    });
+    describe("logging system", () => {
+        it("should create logger with different levels", () => {
+            const silentLogger = createLogger('silent');
+            const defaultLogger = createLogger('default');
+            const verboseLogger = createLogger('verbose');
+            const debugLogger = createLogger('debug');
+            expect(silentLogger.getLevel()).toBe('silent');
+            expect(defaultLogger.getLevel()).toBe('default');
+            expect(verboseLogger.getLevel()).toBe('verbose');
+            expect(debugLogger.getLevel()).toBe('debug');
+        });
+        it("should format import summary correctly", () => {
+            const logger = createLogger('default');
+            const summary = {
+                configName: 'Test Config',
+                repository: 'test/repo',
+                ref: 'main',
+                filesProcessed: 10,
+                filesUpdated: 5,
+                filesUnchanged: 5,
+                assetsDownloaded: 3,
+                assetsCached: 2,
+                duration: 1500,
+                status: 'success'
+            };
+            // This test mainly verifies the types work correctly
+            expect(() => logger.logImportSummary(summary)).not.toThrow();
+        });
+    });
+});