@larkiny/astro-github-loader 0.9.0

package/dist/github.types.d.ts

@@ -0,0 +1,315 @@
+import type { Loader as AstroLoader, LoaderContext as AstroLoaderContext } from "astro/loaders";
+import type { ContentEntryType } from "astro";
+import type { MarkdownHeading } from "@astrojs/markdown-remark";
+import { Octokit } from "octokit";
+import type { LinkHandler } from "./github.link-transform.js";
+import type { LogLevel } from "./github.logger.js";
+/**
+ * Context information for link transformations
+ */
+export interface LinkTransformContext {
+    /** Original source path in the repository */
+    sourcePath: string;
+    /** Target path where the file will be written */
+    targetPath: string;
+    /** Base path for this include pattern */
+    basePath: string;
+    /** Path mappings used for this file */
+    pathMappings?: Record<string, PathMappingValue>;
+    /** The include pattern that matched this file */
+    matchedPattern?: MatchedPattern;
+}
+/**
+ * Link mapping for transforming URLs in markdown links
+ */
+export interface LinkMapping {
+    /** Pattern to match (string or regex) */
+    pattern: string | RegExp;
+    /** Replacement string or function */
+    replacement: string | ((match: string, anchor: string, context: any) => string);
+    /** Apply to all links, not just unresolved internal links (default: false) */
+    global?: boolean;
+    /** Function to determine if this mapping should apply to the current file context */
+    contextFilter?: (context: LinkTransformContext) => boolean;
+    /** Automatically handle relative links by prefixing with target base path (default: false) */
+    relativeLinks?: boolean;
+}
+/**
+ * Configuration for import link transformation
+ */
+export interface ImportLinkTransformOptions {
+    /** Base paths to strip from final URLs (e.g., ["src/content/docs"]) */
+    stripPrefixes: string[];
+    /** Custom handlers for special link types */
+    customHandlers?: LinkHandler[];
+    /** Link mappings to transform URLs in markdown links */
+    linkMappings?: LinkMapping[];
+}
+/**
+ * Information about which include pattern matched a file
+ */
+export interface MatchedPattern {
+    /** The glob pattern that matched */
+    pattern: string;
+    /** The base path for this pattern */
+    basePath: string;
+    /** Index of the pattern in the includes array */
+    index: number;
+}
+/**
+ * Context object passed to transform functions
+ */
+export interface TransformContext {
+    /** Generated ID for the content */
+    id: string;
+    /** File path within the repository */
+    path: string;
+    /** Full configuration options */
+    options: ImportOptions;
+    /** Information about which include pattern matched (if any) */
+    matchedPattern?: MatchedPattern;
+}
+/**
+ * Function type for content transformations
+ * @param content - The markdown content to transform
+ * @param context - Context information about the file being processed
+ * @returns The transformed content
+ */
+export type TransformFunction = (content: string, context: TransformContext) => string;
+/**
+ * Enhanced path mapping configuration that supports cross-section linking
+ */
+export interface EnhancedPathMapping {
+    /** Target path where the file should be imported */
+    target: string;
+    /**
+     * Cross-section path for generating links to this content from other sections.
+     * If not specified, will be inferred from the basePath.
+     * Example: '/reference/algokit-utils-ts/api'
+     */
+    crossSectionPath?: string;
+}
+/**
+ * Path mapping value - can be a simple string or an enhanced configuration object
+ */
+export type PathMappingValue = string | EnhancedPathMapping;
+/**
+ * Configuration for a single include pattern
+ */
+export interface IncludePattern {
+    /** Glob pattern to match files (relative to repository root) */
+    pattern: string;
+    /** Local base path where matching files should be imported */
+    basePath: string;
+    /** Transforms to apply only to files matching this pattern */
+    transforms?: TransformFunction[];
+    /**
+     * Map of source paths to target paths for controlling where files are imported.
+     *
+     * Supports multiple mapping formats:
+     *
+     * **Simple string format:**
+     * - **File mapping**: `'docs/README.md': 'docs/overview.md'` - moves a specific file to a new path
+     * - **Folder mapping**: `'docs/capabilities/': 'docs/'` - moves all files from source folder to target folder
+     *
+     * **Enhanced object format with cross-section linking:**
+     * - `'docs/api/': { target: 'api/', crossSectionPath: '/reference/api' }`
+     *
+     * **Important**: Folder mappings require trailing slashes to distinguish from file mappings.
+     * - ✅ `'docs/capabilities/': 'docs/'` (folder mapping - moves all files)
+     * - ❌ `'docs/capabilities': 'docs/'` (treated as exact file match)
+     *
+     * When using enhanced format, link mappings will be automatically generated for cross-section references.
+     * If `crossSectionPath` is not specified, it will be inferred from the basePath.
+     */
+    pathMappings?: Record<string, PathMappingValue>;
+}
+export type GithubLoaderOptions = {
+    octokit: Octokit;
+    configs: Array<ImportOptions>;
+    clear?: boolean;
+    gitIgnore?: string;
+    basePath?: string;
+    fetchOptions?: FetchOptions;
+    /**
+     * When true, only checks for repository changes without importing.
+     * Returns a report of which repositories have new commits.
+     * @default false
+     */
+    dryRun?: boolean;
+    /**
+     * Global logging level for all import operations
+     * Overrides individual ImportOptions logLevel settings
+     * @default 'default'
+     */
+    logLevel?: LogLevel;
+    /**
+     * When true, forces a full import even if no repository changes are detected.
+     * When false (default), skips processing if repository hasn't changed.
+     * @default false
+     */
+    force?: boolean;
+};
+/**
+ * Represents the configuration options for a collection entry operation.
+ * @internal
+ */
+export type CollectionEntryOptions = {
+    /**
+     * Represents the context object for a loader, providing metadata
+     * and utilities for the current loading process.
+     *
+     * The LoaderContext may contain properties and methods that offer
+     * control or inspection over the loading behavior.
+     */
+    context: LoaderContext;
+    /**
+     * An instance of the Octokit library, which provides a way to interact
+     * with GitHub's REST API. This variable allows you to access and perform
+     * operations such as creating repositories, managing issues, handling
+     * pull requests, fetching user data, and more.
+     *
+     * The Octokit instance must be configured*/
+    octokit: Octokit;
+    /**
+     * Represents the configuration options for initializing or customizing the root application behavior.
+     * The option object may include various properties that control specific features or behavior of the application.
+     */
+    options: ImportOptions;
+    /**
+     * An optional AbortSignal instance that enables observing and controlling the
+     * abort state of an operation. It can be used to signal cancellation requests
+     * to an ongoing task, such as a fetch request or custom asynchronous operations.
+     *
+     * If provided, the corresponding task can listen to the `abort` event of the signal
+     * to handle early termination or cleanup logic appropriately.
+     *
+     * If the signal is already aborted at the time it is assigned or checked, the task
+     * may respond to the abort condition immediately.
+     */
+    signal?: AbortSignal;
+    /**
+     * Represents the optional configuration settings for a fetch operation.
+     * This variable allows customization of the behavior of the fetch process.
+     */
+    fetchOptions?: FetchOptions;
+    /**
+     * When true, forces a full import even if no repository changes are detected.
+     * When false (default), skips processing if repository hasn't changed.
+     * @default false
+     */
+    force?: boolean;
+};
+/**
+ * Interface representing rendered content, including HTML and associated metadata.
+ * @internal
+ */
+export interface RenderedContent {
+    /** Rendered HTML string. If present then `render(entry)` will return a component that renders this HTML. */
+    html: string;
+    metadata?: {
+        /** Any images that are present in this entry. Relative to the {@link DataEntry} filePath. */
+        imagePaths?: Array<string>;
+        /** Any headings that are present in this file. */
+        headings?: MarkdownHeading[];
+        /** Raw frontmatter, parsed parsed from the file. This may include data from remark plugins. */
+        frontmatter?: Record<string, any>;
+        /** Any other metadata that is present in this file. */
+        [key: string]: unknown;
+    };
+}
+/**
+ * Represents configuration options for importing content from GitHub repositories.
+ */
+export type ImportOptions = {
+    /**
+     * Display name for this configuration (used in logging)
+     */
+    name?: string;
+    /**
+     * Repository owner
+     */
+    owner: string;
+    /**
+     * Repository Name
+     */
+    repo: string;
+    /**
+     * A specific reference in Github
+     */
+    ref?: string;
+    /**
+     * Local directory path where downloaded assets should be stored
+     */
+    assetsPath?: string;
+    /**
+     * Base URL prefix for asset references in transformed markdown content
+     */
+    assetsBaseUrl?: string;
+    /**
+     * Array of file extensions to treat as assets (e.g., ['.png', '.jpg', '.svg'])
+     * Defaults to common image formats if not specified
+     */
+    assetPatterns?: string[];
+    /**
+     * Whether this configuration is enabled for processing
+     */
+    enabled?: boolean;
+    /**
+     * Whether to clear target directories before importing content
+     */
+    clear?: boolean;
+    /**
+     * Array of transform functions to apply to all imported content
+     */
+    transforms?: TransformFunction[];
+    /**
+     * Array of include patterns defining which files to import and where to put them
+     * If not specified, all files will be imported (backward compatibility mode)
+     */
+    includes?: IncludePattern[];
+    /**
+     * Link transformation options
+     * Applied after all content transforms and across all include patterns
+     */
+    linkTransform?: ImportLinkTransformOptions;
+    /**
+     * Logging level for this import configuration
+     * Can be overridden by global logLevel in GithubLoaderOptions
+     * @default 'default'
+     */
+    logLevel?: LogLevel;
+};
+export type FetchOptions = RequestInit & {
+    signal?: AbortSignal;
+    concurrency?: number;
+};
+/**
+ * @internal
+ */
+export interface LoaderContext extends AstroLoaderContext {
+    /** @internal */
+    entryTypes?: Map<string, ContentEntryType>;
+}
+/**
+ * @internal
+ */
+export interface Loader extends AstroLoader {
+    /** Do the actual loading of the data */
+    load: (context: LoaderContext) => Promise<void>;
+}
+/**
+ * Statistics for a sync operation
+ */
+export interface SyncStats {
+    /** Number of files added */
+    added: number;
+    /** Number of files updated */
+    updated: number;
+    /** Number of files deleted */
+    deleted: number;
+    /** Number of files unchanged */
+    unchanged: number;
+    /** Total processing time in ms */
+    duration: number;
+}

package/dist/github.types.js

@@ -0,0 +1 @@
+export {};

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from './github.constants.js';
+export * from './github.content.js';
+export * from './github.loader.js';
+export * from './github.types.js';
+export * from './github.link-transform.js';

package/dist/index.js

@@ -0,0 +1,5 @@
+export * from './github.constants.js';
+export * from './github.content.js';
+export * from './github.loader.js';
+export * from './github.types.js';
+export * from './github.link-transform.js';

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@larkiny/astro-github-loader",
+  "type": "module",
+  "version": "0.9.0",
+  "description": "Load content from GitHub repositories into Astro content collections with asset management and content transformations",
+  "keywords": [
+    "astro",
+    "github",
+    "content",
+    "starlight",
+    "loader",
+    "markdown"
+  ],
+  "author": "Michael Feher, Larkin Young",
+  "license": "MIT",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/larkiny/starlight-github-loader-fork.git",
+    "directory": "packages/astro-github-loader"
+  },
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint .",
+    "prettier": "prettier --check .",
+    "preview": "astro preview",
+    "astro": "astro"
+  },
+  "dependencies": {
+    "github-slugger": "^2.0.0",
+    "octokit": "^4.1.2",
+    "picomatch": "^4.0.2"
+  },
+  "peerDependencies": {
+    "astro": "^5.5.6"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.14.0",
+    "@types/picomatch": "^4.0.0",
+    "@typescript-eslint/parser": "^8.29.0",
+    "eslint": "^9.24.0",
+    "eslint-plugin-astro": "^1.3.1",
+    "prettier": "3.5.3",
+    "prettier-plugin-astro": "0.14.1",
+    "starlight-typedoc": "^0.21.0",
+    "typedoc": "^0.28.1",
+    "typedoc-plugin-markdown": "^4.6.1",
+    "vitest": "^3.1.1"
+  }
+}

package/src/github.cleanup.ts

@@ -0,0 +1,243 @@
+import { promises as fs } from "node:fs";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { generateId, generatePath, shouldIncludeFile } from "./github.content.js";
+import type { ImportOptions, LoaderContext, SyncStats } from "./github.types.js";
+
+const SLEEP_BETWEEN_DELETES = 10; // ms between file deletions
+
+/**
+ * Sleep utility for pacing file operations
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Gets all files that should exist locally based on remote repository state
+ */
+async function getExpectedFiles(
+  octokit: any,
+  options: ImportOptions,
+  signal?: AbortSignal
+): Promise<Set<string>> {
+  const { owner, repo, ref = "main" } = options;
+  const expectedFiles = new Set<string>();
+
+  // Get all unique directory prefixes from include patterns to limit scanning
+  const directoriesToScan = new Set<string>();
+  if (options.includes && options.includes.length > 0) {
+    for (const includePattern of options.includes) {
+      // Extract directory part from pattern (before any glob wildcards)
+      const pattern = includePattern.pattern;
+      const beforeGlob = pattern.split(/[*?{]/)[0];
+      const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
+      directoriesToScan.add(dirPart);
+    }
+  } else {
+    // If no includes specified, scan from root
+    directoriesToScan.add('');
+  }
+
+  async function processDirectory(dirPath: string) {
+    try {
+      const { data } = await octokit.rest.repos.getContent({
+        owner,
+        repo,
+        path: dirPath,
+        ref,
+        request: { signal }
+      });
+
+      if (!Array.isArray(data)) {
+        // Single file
+        if (data.type === 'file' && shouldIncludeFile(data.path, options).included) {
+          const id = generateId(data.path);
+          const includeResult = shouldIncludeFile(data.path, options);
+          const localPath = generatePath(data.path, includeResult.included ? includeResult.matchedPattern : null, options);
+          // Convert to absolute path for consistent comparison
+          const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+          expectedFiles.add(absolutePath);
+        }
+        return;
+      }
+
+      // Directory listing
+      const promises = data
+        .filter(({ type, path }) => {
+          if (type === "dir") return true;
+          if (type === "file") return shouldIncludeFile(path, options).included;
+          return false;
+        })
+        .map(async ({ type, path: itemPath }) => {
+          if (type === "dir") {
+            await processDirectory(itemPath);
+          } else if (type === "file") {
+            const id = generateId(itemPath);
+            const includeResult = shouldIncludeFile(itemPath, options);
+            const localPath = generatePath(itemPath, includeResult.included ? includeResult.matchedPattern : null, options);
+            // Convert to absolute path for consistent comparison
+            const absolutePath = localPath.startsWith('/') ? localPath : join(process.cwd(), localPath);
+            expectedFiles.add(absolutePath);
+          }
+        });
+
+      await Promise.all(promises);
+    } catch (error: any) {
+      if (signal?.aborted) throw error;
+      console.warn(`Failed to process directory ${dirPath}:`, error);
+    }
+  }
+
+  // Process only the directories that match our include patterns
+  for (const dirPath of directoriesToScan) {
+    await processDirectory(dirPath);
+  }
+  return expectedFiles;
+}
+
+/**
+ * Gets all existing local files in the basePath as absolute paths
+ */
+async function getExistingFiles(basePath: string): Promise<Set<string>> {
+  const existingFiles = new Set<string>();
+  
+  if (!existsSync(basePath)) {
+    return existingFiles;
+  }
+
+  async function walkDirectory(dirPath: string) {
+    try {
+      const entries = await fs.readdir(dirPath, { withFileTypes: true });
+      
+      for (const entry of entries) {
+        const fullPath = join(dirPath, entry.name);
+        
+        if (entry.isDirectory()) {
+          // Skip manifest files and other system directories
+          if (!entry.name.startsWith('.')) {
+            await walkDirectory(fullPath);
+          }
+        } else if (entry.isFile()) {
+          // Skip manifest and system files
+          if (!entry.name.startsWith('.')) {
+            existingFiles.add(fullPath);
+          }
+        }
+      }
+    } catch (error) {
+      console.warn(`Failed to read directory ${dirPath}:`, error);
+    }
+  }
+
+  await walkDirectory(basePath);
+  return existingFiles;
+}
+
+/**
+ * Performs selective cleanup of obsolete files
+ */
+export async function performSelectiveCleanup(
+  config: ImportOptions,
+  context: LoaderContext,
+  octokit: any,
+  signal?: AbortSignal
+): Promise<SyncStats> {
+  const startTime = Date.now();
+  const { logger } = context;
+  const configName = config.name || `${config.owner}/${config.repo}`;
+  
+  if (!config.includes || config.includes.length === 0) {
+    // No cleanup needed if no include patterns specified
+    return {
+      added: 0,
+      updated: 0,
+      deleted: 0,
+      unchanged: 0,
+      duration: Date.now() - startTime
+    };
+  }
+
+  logger.debug(`Starting selective cleanup for ${configName}`);
+
+  try {
+    // Get existing local files from all include pattern base paths
+    const allExistingFiles = new Set<string>();
+    for (const includePattern of config.includes) {
+      const existingFiles = await getExistingFiles(includePattern.basePath);
+      existingFiles.forEach(file => allExistingFiles.add(file));
+    }
+    
+    // If no existing files, skip cleanup (fresh import)
+    if (allExistingFiles.size === 0) {
+      logger.debug(`No existing files found in any base paths, skipping cleanup`);
+      return {
+        added: 0,
+        updated: 0,
+        deleted: 0,
+        unchanged: 0,
+        duration: Date.now() - startTime
+      };
+    }
+    
+    // Get expected files from remote repository
+    const expectedFiles = await getExpectedFiles(octokit, config, signal);
+    
+    // Find files to delete (exist locally but not in remote)
+    const filesToDelete: string[] = [];
+    for (const existingFile of allExistingFiles) {
+      if (!expectedFiles.has(existingFile)) {
+        filesToDelete.push(existingFile);
+      }
+    }
+    
+    // Delete obsolete files with pacing
+    let deletedCount = 0;
+    for (const filePath of filesToDelete) {
+      try {
+        if (existsSync(filePath)) {
+          await fs.unlink(filePath);
+          logger.debug(`Deleted obsolete file: ${filePath}`);
+          deletedCount++;
+          await sleep(SLEEP_BETWEEN_DELETES);
+        }
+      } catch (error) {
+        logger.warn(`Failed to delete ${filePath}: ${error}`);
+      }
+    }
+    
+    const duration = Date.now() - startTime;
+    const stats: SyncStats = {
+      added: 0, // Will be counted by main sync process
+      updated: 0, // Will be counted by main sync process  
+      deleted: deletedCount,
+      unchanged: 0, // Will be counted by main sync process
+      duration
+    };
+
+    if (deletedCount > 0) {
+      logger.info(`Cleanup completed for ${configName}: ${deletedCount} obsolete files deleted (${duration}ms)`);
+    } else {
+      logger.debug(`No cleanup needed for ${configName} (${duration}ms)`);
+    }
+
+    return stats;
+    
+  } catch (error: any) {
+    if (signal?.aborted) {
+      logger.info(`Cleanup cancelled for ${configName}`);
+      throw error;
+    }
+    
+    const duration = Date.now() - startTime;
+    logger.error(`Cleanup failed for ${configName} after ${duration}ms: ${error}`);
+    // Don't throw - let the main sync process continue
+    return {
+      added: 0,
+      updated: 0, 
+      deleted: 0,
+      unchanged: 0,
+      duration
+    };
+  }
+}

package/src/github.constants.ts

@@ -0,0 +1,25 @@
+/**
+ * This variable represents an error message indicating that the provided string is invalid.
+ * It is typically used to flag inputs or data that do not meet the required string format or criteria.
+ * The value is a constant string: 'Invalid string'.
+ *
+ * @internal
+ */
+export const INVALID_STRING_ERROR = "Invalid string";
+/**
+ * Represents an error message indicating that a provided URL is invalid.
+ * This constant is typically used for validation or error handling when a URL
+ * does not conform to the expected format or requirements.
+ *
+ * @internal
+ */
+export const INVALID_URL_ERROR = "Invalid url";
+
+/**
+ * A constant that holds a default error message indicating that a service response is invalid.
+ * This value is typically used to signify that the response from a service or API call
+ * does not meet the expected format, structure, or criteria.
+ *
+ * @internal
+ */
+export const INVALID_SERVICE_RESPONSE = "Invalid service response";