@larkiny/astro-github-loader 0.10.1 → 0.11.1

package/README.md

@@ -17,11 +17,11 @@ Load content from GitHub repositories into Astro content collections with flexib
 import { defineCollection } from "astro:content";
 import { docsLoader } from "@astrojs/starlight/loaders";
 import { docsSchema } from "@astrojs/starlight/schema";
-import { Octokit } from "octokit";
-import { githubLoader } from "@larkiny/astro-github-loader";
-import type {
-  ImportOptions,
-  LoaderContext,
+import {
+  githubLoader,
+  createOctokitFromEnv,
+  type ImportOptions,
+  type LoaderContext,
 } from "@larkiny/astro-github-loader";
 
 const REMOTE_CONTENT: ImportOptions[] = [
@@ -39,7 +39,8 @@ const REMOTE_CONTENT: ImportOptions[] = [
   },
 ];
 
-const octokit = new Octokit({ auth: import.meta.env.GITHUB_TOKEN });
+// Automatically uses GitHub App or Personal Access Token based on env vars
+const octokit = createOctokitFromEnv();
 
 export const collections = {
   docs: defineCollection({
@@ -63,6 +64,116 @@ export const collections = {
 };
 ```
 
+## Authentication
+
+The loader supports two authentication methods with different rate limits:
+
+| Method | Rate Limit | Best For |
+|--------|-----------|----------|
+| **GitHub App** (Recommended) | 15,000 requests/hour | Production, large imports, organizational use |
+| **Personal Access Token** | 5,000 requests/hour | Development, small imports |
+
+### Option 1: GitHub App Authentication (Recommended - 3x Rate Limit)
+
+**Step 1: Create a GitHub App**
+
+1. Go to GitHub Settings → Developer settings → GitHub Apps → [New GitHub App](https://github.com/settings/apps/new)
+2. Fill in the required fields:
+   - **GitHub App name**: `your-org-docs-loader` (or any name)
+   - **Homepage URL**: Your documentation site URL
+   - **Webhook**: Uncheck "Active" (not needed)
+3. Set **Repository permissions**:
+   - Contents: **Read-only**
+4. Click **Create GitHub App**
+
+**Step 2: Generate Private Key**
+
+1. In your GitHub App settings, scroll to "Private keys"
+2. Click **Generate a private key**
+3. Save the downloaded `.pem` file securely
+
+**Step 3: Install the App**
+
+1. In your GitHub App settings, click **Install App**
+2. Select your organization or personal account
+3. Choose **All repositories** or **Only select repositories**
+4. Note the **Installation ID** from the URL: `https://github.com/settings/installations/{installation_id}`
+
+**Step 4: Configure Environment Variables**
+
+```bash
+# .env
+GITHUB_APP_ID=123456
+GITHUB_APP_INSTALLATION_ID=12345678
+# For the private key, you have two options:
+
+# Option A: Direct PEM content (multiline)
+GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA...
+...
+-----END RSA PRIVATE KEY-----"
+
+# Option B: Base64 encoded (single line - easier for .env files)
+# Run: cat your-app.private-key.pem | base64 | tr -d '\n'
+GITHUB_APP_PRIVATE_KEY="LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0..."
+```
+
+**Step 5: Use in Your Config**
+
+```typescript
+import { createOctokitFromEnv } from "@larkiny/astro-github-loader";
+
+// Automatically uses GitHub App if env vars are set
+const octokit = createOctokitFromEnv();
+```
+
+### Option 2: Personal Access Token (PAT)
+
+**Step 1: Create a Token**
+
+1. Go to GitHub Settings → Developer settings → Personal access tokens → [Tokens (classic)](https://github.com/settings/tokens)
+2. Click **Generate new token (classic)**
+3. Select scopes:
+   - `public_repo` (for public repositories)
+   - `repo` (for private repositories)
+4. Generate and copy the token
+
+**Step 2: Configure Environment Variable**
+
+```bash
+# .env
+GITHUB_TOKEN=ghp_your_token_here
+```
+
+**Step 3: Use in Your Config**
+
+```typescript
+import { createOctokitFromEnv } from "@larkiny/astro-github-loader";
+
+// Automatically falls back to PAT if GitHub App vars aren't set
+const octokit = createOctokitFromEnv();
+```
+
+### Manual Authentication (Advanced)
+
+For more control, you can manually create the Octokit instance:
+
+```typescript
+import { createAuthenticatedOctokit } from "@larkiny/astro-github-loader";
+
+// GitHub App (explicit)
+const octokit = createAuthenticatedOctokit({
+  appId: process.env.GITHUB_APP_ID!,
+  privateKey: process.env.GITHUB_APP_PRIVATE_KEY!,
+  installationId: process.env.GITHUB_APP_INSTALLATION_ID!,
+});
+
+// Personal Access Token (explicit)
+const octokit = createAuthenticatedOctokit({
+  token: process.env.GITHUB_TOKEN!,
+});
+```
+
 ## Multi-Ref Configuration Example
 
 Track multiple git references from the same repository independently:
@@ -71,9 +182,11 @@ Track multiple git references from the same repository independently:
 import { defineCollection } from "astro:content";
 import { docsLoader } from "@astrojs/starlight/loaders";
 import { docsSchema } from "@astrojs/starlight/schema";
-import { Octokit } from "octokit";
-import { githubLoader } from "@larkiny/astro-github-loader";
-import type { ImportOptions } from "@larkiny/astro-github-loader";
+import {
+  githubLoader,
+  createOctokitFromEnv,
+  type ImportOptions,
+} from "@larkiny/astro-github-loader";
 
 const MULTI_REF_CONTENT: ImportOptions[] = [
   {
@@ -114,7 +227,7 @@ const MULTI_REF_CONTENT: ImportOptions[] = [
   },
 ];
 
-const octokit = new Octokit({ auth: import.meta.env.GITHUB_TOKEN });
+const octokit = createOctokitFromEnv();
 
 export const collections = {
   docs: defineCollection({
@@ -661,15 +774,23 @@ The loader includes several optimizations:
 ## Installation & Setup
 
 ```bash
-npm install @larkiny/astro-github-loader octokit
+npm install @larkiny/astro-github-loader
 ```
 
-Set up your GitHub token in `.env`:
+Set up your authentication in `.env`:
 
 ```bash
-GITHUB_TOKEN=your_github_token_here
+# Option 1: GitHub App (recommended - 15,000 requests/hour)
+GITHUB_APP_ID=123456
+GITHUB_APP_INSTALLATION_ID=12345678
+GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----..."
+
+# Option 2: Personal Access Token (5,000 requests/hour)
+GITHUB_TOKEN=ghp_your_token_here
 ```
 
+See the [Authentication](#authentication) section for detailed setup instructions.
+
 ## License
 
 MIT - See LICENSE file for details.

package/dist/github.auth.d.ts

@@ -0,0 +1,83 @@
+import { Octokit } from "octokit";
+/**
+ * Configuration options for GitHub App authentication
+ */
+export interface GitHubAppAuthConfig {
+    /** GitHub App ID */
+    appId: string | number;
+    /** GitHub App private key (PEM format) */
+    privateKey: string;
+    /** GitHub App installation ID */
+    installationId: string | number;
+}
+/**
+ * Configuration options for Personal Access Token authentication
+ */
+export interface GitHubPATAuthConfig {
+    /** Personal Access Token (classic or fine-grained) */
+    token: string;
+}
+/**
+ * Union type for authentication configuration
+ */
+export type GitHubAuthConfig = GitHubAppAuthConfig | GitHubPATAuthConfig;
+/**
+ * Creates an authenticated Octokit instance with support for both Personal Access Tokens
+ * and GitHub App authentication.
+ *
+ * **Rate Limits:**
+ * - Personal Access Token: 5,000 requests/hour
+ * - GitHub App: 15,000 requests/hour (3x higher)
+ *
+ * **GitHub App Setup:**
+ * 1. Create a GitHub App: https://github.com/settings/apps/new
+ * 2. Grant required permissions: Contents (read-only)
+ * 3. Install the app to your organization/repositories
+ * 4. Generate and download a private key
+ * 5. Note your App ID and Installation ID
+ *
+ * @param config - Authentication configuration (PAT or GitHub App)
+ * @returns Authenticated Octokit instance
+ *
+ * @example
+ * // Using Personal Access Token
+ * const octokit = createAuthenticatedOctokit({
+ *   token: process.env.GITHUB_TOKEN
+ * });
+ *
+ * @example
+ * // Using GitHub App (recommended for higher rate limits)
+ * const octokit = createAuthenticatedOctokit({
+ *   appId: process.env.GITHUB_APP_ID,
+ *   privateKey: process.env.GITHUB_APP_PRIVATE_KEY,
+ *   installationId: process.env.GITHUB_APP_INSTALLATION_ID
+ * });
+ */
+export declare function createAuthenticatedOctokit(config: GitHubAuthConfig): Octokit;
+/**
+ * Creates an authenticated Octokit instance from environment variables.
+ * Automatically detects whether to use GitHub App or PAT authentication based on
+ * which environment variables are present.
+ *
+ * **Priority:**
+ * 1. GitHub App (if GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, and GITHUB_APP_INSTALLATION_ID are set)
+ * 2. Personal Access Token (if GITHUB_TOKEN is set)
+ *
+ * **Environment Variables:**
+ *
+ * For GitHub App (recommended - 15,000 req/hour):
+ * - `GITHUB_APP_ID` - Your GitHub App ID
+ * - `GITHUB_APP_PRIVATE_KEY` - Private key in PEM format (can be multiline or base64 encoded)
+ * - `GITHUB_APP_INSTALLATION_ID` - Installation ID for your org/repos
+ *
+ * For Personal Access Token (5,000 req/hour):
+ * - `GITHUB_TOKEN` - Your personal access token
+ *
+ * @returns Authenticated Octokit instance
+ * @throws Error if no valid authentication credentials are found
+ *
+ * @example
+ * // In your Astro config or content.config.ts
+ * const octokit = createOctokitFromEnv();
+ */
+export declare function createOctokitFromEnv(): Octokit;

package/dist/github.auth.js

@@ -0,0 +1,119 @@
+import { Octokit } from "octokit";
+import { createAppAuth } from "@octokit/auth-app";
+/**
+ * Type guard to check if config is GitHub App authentication
+ */
+function isGitHubAppAuth(config) {
+    return 'appId' in config && 'privateKey' in config && 'installationId' in config;
+}
+/**
+ * Creates an authenticated Octokit instance with support for both Personal Access Tokens
+ * and GitHub App authentication.
+ *
+ * **Rate Limits:**
+ * - Personal Access Token: 5,000 requests/hour
+ * - GitHub App: 15,000 requests/hour (3x higher)
+ *
+ * **GitHub App Setup:**
+ * 1. Create a GitHub App: https://github.com/settings/apps/new
+ * 2. Grant required permissions: Contents (read-only)
+ * 3. Install the app to your organization/repositories
+ * 4. Generate and download a private key
+ * 5. Note your App ID and Installation ID
+ *
+ * @param config - Authentication configuration (PAT or GitHub App)
+ * @returns Authenticated Octokit instance
+ *
+ * @example
+ * // Using Personal Access Token
+ * const octokit = createAuthenticatedOctokit({
+ *   token: process.env.GITHUB_TOKEN
+ * });
+ *
+ * @example
+ * // Using GitHub App (recommended for higher rate limits)
+ * const octokit = createAuthenticatedOctokit({
+ *   appId: process.env.GITHUB_APP_ID,
+ *   privateKey: process.env.GITHUB_APP_PRIVATE_KEY,
+ *   installationId: process.env.GITHUB_APP_INSTALLATION_ID
+ * });
+ */
+export function createAuthenticatedOctokit(config) {
+    if (isGitHubAppAuth(config)) {
+        // GitHub App authentication (15,000 requests/hour)
+        return new Octokit({
+            authStrategy: createAppAuth,
+            auth: {
+                appId: config.appId,
+                privateKey: config.privateKey,
+                installationId: config.installationId,
+            },
+        });
+    }
+    else {
+        // Personal Access Token authentication (5,000 requests/hour)
+        return new Octokit({
+            auth: config.token,
+        });
+    }
+}
+/**
+ * Creates an authenticated Octokit instance from environment variables.
+ * Automatically detects whether to use GitHub App or PAT authentication based on
+ * which environment variables are present.
+ *
+ * **Priority:**
+ * 1. GitHub App (if GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, and GITHUB_APP_INSTALLATION_ID are set)
+ * 2. Personal Access Token (if GITHUB_TOKEN is set)
+ *
+ * **Environment Variables:**
+ *
+ * For GitHub App (recommended - 15,000 req/hour):
+ * - `GITHUB_APP_ID` - Your GitHub App ID
+ * - `GITHUB_APP_PRIVATE_KEY` - Private key in PEM format (can be multiline or base64 encoded)
+ * - `GITHUB_APP_INSTALLATION_ID` - Installation ID for your org/repos
+ *
+ * For Personal Access Token (5,000 req/hour):
+ * - `GITHUB_TOKEN` - Your personal access token
+ *
+ * @returns Authenticated Octokit instance
+ * @throws Error if no valid authentication credentials are found
+ *
+ * @example
+ * // In your Astro config or content.config.ts
+ * const octokit = createOctokitFromEnv();
+ */
+export function createOctokitFromEnv() {
+    // Check for GitHub App credentials (preferred)
+    const appId = process.env.GITHUB_APP_ID;
+    const privateKey = process.env.GITHUB_APP_PRIVATE_KEY;
+    const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
+    if (appId && privateKey && installationId) {
+        // Decode private key if it's base64 encoded (for easier .env storage)
+        let decodedPrivateKey = privateKey;
+        if (!privateKey.includes('BEGIN RSA PRIVATE KEY') && !privateKey.includes('BEGIN PRIVATE KEY')) {
+            try {
+                decodedPrivateKey = Buffer.from(privateKey, 'base64').toString('utf-8');
+            }
+            catch {
+                // If decoding fails, use as-is (might already be plaintext)
+            }
+        }
+        console.log('✓ Using GitHub App authentication (15,000 requests/hour)');
+        return createAuthenticatedOctokit({
+            appId,
+            privateKey: decodedPrivateKey,
+            installationId,
+        });
+    }
+    // Fallback to Personal Access Token
+    const token = process.env.GITHUB_TOKEN;
+    if (token) {
+        console.log('✓ Using Personal Access Token authentication (5,000 requests/hour)');
+        console.log('💡 Consider switching to GitHub App for 3x higher rate limits');
+        return createAuthenticatedOctokit({ token });
+    }
+    throw new Error('No GitHub authentication credentials found. Please set either:\n' +
+        '  - GITHUB_TOKEN (for PAT authentication)\n' +
+        '  - GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_APP_INSTALLATION_ID (for GitHub App authentication)');
+}

package/dist/github.content.js

@@ -555,26 +555,71 @@ export async function toCollectionEntry({ context, octokit, options, signal, for
         throw new TypeError(INVALID_STRING_ERROR);
     // Get logger from context - it should be our Logger instance (initialize early)
     const logger = context.logger;
-    // Get all unique directory prefixes from include patterns to limit scanning
-    const directoriesToScan = new Set();
-    if (options.includes && options.includes.length > 0) {
-        for (const includePattern of options.includes) {
-            // Extract directory part from pattern (before any glob wildcards)
-            const pattern = includePattern.pattern;
-            const beforeGlob = pattern.split(/[*?{]/)[0];
-            const dirPart = beforeGlob.includes('/') ? beforeGlob.substring(0, beforeGlob.lastIndexOf('/')) : '';
-            directoriesToScan.add(dirPart);
-        }
-    }
-    else {
-        // If no includes specified, scan from root
-        directoriesToScan.add('');
-    }
+    /**
+     * OPTIMIZATION: Use Git Trees API for efficient file discovery
+     *
+     * This replaces the previous recursive directory traversal approach which made
+     * N API calls (one per directory) with a single API call to fetch the entire
+     * repository tree structure.
+     *
+     * Benefits:
+     * - Reduces API calls by 50-70% for typical repositories
+     * - Single getTree() call retrieves all file paths at once
+     * - Reduces rate limit pressure significantly
+     * - Faster for large repositories with deep directory structures
+     *
+     * Previous approach:
+     *   - Called repos.getContent() recursively for each directory
+     *   - Example: 10 directories = 10 API calls
+     *
+     * New approach:
+     *   - 1 call to repos.listCommits() to get commit SHA
+     *   - 1 call to git.getTree() to get entire file tree
+     *   - Total: 2 API calls regardless of repository structure
+     */
+    logger.debug(`Using Git Trees API for efficient file discovery`);
+    // Get the commit SHA for the ref
+    const { data: commits } = await octokit.rest.repos.listCommits({
+        owner,
+        repo,
+        sha: ref,
+        per_page: 1,
+        request: { signal }
+    });
+    if (commits.length === 0) {
+        throw new Error(`No commits found for ref ${ref}`);
+    }
+    const commitSha = commits[0].sha;
+    const treeSha = commits[0].commit.tree.sha;
+    logger.debug(`Fetching repository tree for commit ${commitSha.slice(0, 7)}`);
+    // Get the entire repository tree in a single API call
+    const { data: treeData } = await octokit.rest.git.getTree({
+        owner,
+        repo,
+        tree_sha: treeSha,
+        recursive: "true",
+        request: { signal }
+    });
+    logger.debug(`Retrieved ${treeData.tree.length} items from repository tree`);
+    // Filter tree to only include files (not dirs/submodules) that match our patterns
+    const fileEntries = treeData.tree.filter((item) => {
+        if (item.type !== 'blob')
+            return false; // Only process files (blobs)
+        const includeCheck = shouldIncludeFile(item.path, options);
+        return includeCheck.included;
+    });
+    logger.info(`Found ${fileEntries.length} files matching include patterns (filtered from ${treeData.tree.length} total items)`);
     // Collect all files first (with content transforms applied)
     const allFiles = [];
-    for (const dirPath of directoriesToScan) {
-        const files = await collectFilesRecursively(dirPath);
-        allFiles.push(...files);
+    for (const treeItem of fileEntries) {
+        const filePath = treeItem.path;
+        // Construct the download URL (raw.githubusercontent.com format)
+        const downloadUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${commitSha}/${filePath}`;
+        const editUrl = treeItem.url || ''; // Git blob URL (use empty string as fallback)
+        const fileData = await collectFileData({ url: downloadUrl, editUrl }, filePath);
+        if (fileData) {
+            allFiles.push(fileData);
+        }
     }
     // Track statistics
     const stats = {
@@ -618,58 +663,6 @@ export async function toCollectionEntry({ context, octokit, options, signal, for
         }
     }
     return stats;
-    // Helper function to collect files without storing them
-    async function collectFilesRecursively(path) {
-        const collectedFiles = [];
-        // Fetch the content
-        const { data, status } = await octokit.rest.repos.getContent({
-            owner,
-            repo,
-            path,
-            ref,
-            request: { signal },
-        });
-        if (status !== 200)
-            throw new Error(INVALID_SERVICE_RESPONSE);
-        // Handle single file
-        if (!Array.isArray(data)) {
-            const filePath = data.path;
-            if (data.type === "file") {
-                const fileData = await collectFileData({ url: data.download_url, editUrl: data.url }, filePath);
-                if (fileData) {
-                    collectedFiles.push(fileData);
-                }
-            }
-            return collectedFiles;
-        }
-        // Directory listing - process files and recurse into subdirectories
-        const filteredEntries = data
-            .filter(({ type, path }) => {
-            // Always include directories for recursion
-            if (type === "dir")
-                return true;
-            // Apply filtering logic to files
-            if (type === "file") {
-                return shouldIncludeFile(path, options).included;
-            }
-            return false;
-        });
-        for (const { type, path, download_url, url } of filteredEntries) {
-            if (type === "dir") {
-                // Recurse into subdirectory
-                const subDirFiles = await collectFilesRecursively(path);
-                collectedFiles.push(...subDirFiles);
-            }
-            else if (type === "file") {
-                // Process file
-                const fileData = await collectFileData({ url: download_url, editUrl: url }, path);
-                if (fileData) {
-                    collectedFiles.push(fileData);
-                }
-            }
-        }
-        return collectedFiles;
-    }
     // Helper function to collect file data with content transforms applied
     async function collectFileData({ url, editUrl }, filePath) {
         if (url === null || typeof url !== "string") {

package/dist/github.content.spec.d.ts

@@ -0,0 +1 @@
+export {};