@larkiny/astro-github-loader 0.11.1 → 0.11.3

package/README.md

@@ -584,17 +584,52 @@ const REMOTE_CONTENT_WITH_ASSETS: ImportOptions[] = [
 
 ## File Management Strategy
 
-> **⚠️ Important: Do not use `clear: true`**
->
-> The `clear: true` option should not be used with the current implementation due to how Astro content collection syncing works. Mass file deletions can cause Astro to invalidate entire content collections, leading to 404 errors and build instability.
->
-> **Instead**: If you need to handle file deletions, renames, or path restructuring from the source repository:
->
-> 1. Manually delete the local import folders (e.g., `src/content/docs/imported`)
-> 2. Re-run the import process
-> 3. Fresh content will be imported with the new structure
->
-> This approach ensures your site remains stable while handling structural changes.
+The `clear` option enables selective replacement of content collection entries during import. When enabled, existing entries are atomically replaced (deleted then re-added) one at a time, preserving content collection stability.
+
+### Using the Clear Option
+
+```typescript
+// Per-config clear (recommended)
+const REMOTE_CONTENT: ImportOptions[] = [
+  {
+    name: "Docs that need clearing",
+    owner: "your-org",
+    repo: "docs-repo",
+    clear: true,  // Enable clearing for this config only
+    includes: [
+      { pattern: "docs/**/*.md", basePath: "src/content/docs/imported" }
+    ],
+  },
+  {
+    name: "Docs that don't need clearing",
+    owner: "your-org",
+    repo: "other-docs",
+    clear: false,  // Explicitly disable (or omit for default behavior)
+    includes: [
+      { pattern: "guides/**/*.md", basePath: "src/content/docs/guides" }
+    ],
+  },
+];
+
+// Or use global clear with per-config override
+await githubLoader({
+  octokit,
+  configs: REMOTE_CONTENT,
+  clear: true,  // Global default - can be overridden per-config
+}).load(context);
+```
+
+### When to Use Clear
+
+- **Use `clear: true`** when you need to ensure stale entries are removed (e.g., files renamed or deleted in the source repo)
+- **Use `clear: false`** (default) for incremental updates where you want to preserve existing entries
+
+### How It Works
+
+Unlike a bulk clear operation, the loader uses a selective delete-before-set approach:
+1. For each file being imported, if an entry already exists, it's deleted immediately before the new entry is added
+2. This atomic replacement ensures the content collection is never empty
+3. Astro's content collection system handles individual deletions gracefully
 
 ## Change Detection & Dry-Run Mode
 

package/dist/github.content.d.ts

@@ -111,7 +111,7 @@ export declare function syncEntry(context: LoaderContext, { url, editUrl }: {
  * Handles both files and directories, recursively processing directories if needed.
  * @internal
  */
-export declare function toCollectionEntry({ context, octokit, options, signal, force, }: CollectionEntryOptions): Promise<ImportStats>;
+export declare function toCollectionEntry({ context, octokit, options, signal, force, clear, }: CollectionEntryOptions): Promise<ImportStats>;
 /**
  * Get the headers needed to make a conditional request.
  * Uses the etag and last-modified values from the meta store.

package/dist/github.content.js

@@ -549,7 +549,7 @@ export async function syncEntry(context, { url, editUrl }, filePath, options, oc
  * Handles both files and directories, recursively processing directories if needed.
  * @internal
  */
-export async function toCollectionEntry({ context, octokit, options, signal, force = false, }) {
+export async function toCollectionEntry({ context, octokit, options, signal, force = false, clear = false, }) {
     const { owner, repo, ref = "main" } = options || {};
     if (typeof repo !== "string" || typeof owner !== "string")
         throw new TypeError(INVALID_STRING_ERROR);
@@ -654,7 +654,7 @@ export async function toCollectionEntry({ context, octokit, options, signal, for
     stats.processed = processedFiles.length;
     for (const file of processedFiles) {
         logger.logFileProcessing("Storing", file.sourcePath);
-        const result = await storeProcessedFile(file, context, options);
+        const result = await storeProcessedFile(file, context, clear);
         if (result) {
             stats.updated++;
         }
@@ -796,7 +796,7 @@ export async function toCollectionEntry({ context, octokit, options, signal, for
         };
     }
     // Helper function to store a processed file
-    async function storeProcessedFile(file, context, options) {
+    async function storeProcessedFile(file, context, clear) {
         const { store, generateDigest, entryTypes, logger, parseData, config } = context;
         function configForFile(filePath) {
             const ext = filePath.split(".").at(-1);
@@ -833,6 +833,13 @@ export async function toCollectionEntry({ context, octokit, options, signal, for
             data,
             filePath: fileUrl.toString(),
         });
+        // When clear mode is enabled, delete the existing entry before setting the new one.
+        // This provides atomic replacement without breaking Astro's content collection,
+        // as opposed to calling store.clear() which empties everything at once.
+        if (clear && existingEntry) {
+            logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
+            store.delete(file.id);
+        }
         // Store in content store
         if (entryType.getRenderFunction) {
             logger.verbose(`Rendering ${file.id}`);

package/dist/github.link-transform.js

@@ -35,17 +35,15 @@ function isExternalLink(link) {
  */
 function normalizePath(linkPath, currentFilePath, logger) {
     logger?.debug(`[normalizePath] BEFORE: linkPath="${linkPath}", currentFilePath="${currentFilePath}"`);
-    // Handle relative paths
-    if (linkPath.startsWith('./') || linkPath.includes('../')) {
+    // Handle relative paths (including simple relative paths without ./ prefix)
+    // A link is relative if it doesn't start with / or contain a protocol
+    const isAbsoluteOrExternal = linkPath.startsWith('/') || linkPath.includes('://') || linkPath.startsWith('#');
+    if (!isAbsoluteOrExternal) {
         const currentDir = path.dirname(currentFilePath);
         const resolved = path.posix.normalize(path.posix.join(currentDir, linkPath));
         logger?.debug(`[normalizePath] RELATIVE PATH RESOLVED: "${linkPath}" -> "${resolved}" (currentDir: "${currentDir}")`);
         return resolved;
     }
-    // Remove leading './'
-    if (linkPath.startsWith('./')) {
-        return linkPath.slice(2);
-    }
     logger?.debug(`[normalizePath] AFTER: "${linkPath}" (no changes)`);
     return linkPath;
 }
@@ -188,7 +186,11 @@ function transformLink(linkText, linkUrl, context) {
         }
     }
     // Check if this links to an imported file
-    const targetPath = context.global.sourceToTargetMap.get(normalizedPath);
+    let targetPath = context.global.sourceToTargetMap.get(normalizedPath);
+    // If not found and path ends with /, try looking for index.md
+    if (!targetPath && normalizedPath.endsWith('/')) {
+        targetPath = context.global.sourceToTargetMap.get(normalizedPath + 'index.md');
+    }
     if (targetPath) {
         // This is an internal link to an imported file
         const siteUrl = generateSiteUrl(targetPath, context.global.stripPrefixes);

package/dist/github.link-transform.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/github.link-transform.spec.js

@@ -0,0 +1,222 @@
+import { describe, it, expect } from 'vitest';
+import { globalLinkTransform } from './github.link-transform.js';
+describe('globalLinkTransform', () => {
+    describe('relative link resolution', () => {
+        it('should resolve simple relative links without ./ prefix', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/front-end-guide/index.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/overview.md',
+                    content: '[Designing a language](02-designing-a-language.md)',
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/02-designing-a-language.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/02-designing-a-language.md',
+                    content: '# Designing a language',
+                    id: 'file2',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Designing a language](/reference/api/front-end-guide/02-designing-a-language/)');
+        });
+        it('should resolve relative links with ./ prefix', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/front-end-guide/index.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/overview.md',
+                    content: '[Intro](./00-introduction.md)',
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/00-introduction.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/00-introduction.md',
+                    content: '# Introduction',
+                    id: 'file2',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Intro](/reference/api/front-end-guide/00-introduction/)');
+        });
+        it('should resolve relative links with ../ prefix', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/front-end-guide/subfolder/page.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/subfolder/page.md',
+                    content: '[Back to overview](../index.md)',
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/index.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/overview.md',
+                    content: '# Overview',
+                    id: 'file2',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Back to overview](/reference/api/front-end-guide/overview/)');
+        });
+        it('should preserve absolute links', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/page.md',
+                    targetPath: 'src/content/docs/page.md',
+                    content: '[Absolute link](/some/absolute/path)',
+                    id: 'file1',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Absolute link](/some/absolute/path)');
+        });
+        it('should preserve external links', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/page.md',
+                    targetPath: 'src/content/docs/page.md',
+                    content: '[External](https://example.com)',
+                    id: 'file1',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[External](https://example.com)');
+        });
+        it('should preserve anchor-only links', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/page.md',
+                    targetPath: 'src/content/docs/page.md',
+                    content: '[Jump to section](#my-section)',
+                    id: 'file1',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Jump to section](#my-section)');
+        });
+        it('should preserve anchors in relative links', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/front-end-guide/index.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/overview.md',
+                    content: '[Section](02-designing-a-language.md#primitive-types)',
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/02-designing-a-language.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/02-designing-a-language.md',
+                    content: '# Designing a language',
+                    id: 'file2',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe('[Section](/reference/api/front-end-guide/02-designing-a-language/#primitive-types)');
+        });
+        it('should handle multiple links in the same file', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/front-end-guide/index.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/overview.md',
+                    content: `
+[Introduction](00-introduction.md)
+[Calling puya](01-calling-puya.md)
+[Designing a language](02-designing-a-language.md)
+          `.trim(),
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/00-introduction.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/00-introduction.md',
+                    content: '# Introduction',
+                    id: 'file2',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/01-calling-puya.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/01-calling-puya.md',
+                    content: '# Calling Puya',
+                    id: 'file3',
+                },
+                {
+                    sourcePath: 'docs/front-end-guide/02-designing-a-language.md',
+                    targetPath: 'src/content/docs/reference/api/front-end-guide/02-designing-a-language.md',
+                    content: '# Designing a language',
+                    id: 'file4',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            expect(result[0].content).toBe(`
+[Introduction](/reference/api/front-end-guide/00-introduction/)
+[Calling puya](/reference/api/front-end-guide/01-calling-puya/)
+[Designing a language](/reference/api/front-end-guide/02-designing-a-language/)
+      `.trim());
+        });
+        it('should strip .md extension from unresolved relative links', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/page.md',
+                    targetPath: 'src/content/docs/page.md',
+                    content: '[Unresolved](some-file-not-in-map.md)',
+                    id: 'file1',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [],
+            });
+            // Should normalize to docs/some-file-not-in-map (current file's dir + relative link)
+            // but not resolve to full path since file not in map, so just strips .md
+            expect(result[0].content).toBe('[Unresolved](docs/some-file-not-in-map)');
+        });
+    });
+    describe('index.md handling', () => {
+        it('should convert index.md to trailing slash', () => {
+            const files = [
+                {
+                    sourcePath: 'docs/page.md',
+                    targetPath: 'src/content/docs/page.md',
+                    content: '[Link](subfolder/index.md)',
+                    id: 'file1',
+                },
+                {
+                    sourcePath: 'docs/subfolder/index.md',
+                    targetPath: 'src/content/docs/subfolder/index.md',
+                    content: '# Index',
+                    id: 'file2',
+                },
+            ];
+            const result = globalLinkTransform(files, {
+                stripPrefixes: ['src/content/docs'],
+                linkMappings: [
+                    {
+                        pattern: /\/index$/,
+                        replacement: '/',
+                        global: true,
+                    },
+                ],
+            });
+            expect(result[0].content).toBe('[Link](/subfolder/)');
+        });
+    });
+});

package/dist/github.loader.js

@@ -44,7 +44,6 @@ export function githubLoader({ octokit, configs, fetchOptions = {}, clear = fals
     return {
         name: "github-loader",
         load: async (context) => {
-            const { store } = context;
             // Create global logger with specified level or default
             const globalLogger = createLogger(logLevel || 'default');
             if (dryRun) {
@@ -62,11 +61,9 @@ export function githubLoader({ octokit, configs, fetchOptions = {}, clear = fals
                 }
             }
             globalLogger.debug(`Loading data from ${configs.length} sources`);
-            // Always use standard processing - no file deletions to avoid Astro issues
-            globalLogger.info(clear ? "Processing with content store clear" : "Processing without content store clear");
-            if (clear) {
-                store.clear();
-            }
+            // Log clear mode status - actual clearing happens per-entry in toCollectionEntry
+            // to avoid breaking Astro's content collection by emptying the store all at once
+            globalLogger.info(clear ? "Processing with selective entry replacement" : "Processing without entry replacement");
             // Process each config sequentially to avoid overwhelming GitHub API/CDN
             for (let i = 0; i < configs.length; i++) {
                 const config = configs[i];
@@ -131,6 +128,18 @@ export function githubLoader({ octokit, configs, fetchOptions = {}, clear = fals
                     else {
                         configLogger.info(`🔄 Force mode enabled for ${configName} - proceeding with full import`);
                     }
+                    // Determine effective clear setting: per-config takes precedence over global
+                    const effectiveClear = config.clear ?? clear;
+                    // Perform selective cleanup before importing if clear is enabled
+                    if (effectiveClear) {
+                        configLogger.info(`🧹 Clearing obsolete files for ${configName}...`);
+                        try {
+                            await performSelectiveCleanup(config, { ...context, logger: configLogger }, octokit);
+                        }
+                        catch (error) {
+                            configLogger.warn(`Cleanup failed for ${configName}, continuing with import: ${error}`);
+                        }
+                    }
                     // Perform the import with spinner
                     const stats = await globalLogger.withSpinner(`🔄 Importing ${configName}...`, () => toCollectionEntry({
                         context: { ...context, logger: configLogger },
@@ -138,6 +147,7 @@ export function githubLoader({ octokit, configs, fetchOptions = {}, clear = fals
                         options: config,
                         fetchOptions,
                         force,
+                        clear: effectiveClear,
                     }), `✅ ${configName} imported successfully`, `❌ ${configName} import failed`);
                     summary.duration = Date.now() - startTime;
                     summary.filesProcessed = stats?.processed || 0;

package/dist/github.types.d.ts

@@ -199,6 +199,13 @@ export type CollectionEntryOptions = {
      * @default false
      */
     force?: boolean;
+    /**
+     * When true, deletes existing store entries before setting new ones.
+     * This enables atomic replacement of entries without breaking the content collection.
+     * Passed from GithubLoaderOptions.clear
+     * @internal
+     */
+    clear?: boolean;
 };
 /**
  * Interface representing rendered content, including HTML and associated metadata.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@larkiny/astro-github-loader",
   "type": "module",
-  "version": "0.11.1",
+  "version": "0.11.3",
   "description": "Load content from GitHub repositories into Astro content collections with asset management and content transformations",
   "keywords": [
     "astro",

package/src/content/docs/algokit/cli/accounts.md

@@ -0,0 +1 @@
+# Content

package/src/content/docs/algokit/cli/generate.md

@@ -0,0 +1 @@
+# Content

package/src/content/docs/algokit/cli/overview.md

@@ -0,0 +1 @@
+# Content

package/src/content/docs/algokit/cli/tasks.md

@@ -0,0 +1 @@
+# Content

package/src/content/docs/reference/algokit-cli/index.md

@@ -0,0 +1 @@
+# Content

package/src/github.content.ts

@@ -654,6 +654,7 @@ export async function toCollectionEntry({
   options,
   signal,
   force = false,
+  clear = false,
 }: CollectionEntryOptions): Promise<ImportStats> {
   const { owner, repo, ref = "main" } = options || {};
   if (typeof repo !== "string" || typeof owner !== "string")
@@ -782,7 +783,7 @@ export async function toCollectionEntry({
   stats.processed = processedFiles.length;
   for (const file of processedFiles) {
     logger.logFileProcessing("Storing", file.sourcePath);
-    const result = await storeProcessedFile(file, context, options);
+    const result = await storeProcessedFile(file, context, clear);
     if (result) {
       stats.updated++;
     } else {
@@ -944,7 +945,7 @@ export async function toCollectionEntry({
   async function storeProcessedFile(
     file: ImportedFile,
     context: any,
-    options: ImportOptions
+    clear: boolean
   ): Promise<any> {
     const { store, generateDigest, entryTypes, logger, parseData, config } = context;
 
@@ -988,6 +989,14 @@ export async function toCollectionEntry({
       filePath: fileUrl.toString(),
     });
 
+    // When clear mode is enabled, delete the existing entry before setting the new one.
+    // This provides atomic replacement without breaking Astro's content collection,
+    // as opposed to calling store.clear() which empties everything at once.
+    if (clear && existingEntry) {
+      logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
+      store.delete(file.id);
+    }
+
     // Store in content store
     if (entryType.getRenderFunction) {
       logger.verbose(`Rendering ${file.id}`);

package/src/github.link-transform.ts

@@ -104,19 +104,17 @@ function isExternalLink(link: string): boolean {
 function normalizePath(linkPath: string, currentFilePath: string, logger?: Logger): string {
   logger?.debug(`[normalizePath] BEFORE: linkPath="${linkPath}", currentFilePath="${currentFilePath}"`);
 
-  // Handle relative paths
-  if (linkPath.startsWith('./') || linkPath.includes('../')) {
+  // Handle relative paths (including simple relative paths without ./ prefix)
+  // A link is relative if it doesn't start with / or contain a protocol
+  const isAbsoluteOrExternal = linkPath.startsWith('/') || linkPath.includes('://') || linkPath.startsWith('#');
+
+  if (!isAbsoluteOrExternal) {
     const currentDir = path.dirname(currentFilePath);
     const resolved = path.posix.normalize(path.posix.join(currentDir, linkPath));
     logger?.debug(`[normalizePath] RELATIVE PATH RESOLVED: "${linkPath}" -> "${resolved}" (currentDir: "${currentDir}")`);
     return resolved;
   }
 
-  // Remove leading './'
-  if (linkPath.startsWith('./')) {
-    return linkPath.slice(2);
-  }
-
   logger?.debug(`[normalizePath] AFTER: "${linkPath}" (no changes)`);
   return linkPath;
 }
@@ -286,7 +284,12 @@ function transformLink(linkText: string, linkUrl: string, context: LinkContext):
   }
 
   // Check if this links to an imported file
-  const targetPath = context.global.sourceToTargetMap.get(normalizedPath);
+  let targetPath = context.global.sourceToTargetMap.get(normalizedPath);
+
+  // If not found and path ends with /, try looking for index.md
+  if (!targetPath && normalizedPath.endsWith('/')) {
+    targetPath = context.global.sourceToTargetMap.get(normalizedPath + 'index.md');
+  }
 
   if (targetPath) {
     // This is an internal link to an imported file

package/src/github.loader.ts

@@ -67,7 +67,6 @@ export function githubLoader({
   return {
     name: "github-loader",
     load: async (context) => {
-      const { store } = context;
 
       // Create global logger with specified level or default
       const globalLogger = createLogger(logLevel || 'default');
@@ -91,12 +90,9 @@ export function githubLoader({
 
       globalLogger.debug(`Loading data from ${configs.length} sources`);
 
-      // Always use standard processing - no file deletions to avoid Astro issues
-      globalLogger.info(clear ? "Processing with content store clear" : "Processing without content store clear");
-
-      if (clear) {
-        store.clear();
-      }
+      // Log clear mode status - actual clearing happens per-entry in toCollectionEntry
+      // to avoid breaking Astro's content collection by emptying the store all at once
+      globalLogger.info(clear ? "Processing with selective entry replacement" : "Processing without entry replacement");
 
       // Process each config sequentially to avoid overwhelming GitHub API/CDN
       for (let i = 0; i < configs.length; i++) {
@@ -171,6 +167,19 @@ export function githubLoader({
             configLogger.info(`🔄 Force mode enabled for ${configName} - proceeding with full import`);
           }
 
+          // Determine effective clear setting: per-config takes precedence over global
+          const effectiveClear = config.clear ?? clear;
+
+          // Perform selective cleanup before importing if clear is enabled
+          if (effectiveClear) {
+            configLogger.info(`🧹 Clearing obsolete files for ${configName}...`);
+            try {
+              await performSelectiveCleanup(config, { ...context, logger: configLogger as any }, octokit);
+            } catch (error) {
+              configLogger.warn(`Cleanup failed for ${configName}, continuing with import: ${error}`);
+            }
+          }
+
           // Perform the import with spinner
           const stats = await globalLogger.withSpinner(
             `🔄 Importing ${configName}...`,
@@ -180,6 +189,7 @@ export function githubLoader({
               options: config,
               fetchOptions,
               force,
+              clear: effectiveClear,
             }),
             `✅ ${configName} imported successfully`,
             `❌ ${configName} import failed`

package/src/github.types.ts

@@ -216,6 +216,13 @@ export type CollectionEntryOptions = {
    * @default false
    */
   force?: boolean;
+  /**
+   * When true, deletes existing store entries before setting new ones.
+   * This enables atomic replacement of entries without breaking the content collection.
+   * Passed from GithubLoaderOptions.clear
+   * @internal
+   */
+  clear?: boolean;
 };
 
 /**