@larkiny/astro-github-loader 0.11.3 β 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -55
- package/dist/github.assets.d.ts +70 -0
- package/dist/github.assets.js +253 -0
- package/dist/github.auth.js +13 -9
- package/dist/github.cleanup.d.ts +3 -2
- package/dist/github.cleanup.js +30 -23
- package/dist/github.constants.d.ts +0 -16
- package/dist/github.constants.js +0 -16
- package/dist/github.content.d.ts +5 -131
- package/dist/github.content.js +152 -794
- package/dist/github.dryrun.d.ts +9 -5
- package/dist/github.dryrun.js +49 -25
- package/dist/github.link-transform.d.ts +2 -2
- package/dist/github.link-transform.js +68 -57
- package/dist/github.loader.js +30 -46
- package/dist/github.logger.d.ts +2 -2
- package/dist/github.logger.js +33 -24
- package/dist/github.paths.d.ts +76 -0
- package/dist/github.paths.js +190 -0
- package/dist/github.storage.d.ts +16 -0
- package/dist/github.storage.js +115 -0
- package/dist/github.types.d.ts +40 -4
- package/dist/index.d.ts +8 -6
- package/dist/index.js +3 -6
- package/dist/test-helpers.d.ts +130 -0
- package/dist/test-helpers.js +194 -0
- package/package.json +3 -1
- package/src/github.assets.spec.ts +717 -0
- package/src/github.assets.ts +365 -0
- package/src/github.auth.spec.ts +245 -0
- package/src/github.auth.ts +24 -10
- package/src/github.cleanup.spec.ts +380 -0
- package/src/github.cleanup.ts +91 -47
- package/src/github.constants.ts +0 -17
- package/src/github.content.spec.ts +305 -454
- package/src/github.content.ts +259 -957
- package/src/github.dryrun.spec.ts +598 -0
- package/src/github.dryrun.ts +108 -54
- package/src/github.link-transform.spec.ts +1345 -0
- package/src/github.link-transform.ts +177 -95
- package/src/github.loader.spec.ts +75 -50
- package/src/github.loader.ts +101 -76
- package/src/github.logger.spec.ts +795 -0
- package/src/github.logger.ts +77 -35
- package/src/github.paths.spec.ts +523 -0
- package/src/github.paths.ts +259 -0
- package/src/github.storage.spec.ts +377 -0
- package/src/github.storage.ts +135 -0
- package/src/github.types.ts +54 -9
- package/src/index.ts +43 -6
- package/src/test-helpers.ts +215 -0
package/README.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Astro GitHub Loader
|
|
2
2
|
|
|
3
|
+
[](https://github.com/larkiny/starlight-github-loader/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/@larkiny/astro-github-loader)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://astro.build)
|
|
7
|
+
[](https://starlight.astro.build)
|
|
8
|
+
[](https://www.typescriptlang.org/)
|
|
9
|
+
|
|
3
10
|
Load content from GitHub repositories into Astro content collections with flexible pattern-based import, asset management, content transformations, and intelligent change detection.
|
|
4
11
|
|
|
5
12
|
## Features
|
|
@@ -9,7 +16,7 @@ Load content from GitHub repositories into Astro content collections with flexib
|
|
|
9
16
|
- π οΈ **Content Transforms** - Apply custom transformations to content during import, with pattern-specific transforms
|
|
10
17
|
- β‘ **Intelligent Change Detection** - Ref-aware commit tracking that only triggers re-imports when your target branch/tag actually changes
|
|
11
18
|
- π **Stable Imports** - Non-destructive approach that preserves local content collections
|
|
12
|
-
- π **Optimized Performance** -
|
|
19
|
+
- π **Optimized Performance** - Git Trees API for efficient file discovery with minimal API calls
|
|
13
20
|
|
|
14
21
|
## Quick Start
|
|
15
22
|
|
|
@@ -68,10 +75,10 @@ export const collections = {
|
|
|
68
75
|
|
|
69
76
|
The loader supports two authentication methods with different rate limits:
|
|
70
77
|
|
|
71
|
-
| Method
|
|
72
|
-
|
|
78
|
+
| Method | Rate Limit | Best For |
|
|
79
|
+
| ---------------------------- | -------------------- | --------------------------------------------- |
|
|
73
80
|
| **GitHub App** (Recommended) | 15,000 requests/hour | Production, large imports, organizational use |
|
|
74
|
-
| **Personal Access Token**
|
|
81
|
+
| **Personal Access Token** | 5,000 requests/hour | Development, small imports |
|
|
75
82
|
|
|
76
83
|
### Option 1: GitHub App Authentication (Recommended - 3x Rate Limit)
|
|
77
84
|
|
|
@@ -252,6 +259,7 @@ export const collections = {
|
|
|
252
259
|
```
|
|
253
260
|
|
|
254
261
|
In this example:
|
|
262
|
+
|
|
255
263
|
- **Stable docs** (v2.0.0 tag): Never re-imports, provides stable reference
|
|
256
264
|
- **Latest docs** (main branch): Only re-imports when main branch changes
|
|
257
265
|
- **Beta features** (beta branch): Only re-imports when beta branch changes
|
|
@@ -500,7 +508,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
500
508
|
return `/reference/algokit-cli`;
|
|
501
509
|
},
|
|
502
510
|
global: true,
|
|
503
|
-
description: "Map CLI reference links to reference section",
|
|
504
511
|
},
|
|
505
512
|
|
|
506
513
|
// Transform README links to introduction
|
|
@@ -510,7 +517,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
510
517
|
return `/introduction`;
|
|
511
518
|
},
|
|
512
519
|
global: true,
|
|
513
|
-
description: "Map README links to introduction page",
|
|
514
520
|
},
|
|
515
521
|
],
|
|
516
522
|
},
|
|
@@ -518,39 +524,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
518
524
|
];
|
|
519
525
|
```
|
|
520
526
|
|
|
521
|
-
## Link Transformation Utilities
|
|
522
|
-
|
|
523
|
-
Handle markdown links with anchor fragments using built-in utilities:
|
|
524
|
-
|
|
525
|
-
```typescript
|
|
526
|
-
import {
|
|
527
|
-
createLinkTransform,
|
|
528
|
-
extractAnchor,
|
|
529
|
-
removeMarkdownExtension,
|
|
530
|
-
} from "@larkiny/astro-github-loader";
|
|
531
|
-
|
|
532
|
-
const linkTransform = createLinkTransform({
|
|
533
|
-
baseUrl: "/docs/imported",
|
|
534
|
-
pathTransform: (path, context) => {
|
|
535
|
-
const { path: cleanPath, anchor } = extractAnchor(path);
|
|
536
|
-
|
|
537
|
-
// Custom link handling logic
|
|
538
|
-
if (cleanPath === "README.md") {
|
|
539
|
-
return `/docs/imported/overview${anchor}`;
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
// Use utility to remove .md extension and preserve anchors
|
|
543
|
-
return `/docs/imported/${removeMarkdownExtension(path)}`;
|
|
544
|
-
},
|
|
545
|
-
});
|
|
546
|
-
```
|
|
547
|
-
|
|
548
|
-
### Link Transform Utilities
|
|
549
|
-
|
|
550
|
-
- **`extractAnchor(path)`** - Returns `{path, anchor}` separating the anchor fragment
|
|
551
|
-
- **`removeMarkdownExtension(path)`** - Removes `.md`/`.mdx` extensions while preserving anchors
|
|
552
|
-
- **`createLinkTransform(options)`** - Main transform with custom path handling
|
|
553
|
-
|
|
554
527
|
## Asset Import and Management
|
|
555
528
|
|
|
556
529
|
Automatically detect, download, and transform asset references:
|
|
@@ -595,18 +568,18 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
595
568
|
name: "Docs that need clearing",
|
|
596
569
|
owner: "your-org",
|
|
597
570
|
repo: "docs-repo",
|
|
598
|
-
clear: true,
|
|
571
|
+
clear: true, // Enable clearing for this config only
|
|
599
572
|
includes: [
|
|
600
|
-
{ pattern: "docs/**/*.md", basePath: "src/content/docs/imported" }
|
|
573
|
+
{ pattern: "docs/**/*.md", basePath: "src/content/docs/imported" },
|
|
601
574
|
],
|
|
602
575
|
},
|
|
603
576
|
{
|
|
604
577
|
name: "Docs that don't need clearing",
|
|
605
578
|
owner: "your-org",
|
|
606
579
|
repo: "other-docs",
|
|
607
|
-
clear: false,
|
|
580
|
+
clear: false, // Explicitly disable (or omit for default behavior)
|
|
608
581
|
includes: [
|
|
609
|
-
{ pattern: "guides/**/*.md", basePath: "src/content/docs/guides" }
|
|
582
|
+
{ pattern: "guides/**/*.md", basePath: "src/content/docs/guides" },
|
|
610
583
|
],
|
|
611
584
|
},
|
|
612
585
|
];
|
|
@@ -615,7 +588,7 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
615
588
|
await githubLoader({
|
|
616
589
|
octokit,
|
|
617
590
|
configs: REMOTE_CONTENT,
|
|
618
|
-
clear: true,
|
|
591
|
+
clear: true, // Global default - can be overridden per-config
|
|
619
592
|
}).load(context);
|
|
620
593
|
```
|
|
621
594
|
|
|
@@ -627,6 +600,7 @@ await githubLoader({
|
|
|
627
600
|
### How It Works
|
|
628
601
|
|
|
629
602
|
Unlike a bulk clear operation, the loader uses a selective delete-before-set approach:
|
|
603
|
+
|
|
630
604
|
1. For each file being imported, if an entry already exists, it's deleted immediately before the new entry is added
|
|
631
605
|
2. This atomic replacement ensures the content collection is never empty
|
|
632
606
|
3. Astro's content collection system handles individual deletions gracefully
|
|
@@ -682,6 +656,7 @@ The loader uses intelligent, ref-aware change detection:
|
|
|
682
656
|
- **Efficient checking**: Only the latest commit of your target ref is checked
|
|
683
657
|
|
|
684
658
|
**Examples**:
|
|
659
|
+
|
|
685
660
|
- Config tracking `main` branch β only `main` commits trigger re-import
|
|
686
661
|
- Config tracking `v2.1.0` tag β never re-imports (tags are immutable)
|
|
687
662
|
- Config tracking `feature-branch` β ignores commits to `main`, `develop`, etc.
|
|
@@ -743,13 +718,16 @@ interface LinkMapping {
|
|
|
743
718
|
/** Replacement string or function */
|
|
744
719
|
replacement:
|
|
745
720
|
| string
|
|
746
|
-
| ((match: string, anchor: string, context:
|
|
721
|
+
| ((match: string, anchor: string, context: LinkTransformContext) => string);
|
|
747
722
|
|
|
748
723
|
/** Apply to all links, not just unresolved internal links (default: false) */
|
|
749
724
|
global?: boolean;
|
|
750
725
|
|
|
751
|
-
/**
|
|
752
|
-
|
|
726
|
+
/** Function to determine if this mapping should apply to the current file context */
|
|
727
|
+
contextFilter?: (context: LinkTransformContext) => boolean;
|
|
728
|
+
|
|
729
|
+
/** Automatically handle relative links by prefixing with target base path (default: false) */
|
|
730
|
+
relativeLinks?: boolean;
|
|
753
731
|
}
|
|
754
732
|
|
|
755
733
|
interface IncludePattern {
|
|
@@ -765,15 +743,17 @@ interface IncludePattern {
|
|
|
765
743
|
/**
|
|
766
744
|
* Map of source paths to target paths for controlling where files are imported.
|
|
767
745
|
*
|
|
768
|
-
* Supports two
|
|
769
|
-
* - **
|
|
770
|
-
* - **
|
|
746
|
+
* Supports two mapping formats:
|
|
747
|
+
* - **Simple string**: `'docs/README.md': 'docs/overview.md'`
|
|
748
|
+
* - **Enhanced object**: `'docs/api/': { target: 'api/', crossSectionPath: '/reference/api' }`
|
|
749
|
+
*
|
|
750
|
+
* And two mapping scopes:
|
|
751
|
+
* - **File mapping**: Exact file path match (e.g., `'docs/README.md': 'docs/overview.md'`)
|
|
752
|
+
* - **Folder mapping**: Trailing slash moves all files (e.g., `'docs/capabilities/': 'docs/'`)
|
|
771
753
|
*
|
|
772
754
|
* **Important**: Folder mappings require trailing slashes to distinguish from file mappings.
|
|
773
|
-
* - β
`'docs/capabilities/': 'docs/'` (folder mapping - moves all files)
|
|
774
|
-
* - β `'docs/capabilities': 'docs/'` (treated as exact file match)
|
|
775
755
|
*/
|
|
776
|
-
pathMappings?: Record<string,
|
|
756
|
+
pathMappings?: Record<string, PathMappingValue>;
|
|
777
757
|
}
|
|
778
758
|
```
|
|
779
759
|
|
|
@@ -801,8 +781,8 @@ type TransformFunction = (content: string, context: TransformContext) => string;
|
|
|
801
781
|
|
|
802
782
|
The loader includes several optimizations:
|
|
803
783
|
|
|
804
|
-
- **
|
|
805
|
-
- **Efficient API usage**: Minimizes GitHub API calls
|
|
784
|
+
- **Git Trees API**: Retrieves the entire repository file tree in a single API call (2 total: 1 for commit SHA + 1 for tree), replacing recursive directory traversal
|
|
785
|
+
- **Efficient API usage**: Minimizes GitHub API calls regardless of repository size or depth
|
|
806
786
|
- **Ref-aware change detection**: Tracks commit SHA for specific git references (branches/tags) to avoid unnecessary downloads when unrelated branches change
|
|
807
787
|
- **Concurrent processing**: Downloads and processes files in parallel
|
|
808
788
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { Octokit } from "octokit";
|
|
2
|
+
import type { Logger } from "./github.logger.js";
|
|
3
|
+
import type { ImportOptions } from "./github.types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Resolves the effective asset configuration for an import.
|
|
6
|
+
*
|
|
7
|
+
* If `assetsPath` and `assetsBaseUrl` are explicitly provided, uses them (existing behavior).
|
|
8
|
+
* If omitted, derives co-located defaults from the matched include pattern's basePath:
|
|
9
|
+
* - assetsPath: `{basePath}/assets/` (physical directory on disk)
|
|
10
|
+
* - assetsBaseUrl: `./assets` (relative reference in markdown)
|
|
11
|
+
*
|
|
12
|
+
* @param options - Import options that may or may not have explicit asset config
|
|
13
|
+
* @param filePath - The file being processed (used to find the matched include pattern)
|
|
14
|
+
* @returns Resolved assetsPath and assetsBaseUrl, or null if assets should not be processed
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export declare function resolveAssetConfig(options: ImportOptions, filePath: string): {
|
|
18
|
+
assetsPath: string;
|
|
19
|
+
assetsBaseUrl: string;
|
|
20
|
+
} | null;
|
|
21
|
+
/**
|
|
22
|
+
* Detects asset references in markdown content using regex patterns
|
|
23
|
+
* @param content - The markdown content to parse
|
|
24
|
+
* @param assetPatterns - File extensions to treat as assets
|
|
25
|
+
* @returns Array of detected asset paths
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export declare function detectAssets(content: string, assetPatterns?: string[]): string[];
|
|
29
|
+
/**
|
|
30
|
+
* Downloads an asset from GitHub and saves it locally
|
|
31
|
+
* @param octokit - GitHub API client
|
|
32
|
+
* @param owner - Repository owner
|
|
33
|
+
* @param repo - Repository name
|
|
34
|
+
* @param ref - Git reference
|
|
35
|
+
* @param assetPath - Path to the asset in the repository
|
|
36
|
+
* @param localPath - Local path where the asset should be saved
|
|
37
|
+
* @param signal - Abort signal for cancellation
|
|
38
|
+
* @returns Promise that resolves when the asset is downloaded
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export declare function downloadAsset(octokit: Octokit, owner: string, repo: string, ref: string, assetPath: string, localPath: string, signal?: AbortSignal): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Transforms asset references in markdown content to use local paths
|
|
44
|
+
* @param content - The markdown content to transform
|
|
45
|
+
* @param assetMap - Map of original asset paths to new local paths
|
|
46
|
+
* @returns Transformed content with updated asset references
|
|
47
|
+
* @internal
|
|
48
|
+
*/
|
|
49
|
+
export declare function transformAssetReferences(content: string, assetMap: Map<string, string>): string;
|
|
50
|
+
/**
|
|
51
|
+
* Resolves an asset path relative to a base path
|
|
52
|
+
* @internal
|
|
53
|
+
*/
|
|
54
|
+
export declare function resolveAssetPath(basePath: string, assetPath: string): string;
|
|
55
|
+
/**
|
|
56
|
+
* Processes assets in markdown content by detecting, downloading, and transforming references
|
|
57
|
+
* @param content - The markdown content to process
|
|
58
|
+
* @param filePath - The file path of the markdown file being processed
|
|
59
|
+
* @param options - Configuration options including asset settings
|
|
60
|
+
* @param octokit - GitHub API client
|
|
61
|
+
* @param logger - Logger instance for output
|
|
62
|
+
* @param signal - Abort signal for cancellation
|
|
63
|
+
* @returns Promise that resolves to transformed content
|
|
64
|
+
* @internal
|
|
65
|
+
*/
|
|
66
|
+
export declare function processAssets(content: string, filePath: string, options: ImportOptions, octokit: Octokit, logger: Logger, signal?: AbortSignal): Promise<{
|
|
67
|
+
content: string;
|
|
68
|
+
assetsDownloaded: number;
|
|
69
|
+
assetsCached: number;
|
|
70
|
+
}>;
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { existsSync, promises as fs } from "node:fs";
|
|
2
|
+
import { join, dirname, basename, extname } from "node:path";
|
|
3
|
+
import { shouldIncludeFile } from "./github.paths.js";
|
|
4
|
+
/**
|
|
5
|
+
* Default asset patterns for common image and media file types
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
const DEFAULT_ASSET_PATTERNS = [
|
|
9
|
+
".png",
|
|
10
|
+
".jpg",
|
|
11
|
+
".jpeg",
|
|
12
|
+
".gif",
|
|
13
|
+
".svg",
|
|
14
|
+
".webp",
|
|
15
|
+
".ico",
|
|
16
|
+
".bmp",
|
|
17
|
+
];
|
|
18
|
+
/**
|
|
19
|
+
* Resolves the effective asset configuration for an import.
|
|
20
|
+
*
|
|
21
|
+
* If `assetsPath` and `assetsBaseUrl` are explicitly provided, uses them (existing behavior).
|
|
22
|
+
* If omitted, derives co-located defaults from the matched include pattern's basePath:
|
|
23
|
+
* - assetsPath: `{basePath}/assets/` (physical directory on disk)
|
|
24
|
+
* - assetsBaseUrl: `./assets` (relative reference in markdown)
|
|
25
|
+
*
|
|
26
|
+
* @param options - Import options that may or may not have explicit asset config
|
|
27
|
+
* @param filePath - The file being processed (used to find the matched include pattern)
|
|
28
|
+
* @returns Resolved assetsPath and assetsBaseUrl, or null if assets should not be processed
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
31
|
+
export function resolveAssetConfig(options, filePath) {
|
|
32
|
+
// Explicit config takes precedence
|
|
33
|
+
if (options.assetsPath && options.assetsBaseUrl) {
|
|
34
|
+
return {
|
|
35
|
+
assetsPath: options.assetsPath,
|
|
36
|
+
assetsBaseUrl: options.assetsBaseUrl,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
// If only one is set, that's a misconfiguration β skip
|
|
40
|
+
if (options.assetsPath || options.assetsBaseUrl) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
// Derive co-located defaults from the matched include pattern's basePath
|
|
44
|
+
const includeResult = shouldIncludeFile(filePath, options);
|
|
45
|
+
if (includeResult.included && includeResult.matchedPattern) {
|
|
46
|
+
const basePath = includeResult.matchedPattern.basePath;
|
|
47
|
+
return {
|
|
48
|
+
assetsPath: join(basePath, "assets"),
|
|
49
|
+
assetsBaseUrl: "./assets",
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Detects asset references in markdown content using regex patterns
|
|
56
|
+
* @param content - The markdown content to parse
|
|
57
|
+
* @param assetPatterns - File extensions to treat as assets
|
|
58
|
+
* @returns Array of detected asset paths
|
|
59
|
+
* @internal
|
|
60
|
+
*/
|
|
61
|
+
export function detectAssets(content, assetPatterns = DEFAULT_ASSET_PATTERNS) {
|
|
62
|
+
const assets = [];
|
|
63
|
+
const patterns = assetPatterns.map((ext) => ext.toLowerCase());
|
|
64
|
+
// Match markdown images: 
|
|
65
|
+
const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
|
|
66
|
+
let match;
|
|
67
|
+
while ((match = imageRegex.exec(content)) !== null) {
|
|
68
|
+
const assetPath = match[1];
|
|
69
|
+
// Only include relative paths and assets matching our patterns
|
|
70
|
+
if (assetPath.startsWith("./") ||
|
|
71
|
+
assetPath.startsWith("../") ||
|
|
72
|
+
!assetPath.includes("://")) {
|
|
73
|
+
const ext = extname(assetPath).toLowerCase();
|
|
74
|
+
if (patterns.includes(ext)) {
|
|
75
|
+
assets.push(assetPath);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// Match HTML img tags: <img src="path">
|
|
80
|
+
const htmlImgRegex = /<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi;
|
|
81
|
+
while ((match = htmlImgRegex.exec(content)) !== null) {
|
|
82
|
+
const assetPath = match[1];
|
|
83
|
+
if (assetPath.startsWith("./") ||
|
|
84
|
+
assetPath.startsWith("../") ||
|
|
85
|
+
!assetPath.includes("://")) {
|
|
86
|
+
const ext = extname(assetPath).toLowerCase();
|
|
87
|
+
if (patterns.includes(ext)) {
|
|
88
|
+
assets.push(assetPath);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return [...new Set(assets)]; // Remove duplicates
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Downloads an asset from GitHub and saves it locally
|
|
96
|
+
* @param octokit - GitHub API client
|
|
97
|
+
* @param owner - Repository owner
|
|
98
|
+
* @param repo - Repository name
|
|
99
|
+
* @param ref - Git reference
|
|
100
|
+
* @param assetPath - Path to the asset in the repository
|
|
101
|
+
* @param localPath - Local path where the asset should be saved
|
|
102
|
+
* @param signal - Abort signal for cancellation
|
|
103
|
+
* @returns Promise that resolves when the asset is downloaded
|
|
104
|
+
* @internal
|
|
105
|
+
*/
|
|
106
|
+
export async function downloadAsset(octokit, owner, repo, ref, assetPath, localPath, signal) {
|
|
107
|
+
try {
|
|
108
|
+
const { data } = await octokit.rest.repos.getContent({
|
|
109
|
+
owner,
|
|
110
|
+
repo,
|
|
111
|
+
path: assetPath,
|
|
112
|
+
ref,
|
|
113
|
+
request: { signal },
|
|
114
|
+
});
|
|
115
|
+
if (Array.isArray(data)) {
|
|
116
|
+
throw new Error(`Asset ${assetPath} is a directory, not a file`);
|
|
117
|
+
}
|
|
118
|
+
if (data.type !== "file" || !data.download_url) {
|
|
119
|
+
throw new Error(`Asset ${assetPath} is not a valid file (type: ${data.type}, downloadUrl: ${data.download_url})`);
|
|
120
|
+
}
|
|
121
|
+
const response = await fetch(data.download_url, { signal });
|
|
122
|
+
if (!response.ok) {
|
|
123
|
+
throw new Error(`Failed to download asset: ${response.status} ${response.statusText}`);
|
|
124
|
+
}
|
|
125
|
+
const buffer = await response.arrayBuffer();
|
|
126
|
+
const dir = dirname(localPath);
|
|
127
|
+
if (!existsSync(dir)) {
|
|
128
|
+
await fs.mkdir(dir, { recursive: true });
|
|
129
|
+
}
|
|
130
|
+
await fs.writeFile(localPath, new Uint8Array(buffer));
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
if (typeof error === "object" &&
|
|
134
|
+
error !== null &&
|
|
135
|
+
"status" in error &&
|
|
136
|
+
error.status === 404) {
|
|
137
|
+
throw new Error(`Asset not found: ${assetPath}`);
|
|
138
|
+
}
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Transforms asset references in markdown content to use local paths
|
|
144
|
+
* @param content - The markdown content to transform
|
|
145
|
+
* @param assetMap - Map of original asset paths to new local paths
|
|
146
|
+
* @returns Transformed content with updated asset references
|
|
147
|
+
* @internal
|
|
148
|
+
*/
|
|
149
|
+
export function transformAssetReferences(content, assetMap) {
|
|
150
|
+
let transformedContent = content;
|
|
151
|
+
for (const [originalPath, newPath] of assetMap) {
|
|
152
|
+
// Transform markdown images
|
|
153
|
+
const imageRegex = new RegExp(`(!)\\[([^\\]]*)\\]\\(\\s*${escapeRegExp(originalPath)}\\s*\\)`, "g");
|
|
154
|
+
transformedContent = transformedContent.replace(imageRegex, `$1[$2](${newPath})`);
|
|
155
|
+
// Transform HTML img tags
|
|
156
|
+
const htmlRegex = new RegExp(`(<img[^>]+src\\s*=\\s*["'])${escapeRegExp(originalPath)}(["'][^>]*>)`, "gi");
|
|
157
|
+
transformedContent = transformedContent.replace(htmlRegex, `$1${newPath}$2`);
|
|
158
|
+
}
|
|
159
|
+
return transformedContent;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Escapes special regex characters in a string
|
|
163
|
+
* @internal
|
|
164
|
+
*/
|
|
165
|
+
function escapeRegExp(string) {
|
|
166
|
+
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Resolves an asset path relative to a base path
|
|
170
|
+
* @internal
|
|
171
|
+
*/
|
|
172
|
+
export function resolveAssetPath(basePath, assetPath) {
|
|
173
|
+
if (assetPath.startsWith("./")) {
|
|
174
|
+
return join(dirname(basePath), assetPath.slice(2));
|
|
175
|
+
}
|
|
176
|
+
else if (assetPath.startsWith("../")) {
|
|
177
|
+
return join(dirname(basePath), assetPath);
|
|
178
|
+
}
|
|
179
|
+
return assetPath;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Processes assets in markdown content by detecting, downloading, and transforming references
|
|
183
|
+
* @param content - The markdown content to process
|
|
184
|
+
* @param filePath - The file path of the markdown file being processed
|
|
185
|
+
* @param options - Configuration options including asset settings
|
|
186
|
+
* @param octokit - GitHub API client
|
|
187
|
+
* @param logger - Logger instance for output
|
|
188
|
+
* @param signal - Abort signal for cancellation
|
|
189
|
+
* @returns Promise that resolves to transformed content
|
|
190
|
+
* @internal
|
|
191
|
+
*/
|
|
192
|
+
export async function processAssets(content, filePath, options, octokit, logger, signal) {
|
|
193
|
+
const { owner, repo, ref = "main", assetsPath, assetsBaseUrl, assetPatterns, } = options;
|
|
194
|
+
logger.verbose(`πΌοΈ Processing assets for ${filePath}`);
|
|
195
|
+
logger.debug(` assetsPath: ${assetsPath}`);
|
|
196
|
+
logger.debug(` assetsBaseUrl: ${assetsBaseUrl}`);
|
|
197
|
+
if (!assetsPath || !assetsBaseUrl) {
|
|
198
|
+
logger.verbose(` βοΈ Skipping asset processing - missing assetsPath or assetsBaseUrl`);
|
|
199
|
+
return { content, assetsDownloaded: 0, assetsCached: 0 };
|
|
200
|
+
}
|
|
201
|
+
// Detect assets in the content
|
|
202
|
+
const detectedAssets = detectAssets(content, assetPatterns);
|
|
203
|
+
logger.verbose(` πΈ Detected ${detectedAssets.length} assets`);
|
|
204
|
+
if (detectedAssets.length > 0) {
|
|
205
|
+
logger.debug(` Assets: ${detectedAssets.join(", ")}`);
|
|
206
|
+
}
|
|
207
|
+
if (detectedAssets.length === 0) {
|
|
208
|
+
return { content, assetsDownloaded: 0, assetsCached: 0 };
|
|
209
|
+
}
|
|
210
|
+
const assetMap = new Map();
|
|
211
|
+
let assetsDownloaded = 0;
|
|
212
|
+
let assetsCached = 0;
|
|
213
|
+
// Process each detected asset
|
|
214
|
+
await Promise.all(detectedAssets.map(async (assetPath) => {
|
|
215
|
+
logger.logAssetProcessing("Processing", assetPath);
|
|
216
|
+
try {
|
|
217
|
+
// Resolve the asset path relative to the current markdown file
|
|
218
|
+
const resolvedAssetPath = resolveAssetPath(filePath, assetPath);
|
|
219
|
+
logger.debug(` π Resolved path: ${resolvedAssetPath}`);
|
|
220
|
+
// Generate unique filename to avoid conflicts
|
|
221
|
+
const originalFilename = basename(assetPath);
|
|
222
|
+
const ext = extname(originalFilename);
|
|
223
|
+
const nameWithoutExt = basename(originalFilename, ext);
|
|
224
|
+
const uniqueFilename = `${nameWithoutExt}-${Date.now()}${ext}`;
|
|
225
|
+
const localPath = join(assetsPath, uniqueFilename);
|
|
226
|
+
logger.debug(` πΎ Local path: ${localPath}`);
|
|
227
|
+
// Check if asset already exists (simple cache check)
|
|
228
|
+
if (existsSync(localPath)) {
|
|
229
|
+
logger.logAssetProcessing("Cached", assetPath);
|
|
230
|
+
assetsCached++;
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
// Download the asset
|
|
234
|
+
logger.logAssetProcessing("Downloading", assetPath, `from ${owner}/${repo}@${ref}:${resolvedAssetPath}`);
|
|
235
|
+
await downloadAsset(octokit, owner, repo, ref, resolvedAssetPath, localPath, signal);
|
|
236
|
+
logger.logAssetProcessing("Downloaded", assetPath);
|
|
237
|
+
assetsDownloaded++;
|
|
238
|
+
}
|
|
239
|
+
// Generate URL for the transformed reference
|
|
240
|
+
const assetUrl = `${assetsBaseUrl}/${uniqueFilename}`.replace(/\/+/g, "/");
|
|
241
|
+
logger.debug(` π Transform: ${assetPath} -> ${assetUrl}`);
|
|
242
|
+
// Map the transformation
|
|
243
|
+
assetMap.set(assetPath, assetUrl);
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
logger.warn(` β Failed to process asset ${assetPath}: ${error}`);
|
|
247
|
+
}
|
|
248
|
+
}));
|
|
249
|
+
logger.verbose(` πΊοΈ Processed ${assetMap.size} assets: ${assetsDownloaded} downloaded, ${assetsCached} cached`);
|
|
250
|
+
// Transform the content with new asset references
|
|
251
|
+
const transformedContent = transformAssetReferences(content, assetMap);
|
|
252
|
+
return { content: transformedContent, assetsDownloaded, assetsCached };
|
|
253
|
+
}
|
package/dist/github.auth.js
CHANGED
|
@@ -4,7 +4,7 @@ import { createAppAuth } from "@octokit/auth-app";
|
|
|
4
4
|
* Type guard to check if config is GitHub App authentication
|
|
5
5
|
*/
|
|
6
6
|
function isGitHubAppAuth(config) {
|
|
7
|
-
return
|
|
7
|
+
return ("appId" in config && "privateKey" in config && "installationId" in config);
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
10
|
* Creates an authenticated Octokit instance with support for both Personal Access Tokens
|
|
@@ -91,15 +91,17 @@ export function createOctokitFromEnv() {
|
|
|
91
91
|
if (appId && privateKey && installationId) {
|
|
92
92
|
// Decode private key if it's base64 encoded (for easier .env storage)
|
|
93
93
|
let decodedPrivateKey = privateKey;
|
|
94
|
-
if (!privateKey.includes(
|
|
94
|
+
if (!privateKey.includes("BEGIN RSA PRIVATE KEY") &&
|
|
95
|
+
!privateKey.includes("BEGIN PRIVATE KEY")) {
|
|
95
96
|
try {
|
|
96
|
-
decodedPrivateKey = Buffer.from(privateKey,
|
|
97
|
+
decodedPrivateKey = Buffer.from(privateKey, "base64").toString("utf-8");
|
|
97
98
|
}
|
|
98
99
|
catch {
|
|
99
100
|
// If decoding fails, use as-is (might already be plaintext)
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
|
-
console
|
|
103
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
104
|
+
console.log("β Using GitHub App authentication (15,000 requests/hour)");
|
|
103
105
|
return createAuthenticatedOctokit({
|
|
104
106
|
appId,
|
|
105
107
|
privateKey: decodedPrivateKey,
|
|
@@ -109,11 +111,13 @@ export function createOctokitFromEnv() {
|
|
|
109
111
|
// Fallback to Personal Access Token
|
|
110
112
|
const token = process.env.GITHUB_TOKEN;
|
|
111
113
|
if (token) {
|
|
112
|
-
console
|
|
113
|
-
console.log(
|
|
114
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
115
|
+
console.log("β Using Personal Access Token authentication (5,000 requests/hour)");
|
|
116
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
117
|
+
console.log("π‘ Consider switching to GitHub App for 3x higher rate limits");
|
|
114
118
|
return createAuthenticatedOctokit({ token });
|
|
115
119
|
}
|
|
116
|
-
throw new Error(
|
|
117
|
-
|
|
118
|
-
|
|
120
|
+
throw new Error("No GitHub authentication credentials found. Please set either:\n" +
|
|
121
|
+
" - GITHUB_TOKEN (for PAT authentication)\n" +
|
|
122
|
+
" - GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_APP_INSTALLATION_ID (for GitHub App authentication)");
|
|
119
123
|
}
|
package/dist/github.cleanup.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Octokit } from "octokit";
|
|
2
|
+
import type { ExtendedLoaderContext, ImportOptions, SyncStats } from "./github.types.js";
|
|
2
3
|
/**
|
|
3
4
|
* Performs selective cleanup of obsolete files
|
|
4
5
|
*/
|
|
5
|
-
export declare function performSelectiveCleanup(config: ImportOptions, context:
|
|
6
|
+
export declare function performSelectiveCleanup(config: ImportOptions, context: ExtendedLoaderContext, octokit: Octokit, signal?: AbortSignal): Promise<SyncStats>;
|