@larkiny/astro-github-loader 0.11.3 β 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -55
- package/dist/github.assets.d.ts +70 -0
- package/dist/github.assets.js +253 -0
- package/dist/github.auth.js +13 -9
- package/dist/github.cleanup.d.ts +3 -2
- package/dist/github.cleanup.js +30 -23
- package/dist/github.constants.d.ts +0 -16
- package/dist/github.constants.js +0 -16
- package/dist/github.content.d.ts +5 -131
- package/dist/github.content.js +152 -794
- package/dist/github.dryrun.d.ts +9 -5
- package/dist/github.dryrun.js +46 -25
- package/dist/github.link-transform.d.ts +2 -2
- package/dist/github.link-transform.js +65 -57
- package/dist/github.loader.js +30 -46
- package/dist/github.logger.d.ts +2 -2
- package/dist/github.logger.js +33 -24
- package/dist/github.paths.d.ts +76 -0
- package/dist/github.paths.js +190 -0
- package/dist/github.storage.d.ts +15 -0
- package/dist/github.storage.js +109 -0
- package/dist/github.types.d.ts +34 -4
- package/dist/index.d.ts +8 -6
- package/dist/index.js +3 -6
- package/dist/test-helpers.d.ts +130 -0
- package/dist/test-helpers.js +194 -0
- package/package.json +3 -1
- package/src/github.assets.spec.ts +717 -0
- package/src/github.assets.ts +365 -0
- package/src/github.auth.spec.ts +245 -0
- package/src/github.auth.ts +24 -10
- package/src/github.cleanup.spec.ts +380 -0
- package/src/github.cleanup.ts +91 -47
- package/src/github.constants.ts +0 -17
- package/src/github.content.spec.ts +305 -454
- package/src/github.content.ts +259 -957
- package/src/github.dryrun.spec.ts +586 -0
- package/src/github.dryrun.ts +105 -54
- package/src/github.link-transform.spec.ts +1345 -0
- package/src/github.link-transform.ts +174 -95
- package/src/github.loader.spec.ts +75 -50
- package/src/github.loader.ts +101 -76
- package/src/github.logger.spec.ts +795 -0
- package/src/github.logger.ts +77 -35
- package/src/github.paths.spec.ts +523 -0
- package/src/github.paths.ts +259 -0
- package/src/github.storage.spec.ts +367 -0
- package/src/github.storage.ts +127 -0
- package/src/github.types.ts +48 -9
- package/src/index.ts +43 -6
- package/src/test-helpers.ts +215 -0
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ Load content from GitHub repositories into Astro content collections with flexib
|
|
|
9
9
|
- π οΈ **Content Transforms** - Apply custom transformations to content during import, with pattern-specific transforms
|
|
10
10
|
- β‘ **Intelligent Change Detection** - Ref-aware commit tracking that only triggers re-imports when your target branch/tag actually changes
|
|
11
11
|
- π **Stable Imports** - Non-destructive approach that preserves local content collections
|
|
12
|
-
- π **Optimized Performance** -
|
|
12
|
+
- π **Optimized Performance** - Git Trees API for efficient file discovery with minimal API calls
|
|
13
13
|
|
|
14
14
|
## Quick Start
|
|
15
15
|
|
|
@@ -68,10 +68,10 @@ export const collections = {
|
|
|
68
68
|
|
|
69
69
|
The loader supports two authentication methods with different rate limits:
|
|
70
70
|
|
|
71
|
-
| Method
|
|
72
|
-
|
|
71
|
+
| Method | Rate Limit | Best For |
|
|
72
|
+
| ---------------------------- | -------------------- | --------------------------------------------- |
|
|
73
73
|
| **GitHub App** (Recommended) | 15,000 requests/hour | Production, large imports, organizational use |
|
|
74
|
-
| **Personal Access Token**
|
|
74
|
+
| **Personal Access Token** | 5,000 requests/hour | Development, small imports |
|
|
75
75
|
|
|
76
76
|
### Option 1: GitHub App Authentication (Recommended - 3x Rate Limit)
|
|
77
77
|
|
|
@@ -252,6 +252,7 @@ export const collections = {
|
|
|
252
252
|
```
|
|
253
253
|
|
|
254
254
|
In this example:
|
|
255
|
+
|
|
255
256
|
- **Stable docs** (v2.0.0 tag): Never re-imports, provides stable reference
|
|
256
257
|
- **Latest docs** (main branch): Only re-imports when main branch changes
|
|
257
258
|
- **Beta features** (beta branch): Only re-imports when beta branch changes
|
|
@@ -500,7 +501,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
500
501
|
return `/reference/algokit-cli`;
|
|
501
502
|
},
|
|
502
503
|
global: true,
|
|
503
|
-
description: "Map CLI reference links to reference section",
|
|
504
504
|
},
|
|
505
505
|
|
|
506
506
|
// Transform README links to introduction
|
|
@@ -510,7 +510,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
510
510
|
return `/introduction`;
|
|
511
511
|
},
|
|
512
512
|
global: true,
|
|
513
|
-
description: "Map README links to introduction page",
|
|
514
513
|
},
|
|
515
514
|
],
|
|
516
515
|
},
|
|
@@ -518,39 +517,6 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
518
517
|
];
|
|
519
518
|
```
|
|
520
519
|
|
|
521
|
-
## Link Transformation Utilities
|
|
522
|
-
|
|
523
|
-
Handle markdown links with anchor fragments using built-in utilities:
|
|
524
|
-
|
|
525
|
-
```typescript
|
|
526
|
-
import {
|
|
527
|
-
createLinkTransform,
|
|
528
|
-
extractAnchor,
|
|
529
|
-
removeMarkdownExtension,
|
|
530
|
-
} from "@larkiny/astro-github-loader";
|
|
531
|
-
|
|
532
|
-
const linkTransform = createLinkTransform({
|
|
533
|
-
baseUrl: "/docs/imported",
|
|
534
|
-
pathTransform: (path, context) => {
|
|
535
|
-
const { path: cleanPath, anchor } = extractAnchor(path);
|
|
536
|
-
|
|
537
|
-
// Custom link handling logic
|
|
538
|
-
if (cleanPath === "README.md") {
|
|
539
|
-
return `/docs/imported/overview${anchor}`;
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
// Use utility to remove .md extension and preserve anchors
|
|
543
|
-
return `/docs/imported/${removeMarkdownExtension(path)}`;
|
|
544
|
-
},
|
|
545
|
-
});
|
|
546
|
-
```
|
|
547
|
-
|
|
548
|
-
### Link Transform Utilities
|
|
549
|
-
|
|
550
|
-
- **`extractAnchor(path)`** - Returns `{path, anchor}` separating the anchor fragment
|
|
551
|
-
- **`removeMarkdownExtension(path)`** - Removes `.md`/`.mdx` extensions while preserving anchors
|
|
552
|
-
- **`createLinkTransform(options)`** - Main transform with custom path handling
|
|
553
|
-
|
|
554
520
|
## Asset Import and Management
|
|
555
521
|
|
|
556
522
|
Automatically detect, download, and transform asset references:
|
|
@@ -595,18 +561,18 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
595
561
|
name: "Docs that need clearing",
|
|
596
562
|
owner: "your-org",
|
|
597
563
|
repo: "docs-repo",
|
|
598
|
-
clear: true,
|
|
564
|
+
clear: true, // Enable clearing for this config only
|
|
599
565
|
includes: [
|
|
600
|
-
{ pattern: "docs/**/*.md", basePath: "src/content/docs/imported" }
|
|
566
|
+
{ pattern: "docs/**/*.md", basePath: "src/content/docs/imported" },
|
|
601
567
|
],
|
|
602
568
|
},
|
|
603
569
|
{
|
|
604
570
|
name: "Docs that don't need clearing",
|
|
605
571
|
owner: "your-org",
|
|
606
572
|
repo: "other-docs",
|
|
607
|
-
clear: false,
|
|
573
|
+
clear: false, // Explicitly disable (or omit for default behavior)
|
|
608
574
|
includes: [
|
|
609
|
-
{ pattern: "guides/**/*.md", basePath: "src/content/docs/guides" }
|
|
575
|
+
{ pattern: "guides/**/*.md", basePath: "src/content/docs/guides" },
|
|
610
576
|
],
|
|
611
577
|
},
|
|
612
578
|
];
|
|
@@ -615,7 +581,7 @@ const REMOTE_CONTENT: ImportOptions[] = [
|
|
|
615
581
|
await githubLoader({
|
|
616
582
|
octokit,
|
|
617
583
|
configs: REMOTE_CONTENT,
|
|
618
|
-
clear: true,
|
|
584
|
+
clear: true, // Global default - can be overridden per-config
|
|
619
585
|
}).load(context);
|
|
620
586
|
```
|
|
621
587
|
|
|
@@ -627,6 +593,7 @@ await githubLoader({
|
|
|
627
593
|
### How It Works
|
|
628
594
|
|
|
629
595
|
Unlike a bulk clear operation, the loader uses a selective delete-before-set approach:
|
|
596
|
+
|
|
630
597
|
1. For each file being imported, if an entry already exists, it's deleted immediately before the new entry is added
|
|
631
598
|
2. This atomic replacement ensures the content collection is never empty
|
|
632
599
|
3. Astro's content collection system handles individual deletions gracefully
|
|
@@ -682,6 +649,7 @@ The loader uses intelligent, ref-aware change detection:
|
|
|
682
649
|
- **Efficient checking**: Only the latest commit of your target ref is checked
|
|
683
650
|
|
|
684
651
|
**Examples**:
|
|
652
|
+
|
|
685
653
|
- Config tracking `main` branch β only `main` commits trigger re-import
|
|
686
654
|
- Config tracking `v2.1.0` tag β never re-imports (tags are immutable)
|
|
687
655
|
- Config tracking `feature-branch` β ignores commits to `main`, `develop`, etc.
|
|
@@ -743,13 +711,16 @@ interface LinkMapping {
|
|
|
743
711
|
/** Replacement string or function */
|
|
744
712
|
replacement:
|
|
745
713
|
| string
|
|
746
|
-
| ((match: string, anchor: string, context:
|
|
714
|
+
| ((match: string, anchor: string, context: LinkTransformContext) => string);
|
|
747
715
|
|
|
748
716
|
/** Apply to all links, not just unresolved internal links (default: false) */
|
|
749
717
|
global?: boolean;
|
|
750
718
|
|
|
751
|
-
/**
|
|
752
|
-
|
|
719
|
+
/** Function to determine if this mapping should apply to the current file context */
|
|
720
|
+
contextFilter?: (context: LinkTransformContext) => boolean;
|
|
721
|
+
|
|
722
|
+
/** Automatically handle relative links by prefixing with target base path (default: false) */
|
|
723
|
+
relativeLinks?: boolean;
|
|
753
724
|
}
|
|
754
725
|
|
|
755
726
|
interface IncludePattern {
|
|
@@ -765,15 +736,17 @@ interface IncludePattern {
|
|
|
765
736
|
/**
|
|
766
737
|
* Map of source paths to target paths for controlling where files are imported.
|
|
767
738
|
*
|
|
768
|
-
* Supports two
|
|
769
|
-
* - **
|
|
770
|
-
* - **
|
|
739
|
+
* Supports two mapping formats:
|
|
740
|
+
* - **Simple string**: `'docs/README.md': 'docs/overview.md'`
|
|
741
|
+
* - **Enhanced object**: `'docs/api/': { target: 'api/', crossSectionPath: '/reference/api' }`
|
|
742
|
+
*
|
|
743
|
+
* And two mapping scopes:
|
|
744
|
+
* - **File mapping**: Exact file path match (e.g., `'docs/README.md': 'docs/overview.md'`)
|
|
745
|
+
* - **Folder mapping**: Trailing slash moves all files (e.g., `'docs/capabilities/': 'docs/'`)
|
|
771
746
|
*
|
|
772
747
|
* **Important**: Folder mappings require trailing slashes to distinguish from file mappings.
|
|
773
|
-
* - β
`'docs/capabilities/': 'docs/'` (folder mapping - moves all files)
|
|
774
|
-
* - β `'docs/capabilities': 'docs/'` (treated as exact file match)
|
|
775
748
|
*/
|
|
776
|
-
pathMappings?: Record<string,
|
|
749
|
+
pathMappings?: Record<string, PathMappingValue>;
|
|
777
750
|
}
|
|
778
751
|
```
|
|
779
752
|
|
|
@@ -801,8 +774,8 @@ type TransformFunction = (content: string, context: TransformContext) => string;
|
|
|
801
774
|
|
|
802
775
|
The loader includes several optimizations:
|
|
803
776
|
|
|
804
|
-
- **
|
|
805
|
-
- **Efficient API usage**: Minimizes GitHub API calls
|
|
777
|
+
- **Git Trees API**: Retrieves the entire repository file tree in a single API call (2 total: 1 for commit SHA + 1 for tree), replacing recursive directory traversal
|
|
778
|
+
- **Efficient API usage**: Minimizes GitHub API calls regardless of repository size or depth
|
|
806
779
|
- **Ref-aware change detection**: Tracks commit SHA for specific git references (branches/tags) to avoid unnecessary downloads when unrelated branches change
|
|
807
780
|
- **Concurrent processing**: Downloads and processes files in parallel
|
|
808
781
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { Octokit } from "octokit";
|
|
2
|
+
import type { Logger } from "./github.logger.js";
|
|
3
|
+
import type { ImportOptions } from "./github.types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Resolves the effective asset configuration for an import.
|
|
6
|
+
*
|
|
7
|
+
* If `assetsPath` and `assetsBaseUrl` are explicitly provided, uses them (existing behavior).
|
|
8
|
+
* If omitted, derives co-located defaults from the matched include pattern's basePath:
|
|
9
|
+
* - assetsPath: `{basePath}/assets/` (physical directory on disk)
|
|
10
|
+
* - assetsBaseUrl: `./assets` (relative reference in markdown)
|
|
11
|
+
*
|
|
12
|
+
* @param options - Import options that may or may not have explicit asset config
|
|
13
|
+
* @param filePath - The file being processed (used to find the matched include pattern)
|
|
14
|
+
* @returns Resolved assetsPath and assetsBaseUrl, or null if assets should not be processed
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export declare function resolveAssetConfig(options: ImportOptions, filePath: string): {
|
|
18
|
+
assetsPath: string;
|
|
19
|
+
assetsBaseUrl: string;
|
|
20
|
+
} | null;
|
|
21
|
+
/**
|
|
22
|
+
* Detects asset references in markdown content using regex patterns
|
|
23
|
+
* @param content - The markdown content to parse
|
|
24
|
+
* @param assetPatterns - File extensions to treat as assets
|
|
25
|
+
* @returns Array of detected asset paths
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export declare function detectAssets(content: string, assetPatterns?: string[]): string[];
|
|
29
|
+
/**
|
|
30
|
+
* Downloads an asset from GitHub and saves it locally
|
|
31
|
+
* @param octokit - GitHub API client
|
|
32
|
+
* @param owner - Repository owner
|
|
33
|
+
* @param repo - Repository name
|
|
34
|
+
* @param ref - Git reference
|
|
35
|
+
* @param assetPath - Path to the asset in the repository
|
|
36
|
+
* @param localPath - Local path where the asset should be saved
|
|
37
|
+
* @param signal - Abort signal for cancellation
|
|
38
|
+
* @returns Promise that resolves when the asset is downloaded
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export declare function downloadAsset(octokit: Octokit, owner: string, repo: string, ref: string, assetPath: string, localPath: string, signal?: AbortSignal): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Transforms asset references in markdown content to use local paths
|
|
44
|
+
* @param content - The markdown content to transform
|
|
45
|
+
* @param assetMap - Map of original asset paths to new local paths
|
|
46
|
+
* @returns Transformed content with updated asset references
|
|
47
|
+
* @internal
|
|
48
|
+
*/
|
|
49
|
+
export declare function transformAssetReferences(content: string, assetMap: Map<string, string>): string;
|
|
50
|
+
/**
|
|
51
|
+
* Resolves an asset path relative to a base path
|
|
52
|
+
* @internal
|
|
53
|
+
*/
|
|
54
|
+
export declare function resolveAssetPath(basePath: string, assetPath: string): string;
|
|
55
|
+
/**
|
|
56
|
+
* Processes assets in markdown content by detecting, downloading, and transforming references
|
|
57
|
+
* @param content - The markdown content to process
|
|
58
|
+
* @param filePath - The file path of the markdown file being processed
|
|
59
|
+
* @param options - Configuration options including asset settings
|
|
60
|
+
* @param octokit - GitHub API client
|
|
61
|
+
* @param logger - Logger instance for output
|
|
62
|
+
* @param signal - Abort signal for cancellation
|
|
63
|
+
* @returns Promise that resolves to transformed content
|
|
64
|
+
* @internal
|
|
65
|
+
*/
|
|
66
|
+
export declare function processAssets(content: string, filePath: string, options: ImportOptions, octokit: Octokit, logger: Logger, signal?: AbortSignal): Promise<{
|
|
67
|
+
content: string;
|
|
68
|
+
assetsDownloaded: number;
|
|
69
|
+
assetsCached: number;
|
|
70
|
+
}>;
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { existsSync, promises as fs } from "node:fs";
|
|
2
|
+
import { join, dirname, basename, extname } from "node:path";
|
|
3
|
+
import { shouldIncludeFile } from "./github.paths.js";
|
|
4
|
+
/**
|
|
5
|
+
* Default asset patterns for common image and media file types
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
const DEFAULT_ASSET_PATTERNS = [
|
|
9
|
+
".png",
|
|
10
|
+
".jpg",
|
|
11
|
+
".jpeg",
|
|
12
|
+
".gif",
|
|
13
|
+
".svg",
|
|
14
|
+
".webp",
|
|
15
|
+
".ico",
|
|
16
|
+
".bmp",
|
|
17
|
+
];
|
|
18
|
+
/**
|
|
19
|
+
* Resolves the effective asset configuration for an import.
|
|
20
|
+
*
|
|
21
|
+
* If `assetsPath` and `assetsBaseUrl` are explicitly provided, uses them (existing behavior).
|
|
22
|
+
* If omitted, derives co-located defaults from the matched include pattern's basePath:
|
|
23
|
+
* - assetsPath: `{basePath}/assets/` (physical directory on disk)
|
|
24
|
+
* - assetsBaseUrl: `./assets` (relative reference in markdown)
|
|
25
|
+
*
|
|
26
|
+
* @param options - Import options that may or may not have explicit asset config
|
|
27
|
+
* @param filePath - The file being processed (used to find the matched include pattern)
|
|
28
|
+
* @returns Resolved assetsPath and assetsBaseUrl, or null if assets should not be processed
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
31
|
+
export function resolveAssetConfig(options, filePath) {
|
|
32
|
+
// Explicit config takes precedence
|
|
33
|
+
if (options.assetsPath && options.assetsBaseUrl) {
|
|
34
|
+
return {
|
|
35
|
+
assetsPath: options.assetsPath,
|
|
36
|
+
assetsBaseUrl: options.assetsBaseUrl,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
// If only one is set, that's a misconfiguration β skip
|
|
40
|
+
if (options.assetsPath || options.assetsBaseUrl) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
// Derive co-located defaults from the matched include pattern's basePath
|
|
44
|
+
const includeResult = shouldIncludeFile(filePath, options);
|
|
45
|
+
if (includeResult.included && includeResult.matchedPattern) {
|
|
46
|
+
const basePath = includeResult.matchedPattern.basePath;
|
|
47
|
+
return {
|
|
48
|
+
assetsPath: join(basePath, "assets"),
|
|
49
|
+
assetsBaseUrl: "./assets",
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Detects asset references in markdown content using regex patterns
|
|
56
|
+
* @param content - The markdown content to parse
|
|
57
|
+
* @param assetPatterns - File extensions to treat as assets
|
|
58
|
+
* @returns Array of detected asset paths
|
|
59
|
+
* @internal
|
|
60
|
+
*/
|
|
61
|
+
export function detectAssets(content, assetPatterns = DEFAULT_ASSET_PATTERNS) {
|
|
62
|
+
const assets = [];
|
|
63
|
+
const patterns = assetPatterns.map((ext) => ext.toLowerCase());
|
|
64
|
+
// Match markdown images: 
|
|
65
|
+
const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
|
|
66
|
+
let match;
|
|
67
|
+
while ((match = imageRegex.exec(content)) !== null) {
|
|
68
|
+
const assetPath = match[1];
|
|
69
|
+
// Only include relative paths and assets matching our patterns
|
|
70
|
+
if (assetPath.startsWith("./") ||
|
|
71
|
+
assetPath.startsWith("../") ||
|
|
72
|
+
!assetPath.includes("://")) {
|
|
73
|
+
const ext = extname(assetPath).toLowerCase();
|
|
74
|
+
if (patterns.includes(ext)) {
|
|
75
|
+
assets.push(assetPath);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// Match HTML img tags: <img src="path">
|
|
80
|
+
const htmlImgRegex = /<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi;
|
|
81
|
+
while ((match = htmlImgRegex.exec(content)) !== null) {
|
|
82
|
+
const assetPath = match[1];
|
|
83
|
+
if (assetPath.startsWith("./") ||
|
|
84
|
+
assetPath.startsWith("../") ||
|
|
85
|
+
!assetPath.includes("://")) {
|
|
86
|
+
const ext = extname(assetPath).toLowerCase();
|
|
87
|
+
if (patterns.includes(ext)) {
|
|
88
|
+
assets.push(assetPath);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return [...new Set(assets)]; // Remove duplicates
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Downloads an asset from GitHub and saves it locally
|
|
96
|
+
* @param octokit - GitHub API client
|
|
97
|
+
* @param owner - Repository owner
|
|
98
|
+
* @param repo - Repository name
|
|
99
|
+
* @param ref - Git reference
|
|
100
|
+
* @param assetPath - Path to the asset in the repository
|
|
101
|
+
* @param localPath - Local path where the asset should be saved
|
|
102
|
+
* @param signal - Abort signal for cancellation
|
|
103
|
+
* @returns Promise that resolves when the asset is downloaded
|
|
104
|
+
* @internal
|
|
105
|
+
*/
|
|
106
|
+
export async function downloadAsset(octokit, owner, repo, ref, assetPath, localPath, signal) {
|
|
107
|
+
try {
|
|
108
|
+
const { data } = await octokit.rest.repos.getContent({
|
|
109
|
+
owner,
|
|
110
|
+
repo,
|
|
111
|
+
path: assetPath,
|
|
112
|
+
ref,
|
|
113
|
+
request: { signal },
|
|
114
|
+
});
|
|
115
|
+
if (Array.isArray(data)) {
|
|
116
|
+
throw new Error(`Asset ${assetPath} is a directory, not a file`);
|
|
117
|
+
}
|
|
118
|
+
if (data.type !== "file" || !data.download_url) {
|
|
119
|
+
throw new Error(`Asset ${assetPath} is not a valid file (type: ${data.type}, downloadUrl: ${data.download_url})`);
|
|
120
|
+
}
|
|
121
|
+
const response = await fetch(data.download_url, { signal });
|
|
122
|
+
if (!response.ok) {
|
|
123
|
+
throw new Error(`Failed to download asset: ${response.status} ${response.statusText}`);
|
|
124
|
+
}
|
|
125
|
+
const buffer = await response.arrayBuffer();
|
|
126
|
+
const dir = dirname(localPath);
|
|
127
|
+
if (!existsSync(dir)) {
|
|
128
|
+
await fs.mkdir(dir, { recursive: true });
|
|
129
|
+
}
|
|
130
|
+
await fs.writeFile(localPath, new Uint8Array(buffer));
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
if (typeof error === "object" &&
|
|
134
|
+
error !== null &&
|
|
135
|
+
"status" in error &&
|
|
136
|
+
error.status === 404) {
|
|
137
|
+
throw new Error(`Asset not found: ${assetPath}`);
|
|
138
|
+
}
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Transforms asset references in markdown content to use local paths
|
|
144
|
+
* @param content - The markdown content to transform
|
|
145
|
+
* @param assetMap - Map of original asset paths to new local paths
|
|
146
|
+
* @returns Transformed content with updated asset references
|
|
147
|
+
* @internal
|
|
148
|
+
*/
|
|
149
|
+
export function transformAssetReferences(content, assetMap) {
|
|
150
|
+
let transformedContent = content;
|
|
151
|
+
for (const [originalPath, newPath] of assetMap) {
|
|
152
|
+
// Transform markdown images
|
|
153
|
+
const imageRegex = new RegExp(`(!)\\[([^\\]]*)\\]\\(\\s*${escapeRegExp(originalPath)}\\s*\\)`, "g");
|
|
154
|
+
transformedContent = transformedContent.replace(imageRegex, `$1[$2](${newPath})`);
|
|
155
|
+
// Transform HTML img tags
|
|
156
|
+
const htmlRegex = new RegExp(`(<img[^>]+src\\s*=\\s*["'])${escapeRegExp(originalPath)}(["'][^>]*>)`, "gi");
|
|
157
|
+
transformedContent = transformedContent.replace(htmlRegex, `$1${newPath}$2`);
|
|
158
|
+
}
|
|
159
|
+
return transformedContent;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Escapes special regex characters in a string
|
|
163
|
+
* @internal
|
|
164
|
+
*/
|
|
165
|
+
function escapeRegExp(string) {
|
|
166
|
+
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Resolves an asset path relative to a base path
|
|
170
|
+
* @internal
|
|
171
|
+
*/
|
|
172
|
+
export function resolveAssetPath(basePath, assetPath) {
|
|
173
|
+
if (assetPath.startsWith("./")) {
|
|
174
|
+
return join(dirname(basePath), assetPath.slice(2));
|
|
175
|
+
}
|
|
176
|
+
else if (assetPath.startsWith("../")) {
|
|
177
|
+
return join(dirname(basePath), assetPath);
|
|
178
|
+
}
|
|
179
|
+
return assetPath;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Processes assets in markdown content by detecting, downloading, and transforming references
|
|
183
|
+
* @param content - The markdown content to process
|
|
184
|
+
* @param filePath - The file path of the markdown file being processed
|
|
185
|
+
* @param options - Configuration options including asset settings
|
|
186
|
+
* @param octokit - GitHub API client
|
|
187
|
+
* @param logger - Logger instance for output
|
|
188
|
+
* @param signal - Abort signal for cancellation
|
|
189
|
+
* @returns Promise that resolves to transformed content
|
|
190
|
+
* @internal
|
|
191
|
+
*/
|
|
192
|
+
export async function processAssets(content, filePath, options, octokit, logger, signal) {
|
|
193
|
+
const { owner, repo, ref = "main", assetsPath, assetsBaseUrl, assetPatterns, } = options;
|
|
194
|
+
logger.verbose(`πΌοΈ Processing assets for ${filePath}`);
|
|
195
|
+
logger.debug(` assetsPath: ${assetsPath}`);
|
|
196
|
+
logger.debug(` assetsBaseUrl: ${assetsBaseUrl}`);
|
|
197
|
+
if (!assetsPath || !assetsBaseUrl) {
|
|
198
|
+
logger.verbose(` βοΈ Skipping asset processing - missing assetsPath or assetsBaseUrl`);
|
|
199
|
+
return { content, assetsDownloaded: 0, assetsCached: 0 };
|
|
200
|
+
}
|
|
201
|
+
// Detect assets in the content
|
|
202
|
+
const detectedAssets = detectAssets(content, assetPatterns);
|
|
203
|
+
logger.verbose(` πΈ Detected ${detectedAssets.length} assets`);
|
|
204
|
+
if (detectedAssets.length > 0) {
|
|
205
|
+
logger.debug(` Assets: ${detectedAssets.join(", ")}`);
|
|
206
|
+
}
|
|
207
|
+
if (detectedAssets.length === 0) {
|
|
208
|
+
return { content, assetsDownloaded: 0, assetsCached: 0 };
|
|
209
|
+
}
|
|
210
|
+
const assetMap = new Map();
|
|
211
|
+
let assetsDownloaded = 0;
|
|
212
|
+
let assetsCached = 0;
|
|
213
|
+
// Process each detected asset
|
|
214
|
+
await Promise.all(detectedAssets.map(async (assetPath) => {
|
|
215
|
+
logger.logAssetProcessing("Processing", assetPath);
|
|
216
|
+
try {
|
|
217
|
+
// Resolve the asset path relative to the current markdown file
|
|
218
|
+
const resolvedAssetPath = resolveAssetPath(filePath, assetPath);
|
|
219
|
+
logger.debug(` π Resolved path: ${resolvedAssetPath}`);
|
|
220
|
+
// Generate unique filename to avoid conflicts
|
|
221
|
+
const originalFilename = basename(assetPath);
|
|
222
|
+
const ext = extname(originalFilename);
|
|
223
|
+
const nameWithoutExt = basename(originalFilename, ext);
|
|
224
|
+
const uniqueFilename = `${nameWithoutExt}-${Date.now()}${ext}`;
|
|
225
|
+
const localPath = join(assetsPath, uniqueFilename);
|
|
226
|
+
logger.debug(` πΎ Local path: ${localPath}`);
|
|
227
|
+
// Check if asset already exists (simple cache check)
|
|
228
|
+
if (existsSync(localPath)) {
|
|
229
|
+
logger.logAssetProcessing("Cached", assetPath);
|
|
230
|
+
assetsCached++;
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
// Download the asset
|
|
234
|
+
logger.logAssetProcessing("Downloading", assetPath, `from ${owner}/${repo}@${ref}:${resolvedAssetPath}`);
|
|
235
|
+
await downloadAsset(octokit, owner, repo, ref, resolvedAssetPath, localPath, signal);
|
|
236
|
+
logger.logAssetProcessing("Downloaded", assetPath);
|
|
237
|
+
assetsDownloaded++;
|
|
238
|
+
}
|
|
239
|
+
// Generate URL for the transformed reference
|
|
240
|
+
const assetUrl = `${assetsBaseUrl}/${uniqueFilename}`.replace(/\/+/g, "/");
|
|
241
|
+
logger.debug(` π Transform: ${assetPath} -> ${assetUrl}`);
|
|
242
|
+
// Map the transformation
|
|
243
|
+
assetMap.set(assetPath, assetUrl);
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
logger.warn(` β Failed to process asset ${assetPath}: ${error}`);
|
|
247
|
+
}
|
|
248
|
+
}));
|
|
249
|
+
logger.verbose(` πΊοΈ Processed ${assetMap.size} assets: ${assetsDownloaded} downloaded, ${assetsCached} cached`);
|
|
250
|
+
// Transform the content with new asset references
|
|
251
|
+
const transformedContent = transformAssetReferences(content, assetMap);
|
|
252
|
+
return { content: transformedContent, assetsDownloaded, assetsCached };
|
|
253
|
+
}
|
package/dist/github.auth.js
CHANGED
|
@@ -4,7 +4,7 @@ import { createAppAuth } from "@octokit/auth-app";
|
|
|
4
4
|
* Type guard to check if config is GitHub App authentication
|
|
5
5
|
*/
|
|
6
6
|
function isGitHubAppAuth(config) {
|
|
7
|
-
return
|
|
7
|
+
return ("appId" in config && "privateKey" in config && "installationId" in config);
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
10
|
* Creates an authenticated Octokit instance with support for both Personal Access Tokens
|
|
@@ -91,15 +91,17 @@ export function createOctokitFromEnv() {
|
|
|
91
91
|
if (appId && privateKey && installationId) {
|
|
92
92
|
// Decode private key if it's base64 encoded (for easier .env storage)
|
|
93
93
|
let decodedPrivateKey = privateKey;
|
|
94
|
-
if (!privateKey.includes(
|
|
94
|
+
if (!privateKey.includes("BEGIN RSA PRIVATE KEY") &&
|
|
95
|
+
!privateKey.includes("BEGIN PRIVATE KEY")) {
|
|
95
96
|
try {
|
|
96
|
-
decodedPrivateKey = Buffer.from(privateKey,
|
|
97
|
+
decodedPrivateKey = Buffer.from(privateKey, "base64").toString("utf-8");
|
|
97
98
|
}
|
|
98
99
|
catch {
|
|
99
100
|
// If decoding fails, use as-is (might already be plaintext)
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
|
-
console
|
|
103
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
104
|
+
console.log("β Using GitHub App authentication (15,000 requests/hour)");
|
|
103
105
|
return createAuthenticatedOctokit({
|
|
104
106
|
appId,
|
|
105
107
|
privateKey: decodedPrivateKey,
|
|
@@ -109,11 +111,13 @@ export function createOctokitFromEnv() {
|
|
|
109
111
|
// Fallback to Personal Access Token
|
|
110
112
|
const token = process.env.GITHUB_TOKEN;
|
|
111
113
|
if (token) {
|
|
112
|
-
console
|
|
113
|
-
console.log(
|
|
114
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
115
|
+
console.log("β Using Personal Access Token authentication (5,000 requests/hour)");
|
|
116
|
+
// eslint-disable-next-line no-console -- startup message before logger exists
|
|
117
|
+
console.log("π‘ Consider switching to GitHub App for 3x higher rate limits");
|
|
114
118
|
return createAuthenticatedOctokit({ token });
|
|
115
119
|
}
|
|
116
|
-
throw new Error(
|
|
117
|
-
|
|
118
|
-
|
|
120
|
+
throw new Error("No GitHub authentication credentials found. Please set either:\n" +
|
|
121
|
+
" - GITHUB_TOKEN (for PAT authentication)\n" +
|
|
122
|
+
" - GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_APP_INSTALLATION_ID (for GitHub App authentication)");
|
|
119
123
|
}
|
package/dist/github.cleanup.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Octokit } from "octokit";
|
|
2
|
+
import type { ExtendedLoaderContext, ImportOptions, SyncStats } from "./github.types.js";
|
|
2
3
|
/**
|
|
3
4
|
* Performs selective cleanup of obsolete files
|
|
4
5
|
*/
|
|
5
|
-
export declare function performSelectiveCleanup(config: ImportOptions, context:
|
|
6
|
+
export declare function performSelectiveCleanup(config: ImportOptions, context: ExtendedLoaderContext, octokit: Octokit, signal?: AbortSignal): Promise<SyncStats>;
|