@larkiny/astro-github-loader 0.11.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -55
- package/dist/github.assets.d.ts +70 -0
- package/dist/github.assets.js +253 -0
- package/dist/github.auth.js +13 -9
- package/dist/github.cleanup.d.ts +3 -2
- package/dist/github.cleanup.js +30 -23
- package/dist/github.constants.d.ts +0 -16
- package/dist/github.constants.js +0 -16
- package/dist/github.content.d.ts +5 -131
- package/dist/github.content.js +152 -794
- package/dist/github.dryrun.d.ts +9 -5
- package/dist/github.dryrun.js +46 -25
- package/dist/github.link-transform.d.ts +2 -2
- package/dist/github.link-transform.js +65 -57
- package/dist/github.loader.js +30 -46
- package/dist/github.logger.d.ts +2 -2
- package/dist/github.logger.js +33 -24
- package/dist/github.paths.d.ts +76 -0
- package/dist/github.paths.js +190 -0
- package/dist/github.storage.d.ts +15 -0
- package/dist/github.storage.js +109 -0
- package/dist/github.types.d.ts +34 -4
- package/dist/index.d.ts +8 -6
- package/dist/index.js +3 -6
- package/dist/test-helpers.d.ts +130 -0
- package/dist/test-helpers.js +194 -0
- package/package.json +3 -1
- package/src/github.assets.spec.ts +717 -0
- package/src/github.assets.ts +365 -0
- package/src/github.auth.spec.ts +245 -0
- package/src/github.auth.ts +24 -10
- package/src/github.cleanup.spec.ts +380 -0
- package/src/github.cleanup.ts +91 -47
- package/src/github.constants.ts +0 -17
- package/src/github.content.spec.ts +305 -454
- package/src/github.content.ts +259 -957
- package/src/github.dryrun.spec.ts +586 -0
- package/src/github.dryrun.ts +105 -54
- package/src/github.link-transform.spec.ts +1345 -0
- package/src/github.link-transform.ts +174 -95
- package/src/github.loader.spec.ts +75 -50
- package/src/github.loader.ts +101 -76
- package/src/github.logger.spec.ts +795 -0
- package/src/github.logger.ts +77 -35
- package/src/github.paths.spec.ts +523 -0
- package/src/github.paths.ts +259 -0
- package/src/github.storage.spec.ts +367 -0
- package/src/github.storage.ts +127 -0
- package/src/github.types.ts +48 -9
- package/src/index.ts +43 -6
- package/src/test-helpers.ts +215 -0
package/dist/github.logger.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Multi-level logging system for astro-github-loader
|
|
3
3
|
*/
|
|
4
|
-
export type LogLevel =
|
|
4
|
+
export type LogLevel = "silent" | "default" | "verbose" | "debug";
|
|
5
5
|
export interface LoggerOptions {
|
|
6
6
|
level: LogLevel;
|
|
7
7
|
prefix?: string;
|
|
@@ -16,7 +16,7 @@ export interface ImportSummary {
|
|
|
16
16
|
assetsDownloaded?: number;
|
|
17
17
|
assetsCached?: number;
|
|
18
18
|
duration: number;
|
|
19
|
-
status:
|
|
19
|
+
status: "success" | "error" | "cancelled";
|
|
20
20
|
error?: string;
|
|
21
21
|
}
|
|
22
22
|
export interface SyncSummary {
|
package/dist/github.logger.js
CHANGED
|
@@ -6,10 +6,10 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export class Logger {
|
|
8
8
|
constructor(options) {
|
|
9
|
-
this.spinnerChars = [
|
|
9
|
+
this.spinnerChars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
10
10
|
this.spinnerIndex = 0;
|
|
11
11
|
this.level = options.level;
|
|
12
|
-
this.prefix = options.prefix ||
|
|
12
|
+
this.prefix = options.prefix || "";
|
|
13
13
|
}
|
|
14
14
|
/**
|
|
15
15
|
* Set the logging level
|
|
@@ -51,7 +51,7 @@ export class Logger {
|
|
|
51
51
|
* Default level - summary information only
|
|
52
52
|
*/
|
|
53
53
|
info(message) {
|
|
54
|
-
if (this.shouldLog(
|
|
54
|
+
if (this.shouldLog("default")) {
|
|
55
55
|
console.log(this.formatMessage(message));
|
|
56
56
|
}
|
|
57
57
|
}
|
|
@@ -59,7 +59,7 @@ export class Logger {
|
|
|
59
59
|
* Verbose level - detailed operation information
|
|
60
60
|
*/
|
|
61
61
|
verbose(message) {
|
|
62
|
-
if (this.shouldLog(
|
|
62
|
+
if (this.shouldLog("verbose")) {
|
|
63
63
|
console.log(this.formatMessage(message));
|
|
64
64
|
}
|
|
65
65
|
}
|
|
@@ -67,7 +67,7 @@ export class Logger {
|
|
|
67
67
|
* Debug level - all information including diagnostics
|
|
68
68
|
*/
|
|
69
69
|
debug(message) {
|
|
70
|
-
if (this.shouldLog(
|
|
70
|
+
if (this.shouldLog("debug")) {
|
|
71
71
|
console.log(this.formatMessage(message));
|
|
72
72
|
}
|
|
73
73
|
}
|
|
@@ -75,7 +75,7 @@ export class Logger {
|
|
|
75
75
|
* Error - always shown unless silent
|
|
76
76
|
*/
|
|
77
77
|
error(message) {
|
|
78
|
-
if (this.shouldLog(
|
|
78
|
+
if (this.shouldLog("default")) {
|
|
79
79
|
console.error(this.formatMessage(message));
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -83,7 +83,7 @@ export class Logger {
|
|
|
83
83
|
* Warning - shown at default level and above
|
|
84
84
|
*/
|
|
85
85
|
warn(message) {
|
|
86
|
-
if (this.shouldLog(
|
|
86
|
+
if (this.shouldLog("default")) {
|
|
87
87
|
console.warn(this.formatMessage(message));
|
|
88
88
|
}
|
|
89
89
|
}
|
|
@@ -91,27 +91,32 @@ export class Logger {
|
|
|
91
91
|
* Log structured import summary (default level)
|
|
92
92
|
*/
|
|
93
93
|
logImportSummary(summary) {
|
|
94
|
-
if (!this.shouldLog(
|
|
94
|
+
if (!this.shouldLog("default"))
|
|
95
95
|
return;
|
|
96
|
-
const statusIcon = summary.status ===
|
|
97
|
-
|
|
96
|
+
const statusIcon = summary.status === "success"
|
|
97
|
+
? "✅"
|
|
98
|
+
: summary.status === "error"
|
|
99
|
+
? "❌"
|
|
100
|
+
: "🚫";
|
|
101
|
+
this.info("");
|
|
98
102
|
this.info(`📊 Import Summary: ${summary.configName}`);
|
|
99
|
-
this.info(`├─ Repository: ${summary.repository}${summary.ref ? `@${summary.ref}` :
|
|
103
|
+
this.info(`├─ Repository: ${summary.repository}${summary.ref ? `@${summary.ref}` : ""}`);
|
|
100
104
|
this.info(`├─ Files: ${summary.filesProcessed} processed, ${summary.filesUpdated} updated, ${summary.filesUnchanged} unchanged`);
|
|
101
|
-
if (summary.assetsDownloaded !== undefined ||
|
|
105
|
+
if (summary.assetsDownloaded !== undefined ||
|
|
106
|
+
summary.assetsCached !== undefined) {
|
|
102
107
|
const downloaded = summary.assetsDownloaded || 0;
|
|
103
108
|
const cached = summary.assetsCached || 0;
|
|
104
109
|
this.info(`├─ Assets: ${downloaded} downloaded, ${cached} cached`);
|
|
105
110
|
}
|
|
106
111
|
this.info(`├─ Duration: ${(summary.duration / 1000).toFixed(1)}s`);
|
|
107
|
-
this.info(`└─ Status: ${statusIcon} ${summary.status ===
|
|
108
|
-
this.info(
|
|
112
|
+
this.info(`└─ Status: ${statusIcon} ${summary.status === "success" ? "Success" : summary.status === "error" ? `Error: ${summary.error}` : "Cancelled"}`);
|
|
113
|
+
this.info("");
|
|
109
114
|
}
|
|
110
115
|
/**
|
|
111
116
|
* Log sync operation summary (default level)
|
|
112
117
|
*/
|
|
113
118
|
logSyncSummary(configName, summary) {
|
|
114
|
-
if (!this.shouldLog(
|
|
119
|
+
if (!this.shouldLog("default"))
|
|
115
120
|
return;
|
|
116
121
|
if (summary.added > 0 || summary.updated > 0 || summary.deleted > 0) {
|
|
117
122
|
this.info(`Sync completed for ${configName}: ${summary.added} added, ${summary.updated} updated, ${summary.deleted} deleted (${summary.duration}ms)`);
|
|
@@ -124,7 +129,7 @@ export class Logger {
|
|
|
124
129
|
* Log cleanup operation summary (default level)
|
|
125
130
|
*/
|
|
126
131
|
logCleanupSummary(configName, summary) {
|
|
127
|
-
if (!this.shouldLog(
|
|
132
|
+
if (!this.shouldLog("default"))
|
|
128
133
|
return;
|
|
129
134
|
if (summary.deleted > 0) {
|
|
130
135
|
this.info(`Cleanup completed for ${configName}: ${summary.deleted} obsolete files deleted (${summary.duration}ms)`);
|
|
@@ -137,14 +142,18 @@ export class Logger {
|
|
|
137
142
|
* Log file-level processing (verbose level)
|
|
138
143
|
*/
|
|
139
144
|
logFileProcessing(action, filePath, details) {
|
|
140
|
-
const message = details
|
|
145
|
+
const message = details
|
|
146
|
+
? `${action}: ${filePath} - ${details}`
|
|
147
|
+
: `${action}: ${filePath}`;
|
|
141
148
|
this.verbose(message);
|
|
142
149
|
}
|
|
143
150
|
/**
|
|
144
151
|
* Log asset processing (verbose level)
|
|
145
152
|
*/
|
|
146
153
|
logAssetProcessing(action, assetPath, details) {
|
|
147
|
-
const message = details
|
|
154
|
+
const message = details
|
|
155
|
+
? `Asset ${action}: ${assetPath} - ${details}`
|
|
156
|
+
: `Asset ${action}: ${assetPath}`;
|
|
148
157
|
this.verbose(message);
|
|
149
158
|
}
|
|
150
159
|
/**
|
|
@@ -195,8 +204,8 @@ export class Logger {
|
|
|
195
204
|
/**
|
|
196
205
|
* Start a spinner with duration timer for long-running operations
|
|
197
206
|
*/
|
|
198
|
-
startSpinner(message =
|
|
199
|
-
if (this.level ===
|
|
207
|
+
startSpinner(message = "Processing...") {
|
|
208
|
+
if (this.level === "silent")
|
|
200
209
|
return;
|
|
201
210
|
this.spinnerStartTime = Date.now();
|
|
202
211
|
this.spinnerIndex = 0;
|
|
@@ -232,7 +241,7 @@ export class Logger {
|
|
|
232
241
|
process.stdout.write(`\r${formattedMessage}\n`);
|
|
233
242
|
}
|
|
234
243
|
else {
|
|
235
|
-
process.stdout.write(
|
|
244
|
+
process.stdout.write("\r\x1b[K"); // Clear the line
|
|
236
245
|
}
|
|
237
246
|
this.spinnerStartTime = undefined;
|
|
238
247
|
}
|
|
@@ -243,11 +252,11 @@ export class Logger {
|
|
|
243
252
|
this.startSpinner(message);
|
|
244
253
|
try {
|
|
245
254
|
const result = await fn();
|
|
246
|
-
this.stopSpinner(successMessage || `✅ ${message.replace(/^[🔄⏳]?\s*/,
|
|
255
|
+
this.stopSpinner(successMessage || `✅ ${message.replace(/^[🔄⏳]?\s*/, "")} completed`);
|
|
247
256
|
return result;
|
|
248
257
|
}
|
|
249
258
|
catch (error) {
|
|
250
|
-
this.stopSpinner(errorMessage || `❌ ${message.replace(/^[🔄⏳]?\s*/,
|
|
259
|
+
this.stopSpinner(errorMessage || `❌ ${message.replace(/^[🔄⏳]?\s*/, "")} failed`);
|
|
251
260
|
throw error;
|
|
252
261
|
}
|
|
253
262
|
}
|
|
@@ -255,6 +264,6 @@ export class Logger {
|
|
|
255
264
|
/**
|
|
256
265
|
* Create a logger instance with the specified level
|
|
257
266
|
*/
|
|
258
|
-
export function createLogger(level =
|
|
267
|
+
export function createLogger(level = "default", prefix) {
|
|
259
268
|
return new Logger({ level, prefix });
|
|
260
269
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { ExtendedLoaderContext, ImportOptions, MatchedPattern } from "./github.types.js";
|
|
2
|
+
export interface ImportStats {
|
|
3
|
+
processed: number;
|
|
4
|
+
updated: number;
|
|
5
|
+
unchanged: number;
|
|
6
|
+
assetsDownloaded?: number;
|
|
7
|
+
assetsCached?: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Generates a unique identifier from a file path by removing the extension
|
|
11
|
+
* @param filePath - The file path to generate ID from
|
|
12
|
+
* @return {string} The generated identifier as a string with extension removed
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
export declare function generateId(filePath: string): string;
|
|
16
|
+
/**
|
|
17
|
+
* Applies path mapping logic to get the final filename for a file
|
|
18
|
+
*
|
|
19
|
+
* Supports two types of path mappings:
|
|
20
|
+
* - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
|
|
21
|
+
* - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
|
|
22
|
+
*
|
|
23
|
+
* @param filePath - Original source file path
|
|
24
|
+
* @param matchedPattern - The pattern that matched this file
|
|
25
|
+
* @param options - Import options containing path mappings
|
|
26
|
+
* @returns Final filename after applying path mapping logic
|
|
27
|
+
* @internal
|
|
28
|
+
*/
|
|
29
|
+
export declare function applyRename(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
|
|
30
|
+
/**
|
|
31
|
+
* Generates a local file path based on the matched pattern and file path
|
|
32
|
+
* @param filePath - The original file path from the repository
|
|
33
|
+
* @param matchedPattern - The pattern that matched this file (or null if no includes specified)
|
|
34
|
+
* @param options - Import options containing includes patterns for path mapping lookups
|
|
35
|
+
* @return {string} The local file path where this content should be stored
|
|
36
|
+
* @internal
|
|
37
|
+
*/
|
|
38
|
+
export declare function generatePath(filePath: string, matchedPattern?: MatchedPattern | null, options?: ImportOptions): string;
|
|
39
|
+
/**
|
|
40
|
+
* Checks if a file path should be included and returns the matching pattern
|
|
41
|
+
* @param filePath - The file path to check (relative to the repository root)
|
|
42
|
+
* @param options - Import options containing includes patterns
|
|
43
|
+
* @returns Object with include status and matched pattern, or null if not included
|
|
44
|
+
* @internal
|
|
45
|
+
*/
|
|
46
|
+
export declare function shouldIncludeFile(filePath: string, options: ImportOptions): {
|
|
47
|
+
included: true;
|
|
48
|
+
matchedPattern: MatchedPattern | null;
|
|
49
|
+
} | {
|
|
50
|
+
included: false;
|
|
51
|
+
matchedPattern: null;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Get the headers needed to make a conditional request.
|
|
55
|
+
* Uses the etag and last-modified values from the meta store.
|
|
56
|
+
* @internal
|
|
57
|
+
*/
|
|
58
|
+
export declare function getHeaders({ init, meta, id, }: {
|
|
59
|
+
/** Initial headers to include */
|
|
60
|
+
init?: RequestInit["headers"];
|
|
61
|
+
/** Meta store to get etag and last-modified values from */
|
|
62
|
+
meta: ExtendedLoaderContext["meta"];
|
|
63
|
+
id: string;
|
|
64
|
+
}): Headers;
|
|
65
|
+
/**
|
|
66
|
+
* Store the etag or last-modified headers from a response in the meta store.
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
export declare function syncHeaders({ headers, meta, id, }: {
|
|
70
|
+
/** Headers from the response */
|
|
71
|
+
headers: Headers;
|
|
72
|
+
/** Meta store to store etag and last-modified values in */
|
|
73
|
+
meta: ExtendedLoaderContext["meta"];
|
|
74
|
+
/** id string */
|
|
75
|
+
id: string;
|
|
76
|
+
}): void;
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import path, { join, basename } from "node:path";
|
|
2
|
+
import picomatch from "picomatch";
|
|
3
|
+
/**
|
|
4
|
+
* Generates a unique identifier from a file path by removing the extension
|
|
5
|
+
* @param filePath - The file path to generate ID from
|
|
6
|
+
* @return {string} The generated identifier as a string with extension removed
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export function generateId(filePath) {
|
|
10
|
+
let id = filePath;
|
|
11
|
+
// Remove file extension for ID generation
|
|
12
|
+
const lastDotIndex = id.lastIndexOf(".");
|
|
13
|
+
if (lastDotIndex > 0) {
|
|
14
|
+
id = id.substring(0, lastDotIndex);
|
|
15
|
+
}
|
|
16
|
+
return id;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Applies path mapping logic to get the final filename for a file
|
|
20
|
+
*
|
|
21
|
+
* Supports two types of path mappings:
|
|
22
|
+
* - **File mapping**: Exact file path match (e.g., 'docs/README.md' -> 'docs/overview.md')
|
|
23
|
+
* - **Folder mapping**: Folder path with trailing slash (e.g., 'docs/capabilities/' -> 'docs/')
|
|
24
|
+
*
|
|
25
|
+
* @param filePath - Original source file path
|
|
26
|
+
* @param matchedPattern - The pattern that matched this file
|
|
27
|
+
* @param options - Import options containing path mappings
|
|
28
|
+
* @returns Final filename after applying path mapping logic
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
31
|
+
export function applyRename(filePath, matchedPattern, options) {
|
|
32
|
+
if (options?.includes &&
|
|
33
|
+
matchedPattern &&
|
|
34
|
+
matchedPattern.index < options.includes.length) {
|
|
35
|
+
const includePattern = options.includes[matchedPattern.index];
|
|
36
|
+
if (includePattern.pathMappings) {
|
|
37
|
+
// First check for exact file match (current behavior - backwards compatible)
|
|
38
|
+
if (includePattern.pathMappings[filePath]) {
|
|
39
|
+
const mappingValue = includePattern.pathMappings[filePath];
|
|
40
|
+
return typeof mappingValue === "string"
|
|
41
|
+
? mappingValue
|
|
42
|
+
: mappingValue.target;
|
|
43
|
+
}
|
|
44
|
+
// Then check for folder-to-folder mappings
|
|
45
|
+
for (const [sourceFolder, mappingValue] of Object.entries(includePattern.pathMappings)) {
|
|
46
|
+
// Check if this is a folder mapping (ends with /) and file is within it
|
|
47
|
+
if (sourceFolder.endsWith("/") && filePath.startsWith(sourceFolder)) {
|
|
48
|
+
// Replace the source folder path with target folder path
|
|
49
|
+
const targetFolder = typeof mappingValue === "string"
|
|
50
|
+
? mappingValue
|
|
51
|
+
: mappingValue.target;
|
|
52
|
+
const relativePath = filePath.slice(sourceFolder.length);
|
|
53
|
+
return path.posix.join(targetFolder, relativePath);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// Return original filename if no path mapping found
|
|
59
|
+
return basename(filePath);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Generates a local file path based on the matched pattern and file path
|
|
63
|
+
* @param filePath - The original file path from the repository
|
|
64
|
+
* @param matchedPattern - The pattern that matched this file (or null if no includes specified)
|
|
65
|
+
* @param options - Import options containing includes patterns for path mapping lookups
|
|
66
|
+
* @return {string} The local file path where this content should be stored
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
export function generatePath(filePath, matchedPattern, options) {
|
|
70
|
+
if (matchedPattern) {
|
|
71
|
+
// Extract the directory part from the pattern (before any glob wildcards)
|
|
72
|
+
const pattern = matchedPattern.pattern;
|
|
73
|
+
const beforeGlob = pattern.split(/[*?{]/)[0];
|
|
74
|
+
// Remove the pattern prefix from the file path to get the relative path
|
|
75
|
+
let relativePath = filePath;
|
|
76
|
+
if (beforeGlob && filePath.startsWith(beforeGlob)) {
|
|
77
|
+
relativePath = filePath.substring(beforeGlob.length);
|
|
78
|
+
// Remove leading slash if present
|
|
79
|
+
if (relativePath.startsWith("/")) {
|
|
80
|
+
relativePath = relativePath.substring(1);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
// If no relative path remains, use just the filename
|
|
84
|
+
if (!relativePath) {
|
|
85
|
+
relativePath = basename(filePath);
|
|
86
|
+
}
|
|
87
|
+
// Apply path mapping logic
|
|
88
|
+
const finalFilename = applyRename(filePath, matchedPattern, options);
|
|
89
|
+
// Always apply path mapping if applyRename returned something different from the original basename
|
|
90
|
+
// OR if there are pathMappings configured (since empty string mappings might return same basename)
|
|
91
|
+
const hasPathMappings = options?.includes?.[matchedPattern.index]?.pathMappings &&
|
|
92
|
+
Object.keys(options.includes[matchedPattern.index].pathMappings).length >
|
|
93
|
+
0;
|
|
94
|
+
if (finalFilename !== basename(filePath) || hasPathMappings) {
|
|
95
|
+
// Check if applyRename returned a full path (contains path separators) or just a filename
|
|
96
|
+
if (finalFilename.includes("/") || finalFilename.includes("\\")) {
|
|
97
|
+
// applyRename returned a full relative path - need to extract relative part
|
|
98
|
+
// Remove the pattern prefix to get the relative path within the pattern context
|
|
99
|
+
const beforeGlob = pattern.split(/[*?{]/)[0];
|
|
100
|
+
if (beforeGlob && finalFilename.startsWith(beforeGlob)) {
|
|
101
|
+
relativePath = finalFilename.substring(beforeGlob.length);
|
|
102
|
+
// Remove leading slash if present
|
|
103
|
+
if (relativePath.startsWith("/")) {
|
|
104
|
+
relativePath = relativePath.substring(1);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
relativePath = finalFilename;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
// applyRename returned just a filename
|
|
113
|
+
// If the filename is different due to pathMapping, use it directly
|
|
114
|
+
// This handles cases where pathMappings flatten directory structures
|
|
115
|
+
relativePath = finalFilename;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return join(matchedPattern.basePath, relativePath);
|
|
119
|
+
}
|
|
120
|
+
// Should not happen since we always use includes
|
|
121
|
+
throw new Error("No matched pattern provided - includes are required");
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Checks if a file path should be included and returns the matching pattern
|
|
125
|
+
* @param filePath - The file path to check (relative to the repository root)
|
|
126
|
+
* @param options - Import options containing includes patterns
|
|
127
|
+
* @returns Object with include status and matched pattern, or null if not included
|
|
128
|
+
* @internal
|
|
129
|
+
*/
|
|
130
|
+
export function shouldIncludeFile(filePath, options) {
|
|
131
|
+
const { includes } = options;
|
|
132
|
+
// If no include patterns specified, include all files
|
|
133
|
+
if (!includes || includes.length === 0) {
|
|
134
|
+
return { included: true, matchedPattern: null };
|
|
135
|
+
}
|
|
136
|
+
// Check each include pattern to find a match
|
|
137
|
+
for (let i = 0; i < includes.length; i++) {
|
|
138
|
+
const includePattern = includes[i];
|
|
139
|
+
const matcher = picomatch(includePattern.pattern);
|
|
140
|
+
if (matcher(filePath)) {
|
|
141
|
+
return {
|
|
142
|
+
included: true,
|
|
143
|
+
matchedPattern: {
|
|
144
|
+
pattern: includePattern.pattern,
|
|
145
|
+
basePath: includePattern.basePath,
|
|
146
|
+
index: i,
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// No patterns matched
|
|
152
|
+
return { included: false, matchedPattern: null };
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Get the headers needed to make a conditional request.
|
|
156
|
+
* Uses the etag and last-modified values from the meta store.
|
|
157
|
+
* @internal
|
|
158
|
+
*/
|
|
159
|
+
export function getHeaders({ init, meta, id, }) {
|
|
160
|
+
const tag = `${id}-etag`;
|
|
161
|
+
const lastModifiedTag = `${id}-last-modified`;
|
|
162
|
+
const etag = meta.get(tag);
|
|
163
|
+
const lastModified = meta.get(lastModifiedTag);
|
|
164
|
+
const headers = new Headers(init);
|
|
165
|
+
if (etag) {
|
|
166
|
+
headers.set("If-None-Match", etag);
|
|
167
|
+
}
|
|
168
|
+
else if (lastModified) {
|
|
169
|
+
headers.set("If-Modified-Since", lastModified);
|
|
170
|
+
}
|
|
171
|
+
return headers;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Store the etag or last-modified headers from a response in the meta store.
|
|
175
|
+
* @internal
|
|
176
|
+
*/
|
|
177
|
+
export function syncHeaders({ headers, meta, id, }) {
|
|
178
|
+
const etag = headers.get("etag");
|
|
179
|
+
const lastModified = headers.get("last-modified");
|
|
180
|
+
const tag = `${id}-etag`;
|
|
181
|
+
const lastModifiedTag = `${id}-last-modified`;
|
|
182
|
+
meta.delete(tag);
|
|
183
|
+
meta.delete(lastModifiedTag);
|
|
184
|
+
if (etag) {
|
|
185
|
+
meta.set(tag, etag);
|
|
186
|
+
}
|
|
187
|
+
else if (lastModified) {
|
|
188
|
+
meta.set(lastModifiedTag, lastModified);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ImportedFile } from "./github.link-transform.js";
|
|
2
|
+
import type { ExtendedLoaderContext } from "./github.types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Ensures directory exists and writes file to disk.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
export declare function syncFile(path: string, content: string): Promise<void>;
|
|
8
|
+
/**
|
|
9
|
+
* Stores a processed file in Astro's content store
|
|
10
|
+
* @internal
|
|
11
|
+
*/
|
|
12
|
+
export declare function storeProcessedFile(file: ImportedFile, context: ExtendedLoaderContext, clear: boolean): Promise<{
|
|
13
|
+
id: string;
|
|
14
|
+
filePath: string;
|
|
15
|
+
}>;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { existsSync, promises as fs } from "node:fs";
|
|
2
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
3
|
+
/**
|
|
4
|
+
* Ensures directory exists and writes file to disk.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
export async function syncFile(path, content) {
|
|
8
|
+
const dir = path.substring(0, path.lastIndexOf("/"));
|
|
9
|
+
if (dir && !existsSync(dir)) {
|
|
10
|
+
await fs.mkdir(dir, { recursive: true });
|
|
11
|
+
}
|
|
12
|
+
await fs.writeFile(path, content, "utf-8");
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Stores a processed file in Astro's content store
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export async function storeProcessedFile(file, context, clear) {
|
|
19
|
+
const { store, generateDigest, entryTypes, logger, parseData, config } = context;
|
|
20
|
+
function configForFile(filePath) {
|
|
21
|
+
const ext = filePath.split(".").at(-1);
|
|
22
|
+
if (!ext) {
|
|
23
|
+
logger.warn(`No extension found for ${filePath}`);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
return entryTypes?.get(`.${ext}`);
|
|
27
|
+
}
|
|
28
|
+
const entryType = configForFile(file.sourcePath || "tmp.md");
|
|
29
|
+
if (!entryType)
|
|
30
|
+
throw new Error("No entry type found");
|
|
31
|
+
const fileUrl = pathToFileURL(file.targetPath);
|
|
32
|
+
const { body, data } = await entryType.getEntryInfo({
|
|
33
|
+
contents: file.content,
|
|
34
|
+
fileUrl: fileUrl,
|
|
35
|
+
});
|
|
36
|
+
// Generate digest for storage (repository-level caching handles change detection)
|
|
37
|
+
const digest = generateDigest(file.content);
|
|
38
|
+
const existingEntry = store.get(file.id);
|
|
39
|
+
if (existingEntry) {
|
|
40
|
+
logger.debug(`🔄 File ${file.id} - updating`);
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
logger.debug(`📄 File ${file.id} - adding`);
|
|
44
|
+
}
|
|
45
|
+
// Write file to disk
|
|
46
|
+
if (!existsSync(fileURLToPath(fileUrl))) {
|
|
47
|
+
logger.verbose(`Writing ${file.id} to ${fileUrl}`);
|
|
48
|
+
await syncFile(fileURLToPath(fileUrl), file.content);
|
|
49
|
+
}
|
|
50
|
+
const parsedData = await parseData({
|
|
51
|
+
id: file.id,
|
|
52
|
+
data,
|
|
53
|
+
filePath: fileUrl.toString(),
|
|
54
|
+
});
|
|
55
|
+
// When clear mode is enabled, delete the existing entry before setting the new one.
|
|
56
|
+
// This provides atomic replacement without breaking Astro's content collection,
|
|
57
|
+
// as opposed to calling store.clear() which empties everything at once.
|
|
58
|
+
if (clear && existingEntry) {
|
|
59
|
+
logger.debug(`🗑️ Clearing existing entry before replacement: ${file.id}`);
|
|
60
|
+
store.delete(file.id);
|
|
61
|
+
}
|
|
62
|
+
// Store in content store
|
|
63
|
+
if (entryType.getRenderFunction) {
|
|
64
|
+
logger.verbose(`Rendering ${file.id}`);
|
|
65
|
+
const render = await entryType.getRenderFunction(config);
|
|
66
|
+
let rendered = undefined;
|
|
67
|
+
try {
|
|
68
|
+
rendered = await render?.({
|
|
69
|
+
id: file.id,
|
|
70
|
+
data,
|
|
71
|
+
body,
|
|
72
|
+
filePath: fileUrl.toString(),
|
|
73
|
+
digest,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
logger.error(`Error rendering ${file.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
78
|
+
}
|
|
79
|
+
logger.debug(`🔍 Storing collection entry: ${file.id} (${file.sourcePath} -> ${file.targetPath})`);
|
|
80
|
+
store.set({
|
|
81
|
+
id: file.id,
|
|
82
|
+
data: parsedData,
|
|
83
|
+
body,
|
|
84
|
+
filePath: file.targetPath,
|
|
85
|
+
digest,
|
|
86
|
+
rendered,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
else if ("contentModuleTypes" in entryType) {
|
|
90
|
+
store.set({
|
|
91
|
+
id: file.id,
|
|
92
|
+
data: parsedData,
|
|
93
|
+
body,
|
|
94
|
+
filePath: file.targetPath,
|
|
95
|
+
digest,
|
|
96
|
+
deferredRender: true,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
store.set({
|
|
101
|
+
id: file.id,
|
|
102
|
+
data: parsedData,
|
|
103
|
+
body,
|
|
104
|
+
filePath: file.targetPath,
|
|
105
|
+
digest,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
return { id: file.id, filePath: file.targetPath };
|
|
109
|
+
}
|
package/dist/github.types.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { ContentEntryType } from "astro";
|
|
|
3
3
|
import type { MarkdownHeading } from "@astrojs/markdown-remark";
|
|
4
4
|
import { Octokit } from "octokit";
|
|
5
5
|
import type { LinkHandler } from "./github.link-transform.js";
|
|
6
|
-
import type { LogLevel } from "./github.logger.js";
|
|
6
|
+
import type { LogLevel, Logger } from "./github.logger.js";
|
|
7
7
|
/**
|
|
8
8
|
* Context information for link transformations
|
|
9
9
|
*/
|
|
@@ -26,7 +26,7 @@ export interface LinkMapping {
|
|
|
26
26
|
/** Pattern to match (string or regex) */
|
|
27
27
|
pattern: string | RegExp;
|
|
28
28
|
/** Replacement string or function */
|
|
29
|
-
replacement: string | ((match: string, anchor: string, context:
|
|
29
|
+
replacement: string | ((match: string, anchor: string, context: LinkTransformContext) => string);
|
|
30
30
|
/** Apply to all links, not just unresolved internal links (default: false) */
|
|
31
31
|
global?: boolean;
|
|
32
32
|
/** Function to determine if this mapping should apply to the current file context */
|
|
@@ -162,7 +162,7 @@ export type CollectionEntryOptions = {
|
|
|
162
162
|
* The LoaderContext may contain properties and methods that offer
|
|
163
163
|
* control or inspection over the loading behavior.
|
|
164
164
|
*/
|
|
165
|
-
context:
|
|
165
|
+
context: ExtendedLoaderContext;
|
|
166
166
|
/**
|
|
167
167
|
* An instance of the Octokit library, which provides a way to interact
|
|
168
168
|
* with GitHub's REST API. This variable allows you to access and perform
|
|
@@ -225,6 +225,16 @@ export interface RenderedContent {
|
|
|
225
225
|
[key: string]: unknown;
|
|
226
226
|
};
|
|
227
227
|
}
|
|
228
|
+
/**
|
|
229
|
+
* Represents a version of a library variant to display in the devportal's version picker.
|
|
230
|
+
* Versions are manually curated in the import config — no auto-discovery.
|
|
231
|
+
*/
|
|
232
|
+
export interface VersionConfig {
|
|
233
|
+
/** URL segment for this version (e.g., "latest", "v8.0.0") */
|
|
234
|
+
slug: string;
|
|
235
|
+
/** Display name for this version (e.g., "Latest", "v8.0.0") */
|
|
236
|
+
label: string;
|
|
237
|
+
}
|
|
228
238
|
/**
|
|
229
239
|
* Represents configuration options for importing content from GitHub repositories.
|
|
230
240
|
*/
|
|
@@ -286,18 +296,38 @@ export type ImportOptions = {
|
|
|
286
296
|
* @default 'default'
|
|
287
297
|
*/
|
|
288
298
|
logLevel?: LogLevel;
|
|
299
|
+
/**
|
|
300
|
+
* Language for this import variant (e.g., "TypeScript", "Python", "Go").
|
|
301
|
+
* Used for logging and passed through to the devportal for UI display.
|
|
302
|
+
*/
|
|
303
|
+
language?: string;
|
|
304
|
+
/**
|
|
305
|
+
* Versions to display in the devportal's version picker.
|
|
306
|
+
* Informational — tells the loader which version folders exist in the source content.
|
|
307
|
+
* The loader imports content as-is; the version folder structure carries through from source to destination.
|
|
308
|
+
*/
|
|
309
|
+
versions?: VersionConfig[];
|
|
289
310
|
};
|
|
290
311
|
export type FetchOptions = RequestInit & {
|
|
291
312
|
signal?: AbortSignal;
|
|
292
313
|
concurrency?: number;
|
|
293
314
|
};
|
|
294
315
|
/**
|
|
295
|
-
*
|
|
316
|
+
* Astro loader context extended with optional entry type support.
|
|
317
|
+
* Use this type when calling `.load(context as LoaderContext)` in multi-loader patterns.
|
|
296
318
|
*/
|
|
297
319
|
export interface LoaderContext extends AstroLoaderContext {
|
|
298
320
|
/** @internal */
|
|
299
321
|
entryTypes?: Map<string, ContentEntryType>;
|
|
300
322
|
}
|
|
323
|
+
/**
|
|
324
|
+
* LoaderContext with Astro's logger replaced by our Logger class.
|
|
325
|
+
* Used by internal functions that need verbose/logFileProcessing/etc.
|
|
326
|
+
* @internal
|
|
327
|
+
*/
|
|
328
|
+
export type ExtendedLoaderContext = Omit<LoaderContext, "logger"> & {
|
|
329
|
+
logger: Logger;
|
|
330
|
+
};
|
|
301
331
|
/**
|
|
302
332
|
* @internal
|
|
303
333
|
*/
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
3
|
-
export
|
|
4
|
-
export
|
|
5
|
-
export
|
|
6
|
-
export
|
|
1
|
+
export { githubLoader } from "./github.loader.js";
|
|
2
|
+
export { createAuthenticatedOctokit, createOctokitFromEnv, } from "./github.auth.js";
|
|
3
|
+
export type { GithubLoaderOptions, ImportOptions, FetchOptions, IncludePattern, PathMappingValue, EnhancedPathMapping, VersionConfig, LoaderContext, } from "./github.types.js";
|
|
4
|
+
export type { TransformFunction, TransformContext, MatchedPattern, } from "./github.types.js";
|
|
5
|
+
export type { LinkMapping, LinkTransformContext, ImportLinkTransformOptions, } from "./github.types.js";
|
|
6
|
+
export type { LinkHandler } from "./github.link-transform.js";
|
|
7
|
+
export type { GitHubAuthConfig, GitHubAppAuthConfig, GitHubPATAuthConfig, } from "./github.auth.js";
|
|
8
|
+
export type { LogLevel } from "./github.logger.js";
|