@ucdjs/pipelines-loader 0.0.1-beta.7 → 0.0.1-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -3
- package/dist/discover-C_YruvBu.d.mts +69 -0
- package/dist/discover.d.mts +2 -0
- package/dist/discover.mjs +53 -0
- package/dist/index.d.mts +112 -45
- package/dist/index.mjs +555 -133
- package/package.json +12 -11
- package/dist/gitlab-BeZb8tDi.mjs +0 -108
- package/dist/internal.d.mts +0 -31
- package/dist/internal.mjs +0 -3
- package/dist/types-C7EhTWo6.d.mts +0 -48
package/README.md
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
[![npm version][npm-version-src]][npm-version-href]
|
|
4
4
|
[![npm downloads][npm-downloads-src]][npm-downloads-href]
|
|
5
|
-
[![codecov][codecov-src]][codecov-href]
|
|
6
5
|
|
|
7
6
|
> [!IMPORTANT]
|
|
8
7
|
> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk.
|
|
@@ -23,5 +22,3 @@ Published under [MIT License](./LICENSE).
|
|
|
23
22
|
[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-loader
|
|
24
23
|
[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-loader?style=flat&colorA=18181B&colorB=4169E1
|
|
25
24
|
[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-loader
|
|
26
|
-
[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1
|
|
27
|
-
[codecov-href]: https://codecov.io/gh/ucdjs/ucd
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
//#region src/errors.d.ts
|
|
2
|
+
declare const PIPELINE_LOADER_ISSUE_CODES: readonly ["INVALID_LOCATOR", "CACHE_MISS", "REF_RESOLUTION_FAILED", "DOWNLOAD_FAILED", "MATERIALIZE_FAILED", "DISCOVERY_FAILED", "BUNDLE_RESOLVE_FAILED", "BUNDLE_TRANSFORM_FAILED", "IMPORT_FAILED", "INVALID_EXPORT"];
|
|
3
|
+
type PipelineLoaderIssueCode = (typeof PIPELINE_LOADER_ISSUE_CODES)[number];
|
|
4
|
+
type PipelineLoaderIssueScope = "locator" | "repository" | "discovery" | "file" | "bundle" | "import";
|
|
5
|
+
interface PipelineLoaderIssue {
|
|
6
|
+
code: PipelineLoaderIssueCode;
|
|
7
|
+
scope: PipelineLoaderIssueScope;
|
|
8
|
+
message: string;
|
|
9
|
+
locator?: unknown;
|
|
10
|
+
repositoryPath?: string;
|
|
11
|
+
filePath?: string;
|
|
12
|
+
relativePath?: string;
|
|
13
|
+
cause?: Error;
|
|
14
|
+
meta?: Record<string, unknown>;
|
|
15
|
+
}
|
|
16
|
+
declare class PipelineLoaderError extends Error {
|
|
17
|
+
readonly code: PipelineLoaderIssueCode;
|
|
18
|
+
constructor(code: PipelineLoaderIssueCode, message: string, options?: ErrorOptions);
|
|
19
|
+
}
|
|
20
|
+
declare class CacheMissError extends PipelineLoaderError {
|
|
21
|
+
readonly provider: string;
|
|
22
|
+
readonly owner: string;
|
|
23
|
+
readonly repo: string;
|
|
24
|
+
readonly ref: string;
|
|
25
|
+
constructor(provider: string, owner: string, repo: string, ref: string);
|
|
26
|
+
}
|
|
27
|
+
declare class BundleError extends PipelineLoaderError {
|
|
28
|
+
readonly entryPath: string;
|
|
29
|
+
constructor(entryPath: string, message: string, options?: ErrorOptions);
|
|
30
|
+
}
|
|
31
|
+
declare class BundleResolveError extends PipelineLoaderError {
|
|
32
|
+
readonly entryPath: string;
|
|
33
|
+
readonly importPath: string;
|
|
34
|
+
constructor(entryPath: string, importPath: string, options?: ErrorOptions);
|
|
35
|
+
}
|
|
36
|
+
declare class BundleTransformError extends PipelineLoaderError {
|
|
37
|
+
readonly entryPath: string;
|
|
38
|
+
readonly line?: number;
|
|
39
|
+
readonly column?: number;
|
|
40
|
+
constructor(entryPath: string, options?: ErrorOptions & {
|
|
41
|
+
line?: number;
|
|
42
|
+
column?: number;
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
//#endregion
|
|
46
|
+
//#region src/discover.d.ts
|
|
47
|
+
interface RemoteOriginMeta {
|
|
48
|
+
provider: "github" | "gitlab";
|
|
49
|
+
owner: string;
|
|
50
|
+
repo: string;
|
|
51
|
+
ref: string;
|
|
52
|
+
path?: string;
|
|
53
|
+
}
|
|
54
|
+
interface DiscoverPipelineFilesOptions {
|
|
55
|
+
repositoryPath: string;
|
|
56
|
+
patterns?: string | string[];
|
|
57
|
+
origin?: RemoteOriginMeta;
|
|
58
|
+
}
|
|
59
|
+
interface DiscoverPipelineFilesResult {
|
|
60
|
+
files: Array<{
|
|
61
|
+
filePath: string;
|
|
62
|
+
relativePath: string;
|
|
63
|
+
origin?: RemoteOriginMeta;
|
|
64
|
+
}>;
|
|
65
|
+
issues: PipelineLoaderIssue[];
|
|
66
|
+
}
|
|
67
|
+
declare function discoverPipelineFiles(options: DiscoverPipelineFilesOptions): Promise<DiscoverPipelineFilesResult>;
|
|
68
|
+
//#endregion
|
|
69
|
+
export { BundleError as a, CacheMissError as c, discoverPipelineFiles as i, PipelineLoaderError as l, DiscoverPipelineFilesResult as n, BundleResolveError as o, RemoteOriginMeta as r, BundleTransformError as s, DiscoverPipelineFilesOptions as t, PipelineLoaderIssue as u };
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { relative } from "node:path";
|
|
2
|
+
import { glob } from "tinyglobby";
|
|
3
|
+
//#region src/discover.ts
|
|
4
|
+
const TRAILING_SLASH_RE = /\/$/;
|
|
5
|
+
const BACKSLASH_RE = /\\/g;
|
|
6
|
+
function joinOriginPath(origin, relativePath) {
|
|
7
|
+
if (!origin) return void 0;
|
|
8
|
+
return {
|
|
9
|
+
...origin,
|
|
10
|
+
path: origin.path ? `${origin.path.replace(TRAILING_SLASH_RE, "")}/${relativePath}` : relativePath
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
async function discoverPipelineFiles(options) {
|
|
14
|
+
const patterns = options.patterns ? Array.isArray(options.patterns) ? options.patterns : [options.patterns] : ["**/*.ucd-pipeline.ts"];
|
|
15
|
+
try {
|
|
16
|
+
return {
|
|
17
|
+
files: (await glob(patterns, {
|
|
18
|
+
cwd: options.repositoryPath,
|
|
19
|
+
ignore: [
|
|
20
|
+
"node_modules/**",
|
|
21
|
+
"**/node_modules/**",
|
|
22
|
+
"**/dist/**",
|
|
23
|
+
"**/build/**",
|
|
24
|
+
"**/.git/**"
|
|
25
|
+
],
|
|
26
|
+
absolute: true,
|
|
27
|
+
onlyFiles: true
|
|
28
|
+
})).map((filePath) => {
|
|
29
|
+
const relativePath = relative(options.repositoryPath, filePath).replace(BACKSLASH_RE, "/");
|
|
30
|
+
return {
|
|
31
|
+
filePath,
|
|
32
|
+
relativePath,
|
|
33
|
+
origin: joinOriginPath(options.origin, relativePath)
|
|
34
|
+
};
|
|
35
|
+
}),
|
|
36
|
+
issues: []
|
|
37
|
+
};
|
|
38
|
+
} catch (err) {
|
|
39
|
+
const cause = err instanceof Error ? err : new Error(String(err));
|
|
40
|
+
return {
|
|
41
|
+
files: [],
|
|
42
|
+
issues: [{
|
|
43
|
+
code: "DISCOVERY_FAILED",
|
|
44
|
+
scope: "discovery",
|
|
45
|
+
message: cause.message,
|
|
46
|
+
repositoryPath: options.repositoryPath,
|
|
47
|
+
cause
|
|
48
|
+
}]
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//#endregion
|
|
53
|
+
export { discoverPipelineFiles };
|
package/dist/index.d.mts
CHANGED
|
@@ -1,58 +1,125 @@
|
|
|
1
|
-
import { a as
|
|
1
|
+
import { a as BundleError, c as CacheMissError, l as PipelineLoaderError, o as BundleResolveError, r as RemoteOriginMeta, s as BundleTransformError, u as PipelineLoaderIssue } from "./discover-C_YruvBu.mjs";
|
|
2
|
+
import { PipelineDefinition } from "@ucdjs/pipelines-core";
|
|
2
3
|
|
|
3
|
-
//#region src/
|
|
4
|
-
type
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
//#region src/cache.d.ts
|
|
5
|
+
type RemoteProvider = "github" | "gitlab";
|
|
6
|
+
interface RemoteCacheStatus {
|
|
7
|
+
provider: RemoteProvider;
|
|
8
|
+
owner: string;
|
|
9
|
+
repo: string;
|
|
10
|
+
ref: string;
|
|
11
|
+
commitSha: string;
|
|
12
|
+
cacheDir: string;
|
|
13
|
+
markerPath: string;
|
|
14
|
+
cached: boolean;
|
|
15
|
+
syncedAt: string | null;
|
|
16
|
+
}
|
|
17
|
+
declare function getRemoteSourceCacheStatus(input: {
|
|
18
|
+
provider: RemoteProvider;
|
|
9
19
|
owner: string;
|
|
10
20
|
repo: string;
|
|
11
21
|
ref?: string;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
22
|
+
}): Promise<RemoteCacheStatus>;
|
|
23
|
+
/**
|
|
24
|
+
* Write a cache marker for a remote source.
|
|
25
|
+
* Call this after downloading and extracting the archive.
|
|
26
|
+
*/
|
|
27
|
+
declare function writeCacheMarker(input: {
|
|
28
|
+
provider: RemoteProvider;
|
|
29
|
+
owner: string;
|
|
30
|
+
repo: string;
|
|
31
|
+
ref: string;
|
|
32
|
+
commitSha: string;
|
|
33
|
+
cacheDir: string;
|
|
34
|
+
markerPath: string;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
declare function clearRemoteSourceCache(input: {
|
|
37
|
+
provider: RemoteProvider;
|
|
15
38
|
owner: string;
|
|
16
39
|
repo: string;
|
|
17
40
|
ref?: string;
|
|
18
|
-
|
|
19
|
-
};
|
|
41
|
+
}): Promise<boolean>;
|
|
20
42
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* Supports:
|
|
24
|
-
* - Local file paths
|
|
25
|
-
* - github://owner/repo?ref=branch&path=file.ts
|
|
26
|
-
* - gitlab://owner/repo?ref=branch&path=file.ts
|
|
43
|
+
* List all cached remote sources.
|
|
27
44
|
*/
|
|
45
|
+
declare function listCachedSources(): Promise<Array<{
|
|
46
|
+
source: string;
|
|
47
|
+
owner: string;
|
|
48
|
+
repo: string;
|
|
49
|
+
ref: string;
|
|
50
|
+
commitSha: string;
|
|
51
|
+
syncedAt: string;
|
|
52
|
+
cacheDir: string;
|
|
53
|
+
}>>;
|
|
54
|
+
//#endregion
|
|
55
|
+
//#region src/loader.d.ts
|
|
56
|
+
interface LoadedPipelineFile {
|
|
57
|
+
filePath: string;
|
|
58
|
+
pipelines: PipelineDefinition[];
|
|
59
|
+
exportNames: string[];
|
|
60
|
+
}
|
|
61
|
+
interface LoadPipelinesResult {
|
|
62
|
+
pipelines: PipelineDefinition[];
|
|
63
|
+
files: LoadedPipelineFile[];
|
|
64
|
+
issues: PipelineLoaderIssue[];
|
|
65
|
+
}
|
|
28
66
|
declare function loadPipelineFile(filePath: string): Promise<LoadedPipelineFile>;
|
|
29
|
-
|
|
30
|
-
|
|
67
|
+
declare function loadPipelinesFromPaths(filePaths: string[]): Promise<LoadPipelinesResult>;
|
|
68
|
+
//#endregion
|
|
69
|
+
//#region src/materialize.d.ts
|
|
70
|
+
interface LocalPipelineLocator {
|
|
71
|
+
kind: "local";
|
|
72
|
+
path: string;
|
|
31
73
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
74
|
+
interface RemotePipelineLocator {
|
|
75
|
+
kind: "remote";
|
|
76
|
+
provider: "github" | "gitlab";
|
|
77
|
+
owner: string;
|
|
78
|
+
repo: string;
|
|
79
|
+
ref?: string;
|
|
80
|
+
path?: string;
|
|
36
81
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
82
|
+
type PipelineLocator = LocalPipelineLocator | RemotePipelineLocator;
|
|
83
|
+
interface MaterializePipelineLocatorResult {
|
|
84
|
+
repositoryPath?: string;
|
|
85
|
+
filePath?: string;
|
|
86
|
+
relativePath?: string;
|
|
87
|
+
origin?: RemoteOriginMeta;
|
|
88
|
+
issues: PipelineLoaderIssue[];
|
|
89
|
+
}
|
|
90
|
+
declare function materializePipelineLocator(locator: PipelineLocator): Promise<MaterializePipelineLocatorResult>;
|
|
91
|
+
//#endregion
|
|
92
|
+
//#region src/locator.d.ts
|
|
93
|
+
declare function parsePipelineLocator(input: string): PipelineLocator;
|
|
94
|
+
declare function parseRemoteSourceUrl(url: string): RemotePipelineLocator | null;
|
|
95
|
+
//#endregion
|
|
96
|
+
//#region src/remote.d.ts
|
|
97
|
+
interface UpdateCheckResult {
|
|
98
|
+
hasUpdate: boolean;
|
|
99
|
+
currentSha: string | null;
|
|
100
|
+
remoteSha: string;
|
|
101
|
+
error?: Error;
|
|
102
|
+
}
|
|
103
|
+
declare function checkRemoteLocatorUpdates(input: {
|
|
104
|
+
provider: "github" | "gitlab";
|
|
105
|
+
owner: string;
|
|
106
|
+
repo: string;
|
|
107
|
+
ref?: string;
|
|
108
|
+
}): Promise<UpdateCheckResult>;
|
|
109
|
+
interface SyncResult {
|
|
110
|
+
success: boolean;
|
|
111
|
+
updated: boolean;
|
|
112
|
+
previousSha: string | null;
|
|
113
|
+
newSha: string;
|
|
114
|
+
cacheDir: string;
|
|
115
|
+
error?: Error;
|
|
116
|
+
}
|
|
117
|
+
declare function ensureRemoteLocator(input: {
|
|
118
|
+
provider: "github" | "gitlab";
|
|
119
|
+
owner: string;
|
|
120
|
+
repo: string;
|
|
121
|
+
ref?: string;
|
|
122
|
+
force?: boolean;
|
|
123
|
+
}): Promise<SyncResult>;
|
|
57
124
|
//#endregion
|
|
58
|
-
export {
|
|
125
|
+
export { BundleError, BundleResolveError, BundleTransformError, CacheMissError, type LoadedPipelineFile, type LocalPipelineLocator, PipelineLoaderError, type PipelineLoaderIssue, type PipelineLocator, type RemoteCacheStatus, type RemoteOriginMeta, type RemotePipelineLocator, type SyncResult, type UpdateCheckResult, checkRemoteLocatorUpdates, clearRemoteSourceCache, ensureRemoteLocator, getRemoteSourceCacheStatus, listCachedSources, loadPipelineFile, loadPipelinesFromPaths, materializePipelineLocator, parsePipelineLocator, parseRemoteSourceUrl, writeCacheMarker };
|
package/dist/index.mjs
CHANGED
|
@@ -1,76 +1,267 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { getUcdConfigPath } from "@ucdjs/env";
|
|
3
4
|
import { isPipelineDefinition } from "@ucdjs/pipelines-core";
|
|
4
|
-
import { glob } from "tinyglobby";
|
|
5
5
|
import { build } from "rolldown";
|
|
6
|
-
|
|
7
|
-
//#region src/
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
input: entryPath,
|
|
11
|
-
write: false,
|
|
12
|
-
output: { format: "esm" }
|
|
13
|
-
});
|
|
14
|
-
const chunk = (Array.isArray(result) ? result : [result]).flatMap((output) => output.output ?? []).find((item) => item.type === "chunk");
|
|
15
|
-
if (!chunk || chunk.type !== "chunk") throw new Error("Failed to bundle module");
|
|
16
|
-
return chunk.code;
|
|
6
|
+
import { parseTarGzip } from "nanotar";
|
|
7
|
+
//#region src/cache.ts
|
|
8
|
+
function getBaseRepoCacheDir() {
|
|
9
|
+
return getUcdConfigPath("cache", "repos");
|
|
17
10
|
}
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
const CACHE_FILE_NAME = ".ucd-cache.json";
|
|
12
|
+
async function readCache(markerPath) {
|
|
13
|
+
try {
|
|
14
|
+
const raw = await readFile(markerPath, "utf8");
|
|
15
|
+
const parsed = JSON.parse(raw);
|
|
16
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
17
|
+
const marker = parsed;
|
|
18
|
+
if (typeof marker.source === "string" && typeof marker.owner === "string" && typeof marker.repo === "string" && typeof marker.ref === "string" && typeof marker.commitSha === "string" && typeof marker.syncedAt === "string") return marker;
|
|
19
|
+
return null;
|
|
20
|
+
} catch {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
async function getRemoteSourceCacheStatus(input) {
|
|
25
|
+
const { provider, owner, repo, ref = "HEAD" } = input;
|
|
26
|
+
const cacheDir = path.join(getBaseRepoCacheDir(), provider, owner, repo, ref);
|
|
27
|
+
const markerPath = path.join(cacheDir, CACHE_FILE_NAME);
|
|
28
|
+
const marker = await readCache(markerPath);
|
|
29
|
+
return {
|
|
30
|
+
provider,
|
|
31
|
+
owner,
|
|
32
|
+
repo,
|
|
33
|
+
ref,
|
|
34
|
+
commitSha: marker?.commitSha ?? "",
|
|
35
|
+
cacheDir,
|
|
36
|
+
markerPath,
|
|
37
|
+
cached: marker !== null,
|
|
38
|
+
syncedAt: marker?.syncedAt ?? null
|
|
39
|
+
};
|
|
20
40
|
}
|
|
21
|
-
|
|
22
|
-
//#endregion
|
|
23
|
-
//#region src/loader.ts
|
|
24
41
|
/**
|
|
25
|
-
*
|
|
42
|
+
* Write a cache marker for a remote source.
|
|
43
|
+
* Call this after downloading and extracting the archive.
|
|
26
44
|
*/
|
|
27
|
-
function
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
async function writeCacheMarker(input) {
|
|
46
|
+
const marker = {
|
|
47
|
+
source: input.provider,
|
|
48
|
+
owner: input.owner,
|
|
49
|
+
repo: input.repo,
|
|
50
|
+
ref: input.ref,
|
|
51
|
+
commitSha: input.commitSha,
|
|
52
|
+
syncedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
53
|
+
};
|
|
54
|
+
await writeFile(input.markerPath, JSON.stringify(marker, null, 2), "utf8");
|
|
55
|
+
}
|
|
56
|
+
async function clearRemoteSourceCache(input) {
|
|
57
|
+
const { provider, owner, repo, ref } = input;
|
|
58
|
+
const status = await getRemoteSourceCacheStatus({
|
|
59
|
+
provider,
|
|
60
|
+
owner,
|
|
61
|
+
repo,
|
|
62
|
+
ref
|
|
63
|
+
});
|
|
64
|
+
if (!status.cached) return false;
|
|
65
|
+
try {
|
|
66
|
+
await rm(status.cacheDir, {
|
|
67
|
+
recursive: true,
|
|
68
|
+
force: true
|
|
69
|
+
});
|
|
70
|
+
return true;
|
|
71
|
+
} catch {
|
|
72
|
+
return false;
|
|
47
73
|
}
|
|
48
|
-
return null;
|
|
49
74
|
}
|
|
50
75
|
/**
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* Supports:
|
|
54
|
-
* - Local file paths
|
|
55
|
-
* - github://owner/repo?ref=branch&path=file.ts
|
|
56
|
-
* - gitlab://owner/repo?ref=branch&path=file.ts
|
|
76
|
+
* List all cached remote sources.
|
|
57
77
|
*/
|
|
58
|
-
async function
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
78
|
+
async function listCachedSources() {
|
|
79
|
+
const baseDir = getBaseRepoCacheDir();
|
|
80
|
+
const results = [];
|
|
81
|
+
try {
|
|
82
|
+
const sources = await readdir(baseDir, { withFileTypes: true });
|
|
83
|
+
for (const sourceEntry of sources) {
|
|
84
|
+
if (!sourceEntry.isDirectory()) continue;
|
|
85
|
+
const source = sourceEntry.name;
|
|
86
|
+
const owners = await readdir(path.join(baseDir, source), { withFileTypes: true });
|
|
87
|
+
for (const ownerEntry of owners) {
|
|
88
|
+
if (!ownerEntry.isDirectory()) continue;
|
|
89
|
+
const owner = ownerEntry.name;
|
|
90
|
+
const repos = await readdir(path.join(baseDir, source, owner), { withFileTypes: true });
|
|
91
|
+
for (const repoEntry of repos) {
|
|
92
|
+
if (!repoEntry.isDirectory()) continue;
|
|
93
|
+
const repo = repoEntry.name;
|
|
94
|
+
const refs = await readdir(path.join(baseDir, source, owner, repo), { withFileTypes: true });
|
|
95
|
+
for (const refEntry of refs) {
|
|
96
|
+
if (!refEntry.isDirectory()) continue;
|
|
97
|
+
const ref = refEntry.name;
|
|
98
|
+
const cacheDir = path.join(baseDir, source, owner, repo, ref);
|
|
99
|
+
const marker = await readCache(path.join(cacheDir, CACHE_FILE_NAME));
|
|
100
|
+
if (marker) results.push({
|
|
101
|
+
source,
|
|
102
|
+
owner,
|
|
103
|
+
repo,
|
|
104
|
+
ref,
|
|
105
|
+
commitSha: marker.commitSha,
|
|
106
|
+
syncedAt: marker.syncedAt,
|
|
107
|
+
cacheDir
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
} catch {}
|
|
114
|
+
return results;
|
|
115
|
+
}
|
|
116
|
+
//#endregion
|
|
117
|
+
//#region src/errors.ts
|
|
118
|
+
var PipelineLoaderError = class extends Error {
|
|
119
|
+
code;
|
|
120
|
+
constructor(code, message, options) {
|
|
121
|
+
super(message, options);
|
|
122
|
+
this.name = "PipelineLoaderError";
|
|
123
|
+
this.code = code;
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
var CacheMissError = class extends PipelineLoaderError {
|
|
127
|
+
provider;
|
|
128
|
+
owner;
|
|
129
|
+
repo;
|
|
130
|
+
ref;
|
|
131
|
+
constructor(provider, owner, repo, ref) {
|
|
132
|
+
super("CACHE_MISS", `Cache miss for ${provider}:${owner}/${repo}@${ref}. Run 'ucd pipelines cache refresh --${provider} ${owner}/${repo} --ref ${ref}' to sync.`);
|
|
133
|
+
this.name = "CacheMissError";
|
|
134
|
+
this.provider = provider;
|
|
135
|
+
this.owner = owner;
|
|
136
|
+
this.repo = repo;
|
|
137
|
+
this.ref = ref;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
var BundleError = class extends PipelineLoaderError {
|
|
141
|
+
entryPath;
|
|
142
|
+
constructor(entryPath, message, options) {
|
|
143
|
+
super("IMPORT_FAILED", message, options);
|
|
144
|
+
this.name = "BundleError";
|
|
145
|
+
this.entryPath = entryPath;
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
var BundleResolveError = class extends PipelineLoaderError {
|
|
149
|
+
entryPath;
|
|
150
|
+
importPath;
|
|
151
|
+
constructor(entryPath, importPath, options) {
|
|
152
|
+
super("BUNDLE_RESOLVE_FAILED", `Cannot resolve import "${importPath}" in ${entryPath}`, options);
|
|
153
|
+
this.name = "BundleResolveError";
|
|
154
|
+
this.entryPath = entryPath;
|
|
155
|
+
this.importPath = importPath;
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
var BundleTransformError = class extends PipelineLoaderError {
|
|
159
|
+
entryPath;
|
|
160
|
+
line;
|
|
161
|
+
column;
|
|
162
|
+
constructor(entryPath, options) {
|
|
163
|
+
const { line, column, ...errorOptions } = options ?? {};
|
|
164
|
+
super("BUNDLE_TRANSFORM_FAILED", `Transform/parse error in ${entryPath}${line != null ? ` at line ${line}` : ""}`, errorOptions);
|
|
165
|
+
this.name = "BundleTransformError";
|
|
166
|
+
this.entryPath = entryPath;
|
|
167
|
+
this.line = line;
|
|
168
|
+
this.column = column;
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
function toPipelineLoaderIssue(error, filePath) {
|
|
172
|
+
if (error instanceof BundleResolveError) return {
|
|
173
|
+
code: "BUNDLE_RESOLVE_FAILED",
|
|
174
|
+
scope: "bundle",
|
|
175
|
+
message: error.message,
|
|
176
|
+
filePath,
|
|
177
|
+
cause: error,
|
|
178
|
+
meta: {
|
|
179
|
+
entryPath: error.entryPath,
|
|
180
|
+
importPath: error.importPath
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
if (error instanceof BundleTransformError) return {
|
|
184
|
+
code: "BUNDLE_TRANSFORM_FAILED",
|
|
185
|
+
scope: "bundle",
|
|
186
|
+
message: error.message,
|
|
187
|
+
filePath,
|
|
188
|
+
cause: error,
|
|
189
|
+
meta: {
|
|
190
|
+
entryPath: error.entryPath,
|
|
191
|
+
...error.line != null ? { line: error.line } : {},
|
|
192
|
+
...error.column != null ? { column: error.column } : {}
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
if (error instanceof BundleError) return {
|
|
196
|
+
code: error.code,
|
|
197
|
+
scope: "import",
|
|
198
|
+
message: error.message,
|
|
199
|
+
filePath,
|
|
200
|
+
cause: error,
|
|
201
|
+
meta: { entryPath: error.entryPath }
|
|
202
|
+
};
|
|
203
|
+
if (error instanceof PipelineLoaderError) return {
|
|
204
|
+
code: error.code,
|
|
205
|
+
scope: "import",
|
|
206
|
+
message: error.message,
|
|
207
|
+
filePath,
|
|
208
|
+
cause: error
|
|
209
|
+
};
|
|
210
|
+
return {
|
|
211
|
+
code: "IMPORT_FAILED",
|
|
212
|
+
scope: "import",
|
|
213
|
+
message: error.message,
|
|
214
|
+
filePath,
|
|
215
|
+
cause: error
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
//#endregion
|
|
219
|
+
//#region src/bundle.ts
|
|
220
|
+
const RESOLVE_ERROR_RE = /could not resolve|cannot find module/i;
|
|
221
|
+
const QUOTED_IMPORT_RE = /"([^"]+)"/;
|
|
222
|
+
const TRANSFORM_ERROR_RE = /unexpected token|syntaxerror|parse error|expected|transform/i;
|
|
223
|
+
async function bundle(options) {
|
|
224
|
+
let result;
|
|
225
|
+
try {
|
|
226
|
+
result = await build({
|
|
227
|
+
input: options.entryPath,
|
|
228
|
+
write: false,
|
|
229
|
+
output: { format: "esm" },
|
|
230
|
+
cwd: options.cwd
|
|
70
231
|
});
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
232
|
+
} catch (err) {
|
|
233
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
234
|
+
if (RESOLVE_ERROR_RE.test(msg)) {
|
|
235
|
+
const importMatch = msg.match(QUOTED_IMPORT_RE);
|
|
236
|
+
throw new BundleResolveError(options.entryPath, importMatch?.[1] ?? "", { cause: err });
|
|
237
|
+
}
|
|
238
|
+
if (TRANSFORM_ERROR_RE.test(msg)) throw new BundleTransformError(options.entryPath, { cause: err });
|
|
239
|
+
throw new BundleTransformError(options.entryPath, { cause: err });
|
|
240
|
+
}
|
|
241
|
+
const chunk = (Array.isArray(result) ? result : [result]).flatMap((output) => output.output ?? []).find((item) => item.type === "chunk");
|
|
242
|
+
if (!chunk || chunk.type !== "chunk") throw new BundleError(options.entryPath, "Failed to bundle module");
|
|
243
|
+
return {
|
|
244
|
+
code: chunk.code,
|
|
245
|
+
dataUrl: `data:text/javascript;base64,${Buffer.from(chunk.code, "utf-8").toString("base64")}`
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
//#endregion
|
|
249
|
+
//#region src/loader.ts
|
|
250
|
+
async function loadPipelineFile(filePath) {
|
|
251
|
+
const bundleResult = await bundle({
|
|
252
|
+
entryPath: filePath,
|
|
253
|
+
cwd: path.dirname(filePath)
|
|
254
|
+
});
|
|
255
|
+
let module;
|
|
256
|
+
try {
|
|
257
|
+
module = await import(
|
|
258
|
+
/* @vite-ignore */
|
|
259
|
+
bundleResult.dataUrl
|
|
260
|
+
);
|
|
261
|
+
} catch (err) {
|
|
262
|
+
const cause = err instanceof Error ? err : new Error(String(err));
|
|
263
|
+
throw new PipelineLoaderError("IMPORT_FAILED", `Failed to import ${filePath}: ${cause.message}`, { cause });
|
|
264
|
+
}
|
|
74
265
|
const pipelines = [];
|
|
75
266
|
const exportNames = [];
|
|
76
267
|
for (const [name, value] of Object.entries(module)) {
|
|
@@ -86,90 +277,321 @@ async function loadPipelineFile(filePath) {
|
|
|
86
277
|
exportNames
|
|
87
278
|
};
|
|
88
279
|
}
|
|
89
|
-
async function loadPipelinesFromPaths(filePaths
|
|
90
|
-
const
|
|
91
|
-
if (throwOnError) {
|
|
92
|
-
const wrapped = filePaths.map((filePath) => loadPipelineFile(filePath).catch((err) => {
|
|
93
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
94
|
-
throw new Error(`Failed to load pipeline file: ${filePath}`, { cause: error });
|
|
95
|
-
}));
|
|
96
|
-
const results = await Promise.all(wrapped);
|
|
97
|
-
return {
|
|
98
|
-
pipelines: results.flatMap((r) => r.pipelines),
|
|
99
|
-
files: results,
|
|
100
|
-
errors: []
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
const settled = await Promise.allSettled(filePaths.map((fp) => loadPipelineFile(fp)));
|
|
280
|
+
async function loadPipelinesFromPaths(filePaths) {
|
|
281
|
+
const settled = await Promise.allSettled(filePaths.map((filePath) => loadPipelineFile(filePath)));
|
|
104
282
|
const files = [];
|
|
105
|
-
const
|
|
106
|
-
for (const [
|
|
283
|
+
const issues = [];
|
|
284
|
+
for (const [index, result] of settled.entries()) {
|
|
107
285
|
if (result.status === "fulfilled") {
|
|
108
286
|
files.push(result.value);
|
|
109
287
|
continue;
|
|
110
288
|
}
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
filePath: filePaths[i],
|
|
114
|
-
error
|
|
115
|
-
});
|
|
289
|
+
const cause = result.reason instanceof Error ? result.reason : new Error(String(result.reason));
|
|
290
|
+
issues.push(toPipelineLoaderIssue(cause, filePaths[index]));
|
|
116
291
|
}
|
|
117
292
|
return {
|
|
118
|
-
pipelines: files.flatMap((
|
|
293
|
+
pipelines: files.flatMap((file) => file.pipelines),
|
|
119
294
|
files,
|
|
120
|
-
|
|
295
|
+
issues
|
|
121
296
|
};
|
|
122
297
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
298
|
+
//#endregion
|
|
299
|
+
//#region src/locator.ts
|
|
300
|
+
function parsePipelineLocator(input) {
|
|
301
|
+
if (input.startsWith("github://") || input.startsWith("gitlab://")) {
|
|
302
|
+
const provider = input.startsWith("github://") ? "github" : "gitlab";
|
|
303
|
+
const parsed = new URL(input.replace(`${provider}://`, "https://fake-host/"));
|
|
304
|
+
const [, owner, repo] = parsed.pathname.split("/");
|
|
305
|
+
if (!owner || !repo) throw new Error(`Invalid ${provider} locator: ${input}`);
|
|
306
|
+
return {
|
|
307
|
+
kind: "remote",
|
|
308
|
+
provider,
|
|
309
|
+
owner,
|
|
310
|
+
repo,
|
|
311
|
+
ref: parsed.searchParams.get("ref") ?? "HEAD",
|
|
312
|
+
path: parsed.searchParams.get("path") ?? void 0
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
kind: "local",
|
|
317
|
+
path: input
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function parseRemoteSourceUrl(url) {
|
|
321
|
+
try {
|
|
322
|
+
const locator = parsePipelineLocator(url);
|
|
323
|
+
return locator.kind === "remote" ? locator : null;
|
|
324
|
+
} catch {
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
//#endregion
|
|
329
|
+
//#region src/materialize.ts
|
|
330
|
+
const BACKSLASH_RE = /\\/g;
|
|
331
|
+
function createRemoteOrigin(locator) {
|
|
332
|
+
return {
|
|
333
|
+
provider: locator.provider,
|
|
334
|
+
owner: locator.owner,
|
|
335
|
+
repo: locator.repo,
|
|
336
|
+
ref: locator.ref ?? "HEAD",
|
|
337
|
+
...locator.path ? { path: locator.path } : {}
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
async function detectPathKind(targetPath) {
|
|
341
|
+
return stat(targetPath).then((value) => value.isFile() ? "file" : value.isDirectory() ? "directory" : null).catch(() => null);
|
|
342
|
+
}
|
|
343
|
+
async function materializePipelineLocator(locator) {
|
|
344
|
+
if (locator.kind === "local") {
|
|
345
|
+
const absolutePath = path.resolve(locator.path);
|
|
346
|
+
const pathKind = await detectPathKind(absolutePath);
|
|
347
|
+
if (!pathKind) return { issues: [{
|
|
348
|
+
code: "MATERIALIZE_FAILED",
|
|
349
|
+
scope: "file",
|
|
350
|
+
message: `Path does not exist: ${absolutePath}`,
|
|
351
|
+
locator,
|
|
352
|
+
meta: { path: absolutePath }
|
|
353
|
+
}] };
|
|
354
|
+
if (pathKind === "directory") return {
|
|
355
|
+
repositoryPath: absolutePath,
|
|
356
|
+
issues: []
|
|
357
|
+
};
|
|
358
|
+
return {
|
|
359
|
+
repositoryPath: path.dirname(absolutePath),
|
|
360
|
+
filePath: absolutePath,
|
|
361
|
+
relativePath: path.basename(absolutePath),
|
|
362
|
+
issues: []
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
const ref = locator.ref ?? "HEAD";
|
|
366
|
+
const status = await getRemoteSourceCacheStatus({
|
|
367
|
+
provider: locator.provider,
|
|
368
|
+
owner: locator.owner,
|
|
369
|
+
repo: locator.repo,
|
|
370
|
+
ref
|
|
371
|
+
});
|
|
372
|
+
if (!status.cached) return { issues: [{
|
|
373
|
+
code: "CACHE_MISS",
|
|
374
|
+
scope: "repository",
|
|
375
|
+
message: `Remote repository ${locator.owner}/${locator.repo}@${ref} is not materialized in cache.`,
|
|
376
|
+
locator,
|
|
377
|
+
meta: {
|
|
378
|
+
provider: locator.provider,
|
|
379
|
+
owner: locator.owner,
|
|
380
|
+
repo: locator.repo,
|
|
381
|
+
ref
|
|
382
|
+
}
|
|
383
|
+
}] };
|
|
384
|
+
const origin = createRemoteOrigin(locator);
|
|
385
|
+
if (!locator.path) return {
|
|
386
|
+
repositoryPath: status.cacheDir,
|
|
387
|
+
origin,
|
|
388
|
+
issues: []
|
|
389
|
+
};
|
|
390
|
+
const targetPath = path.resolve(status.cacheDir, locator.path);
|
|
391
|
+
if (targetPath !== status.cacheDir && !targetPath.startsWith(`${status.cacheDir}${path.sep}`)) return { issues: [{
|
|
392
|
+
code: "INVALID_LOCATOR",
|
|
393
|
+
scope: "locator",
|
|
394
|
+
message: `Remote path resolves outside materialized repository: ${locator.path}`,
|
|
395
|
+
locator,
|
|
396
|
+
repositoryPath: status.cacheDir
|
|
397
|
+
}] };
|
|
398
|
+
const pathKind = await detectPathKind(targetPath);
|
|
399
|
+
if (!pathKind) return { issues: [{
|
|
400
|
+
code: "MATERIALIZE_FAILED",
|
|
401
|
+
scope: "file",
|
|
402
|
+
message: `Materialized path does not exist: ${locator.path}`,
|
|
403
|
+
locator,
|
|
404
|
+
repositoryPath: status.cacheDir,
|
|
405
|
+
meta: { path: locator.path }
|
|
406
|
+
}] };
|
|
407
|
+
if (pathKind === "directory") return {
|
|
408
|
+
repositoryPath: targetPath,
|
|
409
|
+
relativePath: locator.path.replace(BACKSLASH_RE, "/"),
|
|
410
|
+
origin,
|
|
411
|
+
issues: []
|
|
412
|
+
};
|
|
413
|
+
return {
|
|
414
|
+
repositoryPath: path.dirname(targetPath),
|
|
415
|
+
filePath: targetPath,
|
|
416
|
+
relativePath: path.basename(targetPath),
|
|
417
|
+
origin,
|
|
418
|
+
issues: []
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
//#endregion
|
|
422
|
+
//#region src/adapters/github.ts
|
|
423
|
+
const GITHUB_API_BASE = "https://api.github.com";
|
|
424
|
+
const GITHUB_ACCEPT_HEADER = "application/vnd.github.v3+json";
|
|
425
|
+
async function resolveGitHubRef(ref) {
|
|
426
|
+
const { owner, repo, ref: refValue = "HEAD" } = ref;
|
|
427
|
+
const response = await fetch(`${GITHUB_API_BASE}/repos/${owner}/${repo}/commits/${refValue}`, { headers: { Accept: GITHUB_ACCEPT_HEADER } });
|
|
428
|
+
if (!response.ok) throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
|
|
429
|
+
const data = await response.json();
|
|
430
|
+
if (!data || typeof data !== "object" || !("sha" in data) || typeof data.sha !== "string") throw new Error("GitHub API error: invalid response format (missing 'sha')");
|
|
431
|
+
return data.sha;
|
|
432
|
+
}
|
|
433
|
+
async function downloadGitHubArchive(ref) {
|
|
434
|
+
const { owner, repo, commitSha } = ref;
|
|
435
|
+
const archiveUrl = `${GITHUB_API_BASE}/repos/${owner}/${repo}/tarball/${commitSha}`;
|
|
436
|
+
const response = await fetch(archiveUrl, { headers: { Accept: "application/vnd.github.v3+json" } });
|
|
437
|
+
if (!response.ok) throw new Error(`Failed to download GitHub archive: ${response.status} ${response.statusText}`);
|
|
438
|
+
return response.arrayBuffer();
|
|
439
|
+
}
|
|
440
|
+
//#endregion
|
|
441
|
+
//#region src/adapters/gitlab.ts
|
|
442
|
+
const GITLAB_API_BASE = "https://gitlab.com/api/v4";
|
|
443
|
+
async function resolveGitLabRef(ref) {
|
|
444
|
+
const { owner, repo, ref: refValue = "HEAD" } = ref;
|
|
445
|
+
const url = `${GITLAB_API_BASE}/projects/${encodeURIComponent(`${owner}/${repo}`)}/repository/commits/${refValue}`;
|
|
446
|
+
const response = await fetch(url);
|
|
447
|
+
if (!response.ok) throw new Error(`GitLab API error: ${response.status} ${response.statusText}`);
|
|
448
|
+
const data = await response.json();
|
|
449
|
+
if (!data || typeof data !== "object" || !("id" in data) || typeof data.id !== "string") throw new Error("GitLab API error: invalid response format (missing 'id')");
|
|
450
|
+
return data.id;
|
|
451
|
+
}
|
|
452
|
+
async function downloadGitLabArchive(ref) {
|
|
453
|
+
const { owner, repo, commitSha } = ref;
|
|
454
|
+
const archiveUrl = `${GITLAB_API_BASE}/projects/${encodeURIComponent(`${owner}/${repo}`)}/repository/archive.tar.gz?sha=${commitSha}`;
|
|
455
|
+
const response = await fetch(archiveUrl);
|
|
456
|
+
if (!response.ok) throw new Error(`Failed to download GitLab archive: ${response.status} ${response.statusText}`);
|
|
457
|
+
return response.arrayBuffer();
|
|
458
|
+
}
|
|
459
|
+
//#endregion
|
|
460
|
+
//#region src/remote.ts
|
|
461
|
+
const LEADING_PATH_SEPARATORS_RE = /^([/\\])+/;
|
|
462
|
+
async function extractArchiveToDirectory(archiveBuffer, targetDir) {
|
|
463
|
+
const files = await parseTarGzip(archiveBuffer);
|
|
464
|
+
const rootPrefix = files[0]?.name.split("/")[0];
|
|
465
|
+
if (!rootPrefix) throw new Error("Invalid archive: no files found");
|
|
466
|
+
for (const file of files) {
|
|
467
|
+
if (file.type === "directory" || !file.data) continue;
|
|
468
|
+
const relativePath = file.name.slice(rootPrefix.length + 1);
|
|
469
|
+
if (!relativePath) continue;
|
|
470
|
+
let safeRelativePath = path.normalize(relativePath);
|
|
471
|
+
safeRelativePath = safeRelativePath.replace(LEADING_PATH_SEPARATORS_RE, "");
|
|
472
|
+
if (!safeRelativePath) continue;
|
|
473
|
+
const upSegment = `..${path.sep}`;
|
|
474
|
+
if (safeRelativePath === ".." || safeRelativePath.startsWith(upSegment) || safeRelativePath.includes(`${path.sep}..${path.sep}`) || safeRelativePath.endsWith(`${path.sep}..`)) throw new Error(`Invalid archive entry path (path traversal detected): ${file.name}`);
|
|
475
|
+
const outputPath = path.join(targetDir, safeRelativePath);
|
|
476
|
+
const resolvedTargetDir = path.resolve(targetDir);
|
|
477
|
+
const resolvedOutputPath = path.resolve(outputPath);
|
|
478
|
+
if (resolvedOutputPath !== resolvedTargetDir && !resolvedOutputPath.startsWith(resolvedTargetDir + path.sep)) throw new Error(`Invalid archive entry path (outside target dir): ${file.name}`);
|
|
479
|
+
await mkdir(path.dirname(resolvedOutputPath), { recursive: true });
|
|
480
|
+
await writeFile(resolvedOutputPath, file.data);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
async function resolveRemoteLocatorRef(provider, ref) {
|
|
484
|
+
if (provider === "github") return resolveGitHubRef(ref);
|
|
485
|
+
return resolveGitLabRef(ref);
|
|
486
|
+
}
|
|
487
|
+
async function downloadRemoteLocatorArchive(provider, ref) {
|
|
488
|
+
if (provider === "github") return downloadGitHubArchive(ref);
|
|
489
|
+
return downloadGitLabArchive(ref);
|
|
490
|
+
}
|
|
491
|
+
async function extractArchiveToCleanDirectory(archiveBuffer, targetDir) {
|
|
492
|
+
await rm(targetDir, {
|
|
493
|
+
recursive: true,
|
|
494
|
+
force: true
|
|
495
|
+
});
|
|
496
|
+
await mkdir(targetDir, { recursive: true });
|
|
497
|
+
try {
|
|
498
|
+
await extractArchiveToDirectory(archiveBuffer, targetDir);
|
|
499
|
+
} catch (err) {
|
|
500
|
+
await rm(targetDir, {
|
|
501
|
+
recursive: true,
|
|
502
|
+
force: true
|
|
153
503
|
});
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
504
|
+
throw err;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
async function checkRemoteLocatorUpdates(input) {
|
|
508
|
+
const { provider, owner, repo, ref = "HEAD" } = input;
|
|
509
|
+
const status = await getRemoteSourceCacheStatus({
|
|
510
|
+
provider,
|
|
511
|
+
owner,
|
|
512
|
+
repo,
|
|
513
|
+
ref
|
|
514
|
+
});
|
|
515
|
+
const currentSha = status.cached ? status.commitSha : null;
|
|
516
|
+
try {
|
|
517
|
+
const remoteSha = await resolveRemoteLocatorRef(provider, {
|
|
518
|
+
owner,
|
|
519
|
+
repo,
|
|
520
|
+
ref
|
|
158
521
|
});
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
522
|
+
return {
|
|
523
|
+
hasUpdate: currentSha !== remoteSha,
|
|
524
|
+
currentSha,
|
|
525
|
+
remoteSha
|
|
526
|
+
};
|
|
527
|
+
} catch (err) {
|
|
528
|
+
return {
|
|
529
|
+
hasUpdate: false,
|
|
530
|
+
currentSha,
|
|
531
|
+
remoteSha: "",
|
|
532
|
+
error: err instanceof Error ? err : new Error(String(err))
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async function ensureRemoteLocator(input) {
|
|
537
|
+
const { provider, owner, repo, ref = "HEAD", force = false } = input;
|
|
538
|
+
const status = await getRemoteSourceCacheStatus({
|
|
539
|
+
provider,
|
|
540
|
+
owner,
|
|
541
|
+
repo,
|
|
542
|
+
ref
|
|
171
543
|
});
|
|
544
|
+
const previousSha = status.cached ? status.commitSha : null;
|
|
545
|
+
try {
|
|
546
|
+
const commitSha = await resolveRemoteLocatorRef(provider, {
|
|
547
|
+
owner,
|
|
548
|
+
repo,
|
|
549
|
+
ref
|
|
550
|
+
});
|
|
551
|
+
if (!force && status.cached && status.commitSha === commitSha) return {
|
|
552
|
+
success: true,
|
|
553
|
+
updated: false,
|
|
554
|
+
previousSha,
|
|
555
|
+
newSha: commitSha,
|
|
556
|
+
cacheDir: status.cacheDir
|
|
557
|
+
};
|
|
558
|
+
const archiveBuffer = await downloadRemoteLocatorArchive(provider, {
|
|
559
|
+
owner,
|
|
560
|
+
repo,
|
|
561
|
+
ref,
|
|
562
|
+
commitSha
|
|
563
|
+
});
|
|
564
|
+
await rm(status.cacheDir, {
|
|
565
|
+
recursive: true,
|
|
566
|
+
force: true
|
|
567
|
+
});
|
|
568
|
+
await extractArchiveToCleanDirectory(archiveBuffer, status.cacheDir);
|
|
569
|
+
await writeCacheMarker({
|
|
570
|
+
provider,
|
|
571
|
+
owner,
|
|
572
|
+
repo,
|
|
573
|
+
ref,
|
|
574
|
+
commitSha,
|
|
575
|
+
cacheDir: status.cacheDir,
|
|
576
|
+
markerPath: status.markerPath
|
|
577
|
+
});
|
|
578
|
+
return {
|
|
579
|
+
success: true,
|
|
580
|
+
updated: true,
|
|
581
|
+
previousSha,
|
|
582
|
+
newSha: commitSha,
|
|
583
|
+
cacheDir: status.cacheDir
|
|
584
|
+
};
|
|
585
|
+
} catch (err) {
|
|
586
|
+
return {
|
|
587
|
+
success: false,
|
|
588
|
+
updated: false,
|
|
589
|
+
previousSha,
|
|
590
|
+
newSha: "",
|
|
591
|
+
cacheDir: status.cacheDir,
|
|
592
|
+
error: err instanceof Error ? err : new Error(String(err))
|
|
593
|
+
};
|
|
594
|
+
}
|
|
172
595
|
}
|
|
173
|
-
|
|
174
596
|
//#endregion
|
|
175
|
-
export {
|
|
597
|
+
export { BundleError, BundleResolveError, BundleTransformError, CacheMissError, PipelineLoaderError, checkRemoteLocatorUpdates, clearRemoteSourceCache, ensureRemoteLocator, getRemoteSourceCacheStatus, listCachedSources, loadPipelineFile, loadPipelinesFromPaths, materializePipelineLocator, parsePipelineLocator, parseRemoteSourceUrl, writeCacheMarker };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ucdjs/pipelines-loader",
|
|
3
|
-
"version": "0.0.1-beta.
|
|
3
|
+
"version": "0.0.1-beta.8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Lucas Nørgård",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
},
|
|
20
20
|
"exports": {
|
|
21
21
|
".": "./dist/index.mjs",
|
|
22
|
-
"./
|
|
22
|
+
"./discover": "./dist/discover.mjs",
|
|
23
23
|
"./package.json": "./package.json"
|
|
24
24
|
},
|
|
25
25
|
"types": "./dist/index.d.mts",
|
|
@@ -31,18 +31,19 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"nanotar": "0.3.0",
|
|
34
|
-
"rolldown": "1.0.0-rc.
|
|
34
|
+
"rolldown": "1.0.0-rc.11",
|
|
35
35
|
"tinyglobby": "0.2.15",
|
|
36
|
-
"@ucdjs/
|
|
37
|
-
"@ucdjs-
|
|
36
|
+
"@ucdjs/env": "0.1.1-beta.9",
|
|
37
|
+
"@ucdjs/pipelines-core": "0.0.1-beta.8",
|
|
38
|
+
"@ucdjs-internal/shared": "0.1.1-beta.8"
|
|
38
39
|
},
|
|
39
40
|
"devDependencies": {
|
|
40
|
-
"@luxass/eslint-config": "7.
|
|
41
|
-
"eslint": "10.0
|
|
42
|
-
"publint": "0.3.
|
|
43
|
-
"tsdown": "0.
|
|
44
|
-
"typescript": "
|
|
45
|
-
"vitest-testdirs": "4.4.
|
|
41
|
+
"@luxass/eslint-config": "7.4.1",
|
|
42
|
+
"eslint": "10.1.0",
|
|
43
|
+
"publint": "0.3.18",
|
|
44
|
+
"tsdown": "0.21.4",
|
|
45
|
+
"typescript": "6.0.2",
|
|
46
|
+
"vitest-testdirs": "4.4.3",
|
|
46
47
|
"@ucdjs-tooling/tsconfig": "1.0.0",
|
|
47
48
|
"@ucdjs-tooling/tsdown-config": "1.0.0"
|
|
48
49
|
},
|
package/dist/gitlab-BeZb8tDi.mjs
DELETED
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import path from "node:path";
|
|
2
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
|
-
import { getRepositoryCacheDir } from "@ucdjs-internal/shared/config";
|
|
4
|
-
import { parseTarGzip } from "nanotar";
|
|
5
|
-
|
|
6
|
-
//#region src/cache/github.ts
|
|
7
|
-
const GITHUB_API_BASE = "https://api.github.com";
|
|
8
|
-
const GITHUB_ACCEPT_HEADER = "application/vnd.github.v3+json";
|
|
9
|
-
/**
|
|
10
|
-
* Resolve a ref (branch/tag) to a commit SHA
|
|
11
|
-
*/
|
|
12
|
-
async function resolveGitHubRef(repoRef, options = {}) {
|
|
13
|
-
const { owner, repo, ref = "HEAD" } = repoRef;
|
|
14
|
-
const { customFetch = fetch } = options;
|
|
15
|
-
const response = await customFetch(`${GITHUB_API_BASE}/repos/${owner}/${repo}/commits/${ref}`, { headers: { Accept: GITHUB_ACCEPT_HEADER } });
|
|
16
|
-
if (!response.ok) throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
|
|
17
|
-
return (await response.json()).sha;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Download and extract a GitHub repository archive
|
|
21
|
-
*/
|
|
22
|
-
async function downloadGitHubRepo(repoRef, options = {}) {
|
|
23
|
-
const { owner, repo } = repoRef;
|
|
24
|
-
const { customFetch = fetch } = options;
|
|
25
|
-
const commitSha = await resolveGitHubRef(repoRef, options);
|
|
26
|
-
const cacheDir = getRepositoryCacheDir("github", owner, repo, commitSha);
|
|
27
|
-
try {
|
|
28
|
-
if ((await import("node:fs")).existsSync(cacheDir)) return cacheDir;
|
|
29
|
-
} catch {}
|
|
30
|
-
const response = await customFetch(`${GITHUB_API_BASE}/repos/${owner}/${repo}/tarball/${commitSha}`, { headers: { Accept: "application/vnd.github.v3+json" } });
|
|
31
|
-
if (!response.ok) throw new Error(`Failed to download GitHub archive: ${response.status} ${response.statusText}`);
|
|
32
|
-
await mkdir(cacheDir, { recursive: true });
|
|
33
|
-
const files = await parseTarGzip(await response.arrayBuffer());
|
|
34
|
-
const rootPrefix = files[0]?.name.split("/")[0];
|
|
35
|
-
if (!rootPrefix) throw new Error("Invalid archive: no files found");
|
|
36
|
-
for (const file of files) {
|
|
37
|
-
if (file.type === "directory" || !file.data) continue;
|
|
38
|
-
const relativePath = file.name.slice(rootPrefix.length + 1);
|
|
39
|
-
if (!relativePath) continue;
|
|
40
|
-
let safeRelativePath = path.normalize(relativePath);
|
|
41
|
-
safeRelativePath = safeRelativePath.replace(/^([/\\])+/, "");
|
|
42
|
-
if (!safeRelativePath) continue;
|
|
43
|
-
const upSegment = `..${path.sep}`;
|
|
44
|
-
if (safeRelativePath === ".." || safeRelativePath.startsWith(upSegment) || safeRelativePath.includes(`${path.sep}..${path.sep}`) || safeRelativePath.endsWith(`${path.sep}..`)) throw new Error(`Invalid archive entry path (path traversal detected): ${file.name}`);
|
|
45
|
-
const outputPath = path.join(cacheDir, safeRelativePath);
|
|
46
|
-
const resolvedCacheDir = path.resolve(cacheDir);
|
|
47
|
-
const resolvedOutputPath = path.resolve(outputPath);
|
|
48
|
-
if (resolvedOutputPath !== resolvedCacheDir && !resolvedOutputPath.startsWith(resolvedCacheDir + path.sep)) throw new Error(`Invalid archive entry path (outside cache dir): ${file.name}`);
|
|
49
|
-
await mkdir(path.dirname(resolvedOutputPath), { recursive: true });
|
|
50
|
-
await writeFile(resolvedOutputPath, file.data);
|
|
51
|
-
}
|
|
52
|
-
return cacheDir;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
//#endregion
|
|
56
|
-
//#region src/cache/gitlab.ts
|
|
57
|
-
const GITLAB_API_BASE = "https://gitlab.com/api/v4";
|
|
58
|
-
function encodeProjectPath(owner, repo) {
|
|
59
|
-
return encodeURIComponent(`${owner}/${repo}`);
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Resolve a ref (branch/tag) to a commit SHA
|
|
63
|
-
*/
|
|
64
|
-
async function resolveGitLabRef(repoRef, options = {}) {
|
|
65
|
-
const { owner, repo, ref = "HEAD" } = repoRef;
|
|
66
|
-
const { customFetch = fetch } = options;
|
|
67
|
-
const response = await customFetch(`${GITLAB_API_BASE}/projects/${encodeProjectPath(owner, repo)}/repository/commits/${ref === "HEAD" ? "HEAD" : ref}`);
|
|
68
|
-
if (!response.ok) throw new Error(`GitLab API error: ${response.status} ${response.statusText}`);
|
|
69
|
-
return (await response.json()).id;
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Download and extract a GitLab repository archive
|
|
73
|
-
*/
|
|
74
|
-
async function downloadGitLabRepo(repoRef, options = {}) {
|
|
75
|
-
const { owner, repo } = repoRef;
|
|
76
|
-
const { customFetch = fetch } = options;
|
|
77
|
-
const commitSha = await resolveGitLabRef(repoRef, options);
|
|
78
|
-
const cacheDir = getRepositoryCacheDir("gitlab", owner, repo, commitSha);
|
|
79
|
-
try {
|
|
80
|
-
if ((await import("node:fs")).existsSync(cacheDir)) return cacheDir;
|
|
81
|
-
} catch {}
|
|
82
|
-
const response = await customFetch(`${GITLAB_API_BASE}/projects/${encodeProjectPath(owner, repo)}/repository/archive.tar.gz?sha=${commitSha}`);
|
|
83
|
-
if (!response.ok) throw new Error(`Failed to download GitLab archive: ${response.status} ${response.statusText}`);
|
|
84
|
-
await mkdir(cacheDir, { recursive: true });
|
|
85
|
-
const files = await parseTarGzip(await response.arrayBuffer());
|
|
86
|
-
const rootPrefix = files[0]?.name.split("/")[0];
|
|
87
|
-
if (!rootPrefix) throw new Error("Invalid archive: no files found");
|
|
88
|
-
for (const file of files) {
|
|
89
|
-
if (file.type === "directory" || !file.data) continue;
|
|
90
|
-
const relativePath = file.name.slice(rootPrefix.length + 1);
|
|
91
|
-
if (!relativePath) continue;
|
|
92
|
-
let safeRelativePath = path.normalize(relativePath);
|
|
93
|
-
safeRelativePath = safeRelativePath.replace(/^([/\\])+/, "");
|
|
94
|
-
if (!safeRelativePath) continue;
|
|
95
|
-
const upSegment = `..${path.sep}`;
|
|
96
|
-
if (safeRelativePath === ".." || safeRelativePath.startsWith(upSegment) || safeRelativePath.includes(`${path.sep}..${path.sep}`) || safeRelativePath.endsWith(`${path.sep}..`)) throw new Error(`Invalid archive entry path (path traversal detected): ${file.name}`);
|
|
97
|
-
const outputPath = path.join(cacheDir, safeRelativePath);
|
|
98
|
-
const resolvedCacheDir = path.resolve(cacheDir);
|
|
99
|
-
const resolvedOutputPath = path.resolve(outputPath);
|
|
100
|
-
if (resolvedOutputPath !== resolvedCacheDir && !resolvedOutputPath.startsWith(resolvedCacheDir + path.sep)) throw new Error(`Invalid archive entry path (outside cache dir): ${file.name}`);
|
|
101
|
-
await mkdir(path.dirname(resolvedOutputPath), { recursive: true });
|
|
102
|
-
await writeFile(resolvedOutputPath, file.data);
|
|
103
|
-
}
|
|
104
|
-
return cacheDir;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
//#endregion
|
|
108
|
-
export { downloadGitHubRepo as n, downloadGitLabRepo as t };
|
package/dist/internal.d.mts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import { l as RemoteRequestOptions } from "./types-C7EhTWo6.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/cache/github.d.ts
|
|
4
|
-
interface GitHubRepoRef {
|
|
5
|
-
owner: string;
|
|
6
|
-
repo: string;
|
|
7
|
-
ref?: string;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* Resolve a ref (branch/tag) to a commit SHA
|
|
11
|
-
*/
|
|
12
|
-
/**
|
|
13
|
-
* Download and extract a GitHub repository archive
|
|
14
|
-
*/
|
|
15
|
-
declare function downloadGitHubRepo(repoRef: GitHubRepoRef, options?: RemoteRequestOptions): Promise<string>;
|
|
16
|
-
//#endregion
|
|
17
|
-
//#region src/cache/gitlab.d.ts
|
|
18
|
-
interface GitLabRepoRef {
|
|
19
|
-
owner: string;
|
|
20
|
-
repo: string;
|
|
21
|
-
ref?: string;
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Resolve a ref (branch/tag) to a commit SHA
|
|
25
|
-
*/
|
|
26
|
-
/**
|
|
27
|
-
* Download and extract a GitLab repository archive
|
|
28
|
-
*/
|
|
29
|
-
declare function downloadGitLabRepo(repoRef: GitLabRepoRef, options?: RemoteRequestOptions): Promise<string>;
|
|
30
|
-
//#endregion
|
|
31
|
-
export { downloadGitHubRepo, downloadGitLabRepo };
|
package/dist/internal.mjs
DELETED
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import { PipelineDefinition } from "@ucdjs/pipelines-core";
|
|
2
|
-
|
|
3
|
-
//#region src/types.d.ts
|
|
4
|
-
interface LoadedPipelineFile {
|
|
5
|
-
filePath: string;
|
|
6
|
-
pipelines: PipelineDefinition[];
|
|
7
|
-
exportNames: string[];
|
|
8
|
-
}
|
|
9
|
-
interface LoadPipelinesResult {
|
|
10
|
-
pipelines: PipelineDefinition[];
|
|
11
|
-
files: LoadedPipelineFile[];
|
|
12
|
-
errors: PipelineLoadError[];
|
|
13
|
-
}
|
|
14
|
-
interface PipelineLoadError {
|
|
15
|
-
filePath: string;
|
|
16
|
-
error: Error;
|
|
17
|
-
}
|
|
18
|
-
interface GitHubSource {
|
|
19
|
-
type: "github";
|
|
20
|
-
id: string;
|
|
21
|
-
owner: string;
|
|
22
|
-
repo: string;
|
|
23
|
-
ref?: string;
|
|
24
|
-
path?: string;
|
|
25
|
-
}
|
|
26
|
-
interface GitLabSource {
|
|
27
|
-
type: "gitlab";
|
|
28
|
-
id: string;
|
|
29
|
-
owner: string;
|
|
30
|
-
repo: string;
|
|
31
|
-
ref?: string;
|
|
32
|
-
path?: string;
|
|
33
|
-
}
|
|
34
|
-
interface LocalSource {
|
|
35
|
-
type: "local";
|
|
36
|
-
id: string;
|
|
37
|
-
cwd: string;
|
|
38
|
-
}
|
|
39
|
-
type PipelineSource = LocalSource | GitHubSource | GitLabSource;
|
|
40
|
-
interface RemoteFileList {
|
|
41
|
-
files: string[];
|
|
42
|
-
truncated: boolean;
|
|
43
|
-
}
|
|
44
|
-
interface RemoteRequestOptions {
|
|
45
|
-
customFetch?: typeof fetch;
|
|
46
|
-
}
|
|
47
|
-
//#endregion
|
|
48
|
-
export { LocalSource as a, RemoteFileList as c, LoadedPipelineFile as i, RemoteRequestOptions as l, GitLabSource as n, PipelineLoadError as o, LoadPipelinesResult as r, PipelineSource as s, GitHubSource as t };
|