@ucdjs/pipelines-loader 0.0.1-beta.6 → 0.0.1-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -3
- package/dist/discover-C_YruvBu.d.mts +69 -0
- package/dist/discover.d.mts +2 -0
- package/dist/discover.mjs +53 -0
- package/dist/index.d.mts +119 -10
- package/dist/index.mjs +572 -50
- package/package.json +13 -15
- package/dist/chunk-DQk6qfdC.mjs +0 -18
- package/dist/insecure-lzOoh_sk.mjs +0 -346
- package/dist/insecure.d.mts +0 -10
- package/dist/insecure.mjs +0 -3
- package/dist/remote-PQ_FXnt1.d.mts +0 -46
- package/dist/remote.d.mts +0 -3
- package/dist/remote.mjs +0 -76
- package/dist/types-Br8gGmsN.d.mts +0 -41
package/dist/index.mjs
CHANGED
|
@@ -1,17 +1,275 @@
|
|
|
1
|
-
import "
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { getUcdConfigPath } from "@ucdjs/env";
|
|
4
4
|
import { isPipelineDefinition } from "@ucdjs/pipelines-core";
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import { build } from "rolldown";
|
|
6
|
+
import { parseTarGzip } from "nanotar";
|
|
7
|
+
//#region src/cache.ts
|
|
8
|
+
function getBaseRepoCacheDir() {
|
|
9
|
+
return getUcdConfigPath("cache", "repos");
|
|
10
|
+
}
|
|
11
|
+
const CACHE_FILE_NAME = ".ucd-cache.json";
|
|
12
|
+
async function readCache(markerPath) {
|
|
13
|
+
try {
|
|
14
|
+
const raw = await readFile(markerPath, "utf8");
|
|
15
|
+
const parsed = JSON.parse(raw);
|
|
16
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
17
|
+
const marker = parsed;
|
|
18
|
+
if (typeof marker.source === "string" && typeof marker.owner === "string" && typeof marker.repo === "string" && typeof marker.ref === "string" && typeof marker.commitSha === "string" && typeof marker.syncedAt === "string") return marker;
|
|
19
|
+
return null;
|
|
20
|
+
} catch {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
async function getRemoteSourceCacheStatus(input) {
|
|
25
|
+
const { provider, owner, repo, ref = "HEAD" } = input;
|
|
26
|
+
const cacheDir = path.join(getBaseRepoCacheDir(), provider, owner, repo, ref);
|
|
27
|
+
const markerPath = path.join(cacheDir, CACHE_FILE_NAME);
|
|
28
|
+
const marker = await readCache(markerPath);
|
|
29
|
+
return {
|
|
30
|
+
provider,
|
|
31
|
+
owner,
|
|
32
|
+
repo,
|
|
33
|
+
ref,
|
|
34
|
+
commitSha: marker?.commitSha ?? "",
|
|
35
|
+
cacheDir,
|
|
36
|
+
markerPath,
|
|
37
|
+
cached: marker !== null,
|
|
38
|
+
syncedAt: marker?.syncedAt ?? null
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Write a cache marker for a remote source.
|
|
43
|
+
* Call this after downloading and extracting the archive.
|
|
44
|
+
*/
|
|
45
|
+
async function writeCacheMarker(input) {
|
|
46
|
+
const marker = {
|
|
47
|
+
source: input.provider,
|
|
48
|
+
owner: input.owner,
|
|
49
|
+
repo: input.repo,
|
|
50
|
+
ref: input.ref,
|
|
51
|
+
commitSha: input.commitSha,
|
|
52
|
+
syncedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
53
|
+
};
|
|
54
|
+
await writeFile(input.markerPath, JSON.stringify(marker, null, 2), "utf8");
|
|
55
|
+
}
|
|
56
|
+
async function clearRemoteSourceCache(input) {
|
|
57
|
+
const { provider, owner, repo, ref } = input;
|
|
58
|
+
const status = await getRemoteSourceCacheStatus({
|
|
59
|
+
provider,
|
|
60
|
+
owner,
|
|
61
|
+
repo,
|
|
62
|
+
ref
|
|
63
|
+
});
|
|
64
|
+
if (!status.cached) return false;
|
|
65
|
+
try {
|
|
66
|
+
await rm(status.cacheDir, {
|
|
67
|
+
recursive: true,
|
|
68
|
+
force: true
|
|
69
|
+
});
|
|
70
|
+
return true;
|
|
71
|
+
} catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* List all cached remote sources.
|
|
77
|
+
*/
|
|
78
|
+
async function listCachedSources() {
|
|
79
|
+
const baseDir = getBaseRepoCacheDir();
|
|
80
|
+
const results = [];
|
|
81
|
+
try {
|
|
82
|
+
const sources = await readdir(baseDir, { withFileTypes: true });
|
|
83
|
+
for (const sourceEntry of sources) {
|
|
84
|
+
if (!sourceEntry.isDirectory()) continue;
|
|
85
|
+
const source = sourceEntry.name;
|
|
86
|
+
const owners = await readdir(path.join(baseDir, source), { withFileTypes: true });
|
|
87
|
+
for (const ownerEntry of owners) {
|
|
88
|
+
if (!ownerEntry.isDirectory()) continue;
|
|
89
|
+
const owner = ownerEntry.name;
|
|
90
|
+
const repos = await readdir(path.join(baseDir, source, owner), { withFileTypes: true });
|
|
91
|
+
for (const repoEntry of repos) {
|
|
92
|
+
if (!repoEntry.isDirectory()) continue;
|
|
93
|
+
const repo = repoEntry.name;
|
|
94
|
+
const refs = await readdir(path.join(baseDir, source, owner, repo), { withFileTypes: true });
|
|
95
|
+
for (const refEntry of refs) {
|
|
96
|
+
if (!refEntry.isDirectory()) continue;
|
|
97
|
+
const ref = refEntry.name;
|
|
98
|
+
const cacheDir = path.join(baseDir, source, owner, repo, ref);
|
|
99
|
+
const marker = await readCache(path.join(cacheDir, CACHE_FILE_NAME));
|
|
100
|
+
if (marker) results.push({
|
|
101
|
+
source,
|
|
102
|
+
owner,
|
|
103
|
+
repo,
|
|
104
|
+
ref,
|
|
105
|
+
commitSha: marker.commitSha,
|
|
106
|
+
syncedAt: marker.syncedAt,
|
|
107
|
+
cacheDir
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
} catch {}
|
|
114
|
+
return results;
|
|
115
|
+
}
|
|
116
|
+
//#endregion
|
|
117
|
+
//#region src/errors.ts
|
|
118
|
+
var PipelineLoaderError = class extends Error {
|
|
119
|
+
code;
|
|
120
|
+
constructor(code, message, options) {
|
|
121
|
+
super(message, options);
|
|
122
|
+
this.name = "PipelineLoaderError";
|
|
123
|
+
this.code = code;
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
var CacheMissError = class extends PipelineLoaderError {
|
|
127
|
+
provider;
|
|
128
|
+
owner;
|
|
129
|
+
repo;
|
|
130
|
+
ref;
|
|
131
|
+
constructor(provider, owner, repo, ref) {
|
|
132
|
+
super("CACHE_MISS", `Cache miss for ${provider}:${owner}/${repo}@${ref}. Run 'ucd pipelines cache refresh --${provider} ${owner}/${repo} --ref ${ref}' to sync.`);
|
|
133
|
+
this.name = "CacheMissError";
|
|
134
|
+
this.provider = provider;
|
|
135
|
+
this.owner = owner;
|
|
136
|
+
this.repo = repo;
|
|
137
|
+
this.ref = ref;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
var BundleError = class extends PipelineLoaderError {
|
|
141
|
+
entryPath;
|
|
142
|
+
constructor(entryPath, message, options) {
|
|
143
|
+
super("IMPORT_FAILED", message, options);
|
|
144
|
+
this.name = "BundleError";
|
|
145
|
+
this.entryPath = entryPath;
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
var BundleResolveError = class extends PipelineLoaderError {
|
|
149
|
+
entryPath;
|
|
150
|
+
importPath;
|
|
151
|
+
constructor(entryPath, importPath, options) {
|
|
152
|
+
super("BUNDLE_RESOLVE_FAILED", `Cannot resolve import "${importPath}" in ${entryPath}`, options);
|
|
153
|
+
this.name = "BundleResolveError";
|
|
154
|
+
this.entryPath = entryPath;
|
|
155
|
+
this.importPath = importPath;
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
var BundleTransformError = class extends PipelineLoaderError {
|
|
159
|
+
entryPath;
|
|
160
|
+
line;
|
|
161
|
+
column;
|
|
162
|
+
constructor(entryPath, options) {
|
|
163
|
+
const { line, column, ...errorOptions } = options ?? {};
|
|
164
|
+
super("BUNDLE_TRANSFORM_FAILED", `Transform/parse error in ${entryPath}${line != null ? ` at line ${line}` : ""}`, errorOptions);
|
|
165
|
+
this.name = "BundleTransformError";
|
|
166
|
+
this.entryPath = entryPath;
|
|
167
|
+
this.line = line;
|
|
168
|
+
this.column = column;
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
function toPipelineLoaderIssue(error, filePath) {
|
|
172
|
+
if (error instanceof BundleResolveError) return {
|
|
173
|
+
code: "BUNDLE_RESOLVE_FAILED",
|
|
174
|
+
scope: "bundle",
|
|
175
|
+
message: error.message,
|
|
176
|
+
filePath,
|
|
177
|
+
cause: error,
|
|
178
|
+
meta: {
|
|
179
|
+
entryPath: error.entryPath,
|
|
180
|
+
importPath: error.importPath
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
if (error instanceof BundleTransformError) return {
|
|
184
|
+
code: "BUNDLE_TRANSFORM_FAILED",
|
|
185
|
+
scope: "bundle",
|
|
186
|
+
message: error.message,
|
|
187
|
+
filePath,
|
|
188
|
+
cause: error,
|
|
189
|
+
meta: {
|
|
190
|
+
entryPath: error.entryPath,
|
|
191
|
+
...error.line != null ? { line: error.line } : {},
|
|
192
|
+
...error.column != null ? { column: error.column } : {}
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
if (error instanceof BundleError) return {
|
|
196
|
+
code: error.code,
|
|
197
|
+
scope: "import",
|
|
198
|
+
message: error.message,
|
|
199
|
+
filePath,
|
|
200
|
+
cause: error,
|
|
201
|
+
meta: { entryPath: error.entryPath }
|
|
202
|
+
};
|
|
203
|
+
if (error instanceof PipelineLoaderError) return {
|
|
204
|
+
code: error.code,
|
|
205
|
+
scope: "import",
|
|
206
|
+
message: error.message,
|
|
207
|
+
filePath,
|
|
208
|
+
cause: error
|
|
209
|
+
};
|
|
210
|
+
return {
|
|
211
|
+
code: "IMPORT_FAILED",
|
|
212
|
+
scope: "import",
|
|
213
|
+
message: error.message,
|
|
214
|
+
filePath,
|
|
215
|
+
cause: error
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
//#endregion
|
|
219
|
+
//#region src/bundle.ts
|
|
220
|
+
const RESOLVE_ERROR_RE = /could not resolve|cannot find module/i;
|
|
221
|
+
const QUOTED_IMPORT_RE = /"([^"]+)"/;
|
|
222
|
+
const TRANSFORM_ERROR_RE = /unexpected token|syntaxerror|parse error|expected|transform/i;
|
|
223
|
+
async function bundle(options) {
|
|
224
|
+
let result;
|
|
225
|
+
try {
|
|
226
|
+
result = await build({
|
|
227
|
+
input: options.entryPath,
|
|
228
|
+
write: false,
|
|
229
|
+
output: { format: "esm" },
|
|
230
|
+
cwd: options.cwd
|
|
231
|
+
});
|
|
232
|
+
} catch (err) {
|
|
233
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
234
|
+
if (RESOLVE_ERROR_RE.test(msg)) {
|
|
235
|
+
const importMatch = msg.match(QUOTED_IMPORT_RE);
|
|
236
|
+
throw new BundleResolveError(options.entryPath, importMatch?.[1] ?? "", { cause: err });
|
|
237
|
+
}
|
|
238
|
+
if (TRANSFORM_ERROR_RE.test(msg)) throw new BundleTransformError(options.entryPath, { cause: err });
|
|
239
|
+
throw new BundleTransformError(options.entryPath, { cause: err });
|
|
240
|
+
}
|
|
241
|
+
const chunk = (Array.isArray(result) ? result : [result]).flatMap((output) => output.output ?? []).find((item) => item.type === "chunk");
|
|
242
|
+
if (!chunk || chunk.type !== "chunk") throw new BundleError(options.entryPath, "Failed to bundle module");
|
|
243
|
+
return {
|
|
244
|
+
code: chunk.code,
|
|
245
|
+
dataUrl: `data:text/javascript;base64,${Buffer.from(chunk.code, "utf-8").toString("base64")}`
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
//#endregion
|
|
7
249
|
//#region src/loader.ts
|
|
8
250
|
async function loadPipelineFile(filePath) {
|
|
9
|
-
const
|
|
251
|
+
const bundleResult = await bundle({
|
|
252
|
+
entryPath: filePath,
|
|
253
|
+
cwd: path.dirname(filePath)
|
|
254
|
+
});
|
|
255
|
+
let module;
|
|
256
|
+
try {
|
|
257
|
+
module = await import(
|
|
258
|
+
/* @vite-ignore */
|
|
259
|
+
bundleResult.dataUrl
|
|
260
|
+
);
|
|
261
|
+
} catch (err) {
|
|
262
|
+
const cause = err instanceof Error ? err : new Error(String(err));
|
|
263
|
+
throw new PipelineLoaderError("IMPORT_FAILED", `Failed to import ${filePath}: ${cause.message}`, { cause });
|
|
264
|
+
}
|
|
10
265
|
const pipelines = [];
|
|
11
266
|
const exportNames = [];
|
|
12
|
-
for (const [name, value] of Object.entries(module))
|
|
13
|
-
|
|
14
|
-
|
|
267
|
+
for (const [name, value] of Object.entries(module)) {
|
|
268
|
+
if (name === "default") continue;
|
|
269
|
+
if (isPipelineDefinition(value)) {
|
|
270
|
+
pipelines.push(value);
|
|
271
|
+
exportNames.push(name);
|
|
272
|
+
}
|
|
15
273
|
}
|
|
16
274
|
return {
|
|
17
275
|
filePath,
|
|
@@ -19,57 +277,321 @@ async function loadPipelineFile(filePath) {
|
|
|
19
277
|
exportNames
|
|
20
278
|
};
|
|
21
279
|
}
|
|
22
|
-
async function loadPipelinesFromPaths(filePaths
|
|
23
|
-
const
|
|
24
|
-
if (throwOnError) {
|
|
25
|
-
const wrapped = filePaths.map((filePath) => loadPipelineFile(filePath).catch((err) => {
|
|
26
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
27
|
-
throw new Error(`Failed to load pipeline file: ${filePath}`, { cause: error });
|
|
28
|
-
}));
|
|
29
|
-
const results = await Promise.all(wrapped);
|
|
30
|
-
return {
|
|
31
|
-
pipelines: results.flatMap((r) => r.pipelines),
|
|
32
|
-
files: results,
|
|
33
|
-
errors: []
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
|
-
const settled = await Promise.allSettled(filePaths.map((fp) => loadPipelineFile(fp)));
|
|
280
|
+
async function loadPipelinesFromPaths(filePaths) {
|
|
281
|
+
const settled = await Promise.allSettled(filePaths.map((filePath) => loadPipelineFile(filePath)));
|
|
37
282
|
const files = [];
|
|
38
|
-
const
|
|
39
|
-
for (const [
|
|
283
|
+
const issues = [];
|
|
284
|
+
for (const [index, result] of settled.entries()) {
|
|
40
285
|
if (result.status === "fulfilled") {
|
|
41
286
|
files.push(result.value);
|
|
42
287
|
continue;
|
|
43
288
|
}
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
filePath: filePaths[i],
|
|
47
|
-
error
|
|
48
|
-
});
|
|
289
|
+
const cause = result.reason instanceof Error ? result.reason : new Error(String(result.reason));
|
|
290
|
+
issues.push(toPipelineLoaderIssue(cause, filePaths[index]));
|
|
49
291
|
}
|
|
50
292
|
return {
|
|
51
|
-
pipelines: files.flatMap((
|
|
293
|
+
pipelines: files.flatMap((file) => file.pipelines),
|
|
52
294
|
files,
|
|
53
|
-
|
|
295
|
+
issues
|
|
54
296
|
};
|
|
55
297
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
if (
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
"
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
298
|
+
//#endregion
|
|
299
|
+
//#region src/locator.ts
|
|
300
|
+
function parsePipelineLocator(input) {
|
|
301
|
+
if (input.startsWith("github://") || input.startsWith("gitlab://")) {
|
|
302
|
+
const provider = input.startsWith("github://") ? "github" : "gitlab";
|
|
303
|
+
const parsed = new URL(input.replace(`${provider}://`, "https://fake-host/"));
|
|
304
|
+
const [, owner, repo] = parsed.pathname.split("/");
|
|
305
|
+
if (!owner || !repo) throw new Error(`Invalid ${provider} locator: ${input}`);
|
|
306
|
+
return {
|
|
307
|
+
kind: "remote",
|
|
308
|
+
provider,
|
|
309
|
+
owner,
|
|
310
|
+
repo,
|
|
311
|
+
ref: parsed.searchParams.get("ref") ?? "HEAD",
|
|
312
|
+
path: parsed.searchParams.get("path") ?? void 0
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
kind: "local",
|
|
317
|
+
path: input
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function parseRemoteSourceUrl(url) {
|
|
321
|
+
try {
|
|
322
|
+
const locator = parsePipelineLocator(url);
|
|
323
|
+
return locator.kind === "remote" ? locator : null;
|
|
324
|
+
} catch {
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
//#endregion
|
|
329
|
+
//#region src/materialize.ts
|
|
330
|
+
const BACKSLASH_RE = /\\/g;
|
|
331
|
+
function createRemoteOrigin(locator) {
|
|
332
|
+
return {
|
|
333
|
+
provider: locator.provider,
|
|
334
|
+
owner: locator.owner,
|
|
335
|
+
repo: locator.repo,
|
|
336
|
+
ref: locator.ref ?? "HEAD",
|
|
337
|
+
...locator.path ? { path: locator.path } : {}
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
async function detectPathKind(targetPath) {
|
|
341
|
+
return stat(targetPath).then((value) => value.isFile() ? "file" : value.isDirectory() ? "directory" : null).catch(() => null);
|
|
342
|
+
}
|
|
343
|
+
async function materializePipelineLocator(locator) {
|
|
344
|
+
if (locator.kind === "local") {
|
|
345
|
+
const absolutePath = path.resolve(locator.path);
|
|
346
|
+
const pathKind = await detectPathKind(absolutePath);
|
|
347
|
+
if (!pathKind) return { issues: [{
|
|
348
|
+
code: "MATERIALIZE_FAILED",
|
|
349
|
+
scope: "file",
|
|
350
|
+
message: `Path does not exist: ${absolutePath}`,
|
|
351
|
+
locator,
|
|
352
|
+
meta: { path: absolutePath }
|
|
353
|
+
}] };
|
|
354
|
+
if (pathKind === "directory") return {
|
|
355
|
+
repositoryPath: absolutePath,
|
|
356
|
+
issues: []
|
|
357
|
+
};
|
|
358
|
+
return {
|
|
359
|
+
repositoryPath: path.dirname(absolutePath),
|
|
360
|
+
filePath: absolutePath,
|
|
361
|
+
relativePath: path.basename(absolutePath),
|
|
362
|
+
issues: []
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
const ref = locator.ref ?? "HEAD";
|
|
366
|
+
const status = await getRemoteSourceCacheStatus({
|
|
367
|
+
provider: locator.provider,
|
|
368
|
+
owner: locator.owner,
|
|
369
|
+
repo: locator.repo,
|
|
370
|
+
ref
|
|
71
371
|
});
|
|
372
|
+
if (!status.cached) return { issues: [{
|
|
373
|
+
code: "CACHE_MISS",
|
|
374
|
+
scope: "repository",
|
|
375
|
+
message: `Remote repository ${locator.owner}/${locator.repo}@${ref} is not materialized in cache.`,
|
|
376
|
+
locator,
|
|
377
|
+
meta: {
|
|
378
|
+
provider: locator.provider,
|
|
379
|
+
owner: locator.owner,
|
|
380
|
+
repo: locator.repo,
|
|
381
|
+
ref
|
|
382
|
+
}
|
|
383
|
+
}] };
|
|
384
|
+
const origin = createRemoteOrigin(locator);
|
|
385
|
+
if (!locator.path) return {
|
|
386
|
+
repositoryPath: status.cacheDir,
|
|
387
|
+
origin,
|
|
388
|
+
issues: []
|
|
389
|
+
};
|
|
390
|
+
const targetPath = path.resolve(status.cacheDir, locator.path);
|
|
391
|
+
if (targetPath !== status.cacheDir && !targetPath.startsWith(`${status.cacheDir}${path.sep}`)) return { issues: [{
|
|
392
|
+
code: "INVALID_LOCATOR",
|
|
393
|
+
scope: "locator",
|
|
394
|
+
message: `Remote path resolves outside materialized repository: ${locator.path}`,
|
|
395
|
+
locator,
|
|
396
|
+
repositoryPath: status.cacheDir
|
|
397
|
+
}] };
|
|
398
|
+
const pathKind = await detectPathKind(targetPath);
|
|
399
|
+
if (!pathKind) return { issues: [{
|
|
400
|
+
code: "MATERIALIZE_FAILED",
|
|
401
|
+
scope: "file",
|
|
402
|
+
message: `Materialized path does not exist: ${locator.path}`,
|
|
403
|
+
locator,
|
|
404
|
+
repositoryPath: status.cacheDir,
|
|
405
|
+
meta: { path: locator.path }
|
|
406
|
+
}] };
|
|
407
|
+
if (pathKind === "directory") return {
|
|
408
|
+
repositoryPath: targetPath,
|
|
409
|
+
relativePath: locator.path.replace(BACKSLASH_RE, "/"),
|
|
410
|
+
origin,
|
|
411
|
+
issues: []
|
|
412
|
+
};
|
|
413
|
+
return {
|
|
414
|
+
repositoryPath: path.dirname(targetPath),
|
|
415
|
+
filePath: targetPath,
|
|
416
|
+
relativePath: path.basename(targetPath),
|
|
417
|
+
origin,
|
|
418
|
+
issues: []
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
//#endregion
|
|
422
|
+
//#region src/adapters/github.ts
|
|
423
|
+
const GITHUB_API_BASE = "https://api.github.com";
|
|
424
|
+
const GITHUB_ACCEPT_HEADER = "application/vnd.github.v3+json";
|
|
425
|
+
async function resolveGitHubRef(ref) {
|
|
426
|
+
const { owner, repo, ref: refValue = "HEAD" } = ref;
|
|
427
|
+
const response = await fetch(`${GITHUB_API_BASE}/repos/${owner}/${repo}/commits/${refValue}`, { headers: { Accept: GITHUB_ACCEPT_HEADER } });
|
|
428
|
+
if (!response.ok) throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
|
|
429
|
+
const data = await response.json();
|
|
430
|
+
if (!data || typeof data !== "object" || !("sha" in data) || typeof data.sha !== "string") throw new Error("GitHub API error: invalid response format (missing 'sha')");
|
|
431
|
+
return data.sha;
|
|
432
|
+
}
|
|
433
|
+
async function downloadGitHubArchive(ref) {
|
|
434
|
+
const { owner, repo, commitSha } = ref;
|
|
435
|
+
const archiveUrl = `${GITHUB_API_BASE}/repos/${owner}/${repo}/tarball/${commitSha}`;
|
|
436
|
+
const response = await fetch(archiveUrl, { headers: { Accept: "application/vnd.github.v3+json" } });
|
|
437
|
+
if (!response.ok) throw new Error(`Failed to download GitHub archive: ${response.status} ${response.statusText}`);
|
|
438
|
+
return response.arrayBuffer();
|
|
439
|
+
}
|
|
440
|
+
//#endregion
|
|
441
|
+
//#region src/adapters/gitlab.ts
|
|
442
|
+
const GITLAB_API_BASE = "https://gitlab.com/api/v4";
|
|
443
|
+
async function resolveGitLabRef(ref) {
|
|
444
|
+
const { owner, repo, ref: refValue = "HEAD" } = ref;
|
|
445
|
+
const url = `${GITLAB_API_BASE}/projects/${encodeURIComponent(`${owner}/${repo}`)}/repository/commits/${refValue}`;
|
|
446
|
+
const response = await fetch(url);
|
|
447
|
+
if (!response.ok) throw new Error(`GitLab API error: ${response.status} ${response.statusText}`);
|
|
448
|
+
const data = await response.json();
|
|
449
|
+
if (!data || typeof data !== "object" || !("id" in data) || typeof data.id !== "string") throw new Error("GitLab API error: invalid response format (missing 'id')");
|
|
450
|
+
return data.id;
|
|
451
|
+
}
|
|
452
|
+
async function downloadGitLabArchive(ref) {
|
|
453
|
+
const { owner, repo, commitSha } = ref;
|
|
454
|
+
const archiveUrl = `${GITLAB_API_BASE}/projects/${encodeURIComponent(`${owner}/${repo}`)}/repository/archive.tar.gz?sha=${commitSha}`;
|
|
455
|
+
const response = await fetch(archiveUrl);
|
|
456
|
+
if (!response.ok) throw new Error(`Failed to download GitLab archive: ${response.status} ${response.statusText}`);
|
|
457
|
+
return response.arrayBuffer();
|
|
458
|
+
}
|
|
459
|
+
//#endregion
|
|
460
|
+
//#region src/remote.ts
|
|
461
|
+
const LEADING_PATH_SEPARATORS_RE = /^([/\\])+/;
|
|
462
|
+
async function extractArchiveToDirectory(archiveBuffer, targetDir) {
|
|
463
|
+
const files = await parseTarGzip(archiveBuffer);
|
|
464
|
+
const rootPrefix = files[0]?.name.split("/")[0];
|
|
465
|
+
if (!rootPrefix) throw new Error("Invalid archive: no files found");
|
|
466
|
+
for (const file of files) {
|
|
467
|
+
if (file.type === "directory" || !file.data) continue;
|
|
468
|
+
const relativePath = file.name.slice(rootPrefix.length + 1);
|
|
469
|
+
if (!relativePath) continue;
|
|
470
|
+
let safeRelativePath = path.normalize(relativePath);
|
|
471
|
+
safeRelativePath = safeRelativePath.replace(LEADING_PATH_SEPARATORS_RE, "");
|
|
472
|
+
if (!safeRelativePath) continue;
|
|
473
|
+
const upSegment = `..${path.sep}`;
|
|
474
|
+
if (safeRelativePath === ".." || safeRelativePath.startsWith(upSegment) || safeRelativePath.includes(`${path.sep}..${path.sep}`) || safeRelativePath.endsWith(`${path.sep}..`)) throw new Error(`Invalid archive entry path (path traversal detected): ${file.name}`);
|
|
475
|
+
const outputPath = path.join(targetDir, safeRelativePath);
|
|
476
|
+
const resolvedTargetDir = path.resolve(targetDir);
|
|
477
|
+
const resolvedOutputPath = path.resolve(outputPath);
|
|
478
|
+
if (resolvedOutputPath !== resolvedTargetDir && !resolvedOutputPath.startsWith(resolvedTargetDir + path.sep)) throw new Error(`Invalid archive entry path (outside target dir): ${file.name}`);
|
|
479
|
+
await mkdir(path.dirname(resolvedOutputPath), { recursive: true });
|
|
480
|
+
await writeFile(resolvedOutputPath, file.data);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
async function resolveRemoteLocatorRef(provider, ref) {
|
|
484
|
+
if (provider === "github") return resolveGitHubRef(ref);
|
|
485
|
+
return resolveGitLabRef(ref);
|
|
486
|
+
}
|
|
487
|
+
async function downloadRemoteLocatorArchive(provider, ref) {
|
|
488
|
+
if (provider === "github") return downloadGitHubArchive(ref);
|
|
489
|
+
return downloadGitLabArchive(ref);
|
|
490
|
+
}
|
|
491
|
+
async function extractArchiveToCleanDirectory(archiveBuffer, targetDir) {
|
|
492
|
+
await rm(targetDir, {
|
|
493
|
+
recursive: true,
|
|
494
|
+
force: true
|
|
495
|
+
});
|
|
496
|
+
await mkdir(targetDir, { recursive: true });
|
|
497
|
+
try {
|
|
498
|
+
await extractArchiveToDirectory(archiveBuffer, targetDir);
|
|
499
|
+
} catch (err) {
|
|
500
|
+
await rm(targetDir, {
|
|
501
|
+
recursive: true,
|
|
502
|
+
force: true
|
|
503
|
+
});
|
|
504
|
+
throw err;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
async function checkRemoteLocatorUpdates(input) {
|
|
508
|
+
const { provider, owner, repo, ref = "HEAD" } = input;
|
|
509
|
+
const status = await getRemoteSourceCacheStatus({
|
|
510
|
+
provider,
|
|
511
|
+
owner,
|
|
512
|
+
repo,
|
|
513
|
+
ref
|
|
514
|
+
});
|
|
515
|
+
const currentSha = status.cached ? status.commitSha : null;
|
|
516
|
+
try {
|
|
517
|
+
const remoteSha = await resolveRemoteLocatorRef(provider, {
|
|
518
|
+
owner,
|
|
519
|
+
repo,
|
|
520
|
+
ref
|
|
521
|
+
});
|
|
522
|
+
return {
|
|
523
|
+
hasUpdate: currentSha !== remoteSha,
|
|
524
|
+
currentSha,
|
|
525
|
+
remoteSha
|
|
526
|
+
};
|
|
527
|
+
} catch (err) {
|
|
528
|
+
return {
|
|
529
|
+
hasUpdate: false,
|
|
530
|
+
currentSha,
|
|
531
|
+
remoteSha: "",
|
|
532
|
+
error: err instanceof Error ? err : new Error(String(err))
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async function ensureRemoteLocator(input) {
|
|
537
|
+
const { provider, owner, repo, ref = "HEAD", force = false } = input;
|
|
538
|
+
const status = await getRemoteSourceCacheStatus({
|
|
539
|
+
provider,
|
|
540
|
+
owner,
|
|
541
|
+
repo,
|
|
542
|
+
ref
|
|
543
|
+
});
|
|
544
|
+
const previousSha = status.cached ? status.commitSha : null;
|
|
545
|
+
try {
|
|
546
|
+
const commitSha = await resolveRemoteLocatorRef(provider, {
|
|
547
|
+
owner,
|
|
548
|
+
repo,
|
|
549
|
+
ref
|
|
550
|
+
});
|
|
551
|
+
if (!force && status.cached && status.commitSha === commitSha) return {
|
|
552
|
+
success: true,
|
|
553
|
+
updated: false,
|
|
554
|
+
previousSha,
|
|
555
|
+
newSha: commitSha,
|
|
556
|
+
cacheDir: status.cacheDir
|
|
557
|
+
};
|
|
558
|
+
const archiveBuffer = await downloadRemoteLocatorArchive(provider, {
|
|
559
|
+
owner,
|
|
560
|
+
repo,
|
|
561
|
+
ref,
|
|
562
|
+
commitSha
|
|
563
|
+
});
|
|
564
|
+
await rm(status.cacheDir, {
|
|
565
|
+
recursive: true,
|
|
566
|
+
force: true
|
|
567
|
+
});
|
|
568
|
+
await extractArchiveToCleanDirectory(archiveBuffer, status.cacheDir);
|
|
569
|
+
await writeCacheMarker({
|
|
570
|
+
provider,
|
|
571
|
+
owner,
|
|
572
|
+
repo,
|
|
573
|
+
ref,
|
|
574
|
+
commitSha,
|
|
575
|
+
cacheDir: status.cacheDir,
|
|
576
|
+
markerPath: status.markerPath
|
|
577
|
+
});
|
|
578
|
+
return {
|
|
579
|
+
success: true,
|
|
580
|
+
updated: true,
|
|
581
|
+
previousSha,
|
|
582
|
+
newSha: commitSha,
|
|
583
|
+
cacheDir: status.cacheDir
|
|
584
|
+
};
|
|
585
|
+
} catch (err) {
|
|
586
|
+
return {
|
|
587
|
+
success: false,
|
|
588
|
+
updated: false,
|
|
589
|
+
previousSha,
|
|
590
|
+
newSha: "",
|
|
591
|
+
cacheDir: status.cacheDir,
|
|
592
|
+
error: err instanceof Error ? err : new Error(String(err))
|
|
593
|
+
};
|
|
594
|
+
}
|
|
72
595
|
}
|
|
73
|
-
|
|
74
596
|
//#endregion
|
|
75
|
-
export {
|
|
597
|
+
export { BundleError, BundleResolveError, BundleTransformError, CacheMissError, PipelineLoaderError, checkRemoteLocatorUpdates, clearRemoteSourceCache, ensureRemoteLocator, getRemoteSourceCacheStatus, listCachedSources, loadPipelineFile, loadPipelinesFromPaths, materializePipelineLocator, parsePipelineLocator, parseRemoteSourceUrl, writeCacheMarker };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ucdjs/pipelines-loader",
|
|
3
|
-
"version": "0.0.1-beta.
|
|
3
|
+
"version": "0.0.1-beta.8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Lucas Nørgård",
|
|
@@ -19,8 +19,7 @@
|
|
|
19
19
|
},
|
|
20
20
|
"exports": {
|
|
21
21
|
".": "./dist/index.mjs",
|
|
22
|
-
"./
|
|
23
|
-
"./remote": "./dist/remote.mjs",
|
|
22
|
+
"./discover": "./dist/discover.mjs",
|
|
24
23
|
"./package.json": "./package.json"
|
|
25
24
|
},
|
|
26
25
|
"types": "./dist/index.d.mts",
|
|
@@ -31,21 +30,20 @@
|
|
|
31
30
|
"node": ">=24.13"
|
|
32
31
|
},
|
|
33
32
|
"dependencies": {
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"picomatch": "4.0.3",
|
|
37
|
-
"rolldown": "1.0.0-rc.5",
|
|
33
|
+
"nanotar": "0.3.0",
|
|
34
|
+
"rolldown": "1.0.0-rc.11",
|
|
38
35
|
"tinyglobby": "0.2.15",
|
|
39
|
-
"@ucdjs/
|
|
36
|
+
"@ucdjs/env": "0.1.1-beta.9",
|
|
37
|
+
"@ucdjs/pipelines-core": "0.0.1-beta.8",
|
|
38
|
+
"@ucdjs-internal/shared": "0.1.1-beta.8"
|
|
40
39
|
},
|
|
41
40
|
"devDependencies": {
|
|
42
|
-
"@luxass/eslint-config": "7.
|
|
43
|
-
"
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"vitest-testdirs": "4.4.2",
|
|
41
|
+
"@luxass/eslint-config": "7.4.1",
|
|
42
|
+
"eslint": "10.1.0",
|
|
43
|
+
"publint": "0.3.18",
|
|
44
|
+
"tsdown": "0.21.4",
|
|
45
|
+
"typescript": "6.0.2",
|
|
46
|
+
"vitest-testdirs": "4.4.3",
|
|
49
47
|
"@ucdjs-tooling/tsconfig": "1.0.0",
|
|
50
48
|
"@ucdjs-tooling/tsdown-config": "1.0.0"
|
|
51
49
|
},
|