@serve.zone/gitops 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +24 -0
- package/deno.json +1 -1
- package/dist_serve/bundle.js +726 -694
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes/connectionmanager.d.ts +5 -1
- package/dist_ts/classes/connectionmanager.js +62 -7
- package/dist_ts/classes/syncmanager.d.ts +40 -0
- package/dist_ts/classes/syncmanager.js +804 -111
- package/dist_ts/classes/syncpath.d.ts +13 -0
- package/dist_ts/classes/syncpath.js +53 -0
- package/dist_ts/opsserver/handlers/connections.handler.js +8 -2
- package/dist_ts/opsserver/handlers/sync.handler.js +5 -1
- package/dist_ts/providers/classes.baseprovider.d.ts +22 -0
- package/dist_ts/providers/classes.baseprovider.js +93 -1
- package/dist_ts/providers/classes.giteaprovider.d.ts +18 -1
- package/dist_ts/providers/classes.giteaprovider.js +236 -1
- package/dist_ts/providers/classes.gitlabprovider.d.ts +16 -1
- package/dist_ts/providers/classes.gitlabprovider.js +235 -1
- package/dist_ts_interfaces/data/artifact.d.ts +39 -0
- package/dist_ts_interfaces/data/artifact.js +2 -0
- package/dist_ts_interfaces/data/connection.d.ts +3 -0
- package/dist_ts_interfaces/data/index.d.ts +1 -0
- package/dist_ts_interfaces/data/index.js +2 -1
- package/dist_ts_interfaces/data/sync.d.ts +21 -0
- package/dist_ts_interfaces/requests/connections.d.ts +6 -0
- package/dist_ts_interfaces/requests/sync.d.ts +4 -0
- package/package.json +4 -4
- package/readme.md +8 -4
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes/connectionmanager.ts +61 -6
- package/ts/classes/syncmanager.ts +1095 -119
- package/ts/classes/syncpath.ts +71 -0
- package/ts/opsserver/handlers/connections.handler.ts +9 -0
- package/ts/opsserver/handlers/sync.handler.ts +4 -0
- package/ts/providers/classes.baseprovider.ts +137 -0
- package/ts/providers/classes.giteaprovider.ts +293 -1
- package/ts/providers/classes.gitlabprovider.ts +292 -1
- package/ts_interfaces/data/artifact.ts +43 -0
- package/ts_interfaces/data/connection.ts +3 -0
- package/ts_interfaces/data/index.ts +1 -0
- package/ts_interfaces/data/sync.ts +22 -0
- package/ts_interfaces/requests/connections.ts +6 -0
- package/ts_interfaces/requests/sync.ts +4 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/appstate.ts +10 -0
- package/ts_web/elements/views/connections/index.ts +26 -0
- package/ts_web/elements/views/sync/index.ts +26 -4
|
@@ -1,15 +1,32 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
2
|
import { logger } from '../logging.js';
|
|
3
3
|
import { intervalMinutesToMs, unrefTimer, validateIntervalMinutes } from '../timers.js';
|
|
4
|
-
import type { ChildProcess } from 'node:child_process';
|
|
5
4
|
import type * as interfaces from '../../ts_interfaces/index.js';
|
|
6
5
|
import type { ConnectionManager } from './connectionmanager.js';
|
|
7
6
|
import type { ActionLog } from './actionlog.js';
|
|
8
7
|
import type { StorageManager } from '../storage/index.js';
|
|
9
8
|
import type { BaseProvider } from '../providers/classes.baseprovider.js';
|
|
9
|
+
import * as syncpath from './syncpath.js';
|
|
10
10
|
|
|
11
11
|
const SYNC_PREFIX = '/sync/';
|
|
12
12
|
const SYNC_STATUS_PREFIX = '/sync-status/';
|
|
13
|
+
const SYNC_MAPPING_PREFIX = '/sync-mappings/';
|
|
14
|
+
const REPO_SYNC_CONCURRENCY = 10;
|
|
15
|
+
const ARTIFACT_REPO_SYNC_CONCURRENCY = 2;
|
|
16
|
+
const IMAGE_COPY_CONCURRENCY = 2;
|
|
17
|
+
const SKOPEO_VERSION_TIMEOUT_MS = 10_000;
|
|
18
|
+
const SKOPEO_COPY_TIMEOUT_MS = 30 * 60 * 1000;
|
|
19
|
+
const RELEASE_ASSET_TRANSFER_TIMEOUT_MS = 10 * 60 * 1000;
|
|
20
|
+
const RELEASE_ASSET_MAX_BYTES = 250 * 1024 * 1024;
|
|
21
|
+
const RAW_API_TIMEOUT_MS = 2 * 60 * 1000;
|
|
22
|
+
const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
|
|
23
|
+
|
|
24
|
+
type TSpawnedChild = ReturnType<typeof plugins.childProcess.spawn>;
|
|
25
|
+
|
|
26
|
+
interface IRunCommandOptions {
|
|
27
|
+
timeoutMs?: number;
|
|
28
|
+
env?: NodeJS.ProcessEnv;
|
|
29
|
+
}
|
|
13
30
|
|
|
14
31
|
/**
|
|
15
32
|
* Manages sync configurations and executes periodic git mirror operations.
|
|
@@ -23,8 +40,12 @@ export class SyncManager {
|
|
|
23
40
|
private syncedGroupMeta: Set<string> = new Set();
|
|
24
41
|
private currentSyncConfig: interfaces.data.ISyncConfig | null = null;
|
|
25
42
|
private avatarUploadCache: Map<string, string> = new Map();
|
|
26
|
-
private activeGitChildren = new Set<
|
|
43
|
+
private activeGitChildren = new Set<TSpawnedChild>();
|
|
27
44
|
private stopping = false;
|
|
45
|
+
private skopeoAvailable: boolean | null = null;
|
|
46
|
+
private abortController = new AbortController();
|
|
47
|
+
private activeImageCopies = 0;
|
|
48
|
+
private imageCopyWaiters: Array<() => void> = [];
|
|
28
49
|
|
|
29
50
|
private mirrorsPath = '';
|
|
30
51
|
|
|
@@ -36,6 +57,7 @@ export class SyncManager {
|
|
|
36
57
|
|
|
37
58
|
async init(): Promise<void> {
|
|
38
59
|
this.stopping = false;
|
|
60
|
+
this.abortController = new AbortController();
|
|
39
61
|
// Create temp directory for mirrors (RAM-backed on most Linux systems via tmpfs)
|
|
40
62
|
this.mirrorsPath = await plugins.fs.mkdtemp(plugins.path.join(plugins.os.tmpdir(), 'gitops-mirrors-'));
|
|
41
63
|
await this.loadConfigs();
|
|
@@ -51,6 +73,12 @@ export class SyncManager {
|
|
|
51
73
|
|
|
52
74
|
async stop(): Promise<void> {
|
|
53
75
|
this.stopping = true;
|
|
76
|
+
if (!this.abortController.signal.aborted) {
|
|
77
|
+
this.abortController.abort(new Error('SyncManager is stopping'));
|
|
78
|
+
}
|
|
79
|
+
for (const wake of this.imageCopyWaiters.splice(0)) {
|
|
80
|
+
wake();
|
|
81
|
+
}
|
|
54
82
|
for (const [_id, timer] of this.timers) {
|
|
55
83
|
clearInterval(timer);
|
|
56
84
|
}
|
|
@@ -105,6 +133,8 @@ export class SyncManager {
|
|
|
105
133
|
intervalMinutes?: number;
|
|
106
134
|
enforceDelete?: boolean;
|
|
107
135
|
enforceGroupDelete?: boolean;
|
|
136
|
+
syncReleases?: boolean;
|
|
137
|
+
syncContainerImages?: boolean;
|
|
108
138
|
addMirrorHint?: boolean;
|
|
109
139
|
useGroupAvatarsForProjects?: boolean;
|
|
110
140
|
}): Promise<interfaces.data.ISyncConfig> {
|
|
@@ -118,8 +148,12 @@ export class SyncManager {
|
|
|
118
148
|
status: 'paused',
|
|
119
149
|
lastSyncAt: 0,
|
|
120
150
|
reposSynced: 0,
|
|
151
|
+
releasesSynced: 0,
|
|
152
|
+
imagesSynced: 0,
|
|
121
153
|
enforceDelete: data.enforceDelete ?? false,
|
|
122
154
|
enforceGroupDelete: data.enforceGroupDelete ?? false,
|
|
155
|
+
syncReleases: data.syncReleases ?? false,
|
|
156
|
+
syncContainerImages: data.syncContainerImages ?? false,
|
|
123
157
|
addMirrorHint: data.addMirrorHint ?? false,
|
|
124
158
|
useGroupAvatarsForProjects: data.useGroupAvatarsForProjects ?? false,
|
|
125
159
|
createdAt: Date.now(),
|
|
@@ -133,7 +167,7 @@ export class SyncManager {
|
|
|
133
167
|
|
|
134
168
|
async updateConfig(
|
|
135
169
|
id: string,
|
|
136
|
-
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean },
|
|
170
|
+
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; syncReleases?: boolean; syncContainerImages?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean },
|
|
137
171
|
): Promise<interfaces.data.ISyncConfig> {
|
|
138
172
|
const config = this.configs.find((c) => c.id === id);
|
|
139
173
|
if (!config) throw new Error(`Sync config not found: ${id}`);
|
|
@@ -143,9 +177,11 @@ export class SyncManager {
|
|
|
143
177
|
}
|
|
144
178
|
if (updates.enforceDelete !== undefined) config.enforceDelete = updates.enforceDelete;
|
|
145
179
|
if (updates.enforceGroupDelete !== undefined) config.enforceGroupDelete = updates.enforceGroupDelete;
|
|
180
|
+
if (updates.syncReleases !== undefined) config.syncReleases = updates.syncReleases;
|
|
181
|
+
if (updates.syncContainerImages !== undefined) config.syncContainerImages = updates.syncContainerImages;
|
|
146
182
|
if (updates.addMirrorHint !== undefined) config.addMirrorHint = updates.addMirrorHint;
|
|
147
183
|
if (updates.useGroupAvatarsForProjects !== undefined) config.useGroupAvatarsForProjects = updates.useGroupAvatarsForProjects;
|
|
148
|
-
if (updates.targetGroupOffset !== undefined) config.targetGroupOffset = updates.targetGroupOffset;
|
|
184
|
+
if (updates.targetGroupOffset !== undefined) config.targetGroupOffset = updates.targetGroupOffset || undefined;
|
|
149
185
|
this.validateSyncConfig(config);
|
|
150
186
|
await this.persistConfig(config);
|
|
151
187
|
// Restart timer with new interval
|
|
@@ -166,6 +202,10 @@ export class SyncManager {
|
|
|
166
202
|
for (const key of statusKeys) {
|
|
167
203
|
await this.storageManager.delete(key);
|
|
168
204
|
}
|
|
205
|
+
const mappingKeys = await this.storageManager.list(`${SYNC_MAPPING_PREFIX}${id}/`);
|
|
206
|
+
for (const key of mappingKeys) {
|
|
207
|
+
await this.storageManager.delete(key);
|
|
208
|
+
}
|
|
169
209
|
// Clean up mirror directory
|
|
170
210
|
const mirrorDir = plugins.path.join(this.mirrorsPath, id);
|
|
171
211
|
try {
|
|
@@ -222,11 +262,11 @@ export class SyncManager {
|
|
|
222
262
|
const sourceProvider = this.connectionManager.getProvider(config.sourceConnectionId);
|
|
223
263
|
const targetProvider = this.connectionManager.getProvider(config.targetConnectionId);
|
|
224
264
|
logger.syncLog('info', `Fetching source projects from "${sourceConn.name}"...`, 'preview');
|
|
225
|
-
const allProjects = await sourceProvider.getProjects();
|
|
226
|
-
const projects = allProjects.filter(p => !this.isObsoletePath(p.fullPath));
|
|
265
|
+
const allProjects: interfaces.data.IProject[] = await sourceProvider.getProjects();
|
|
266
|
+
const projects = allProjects.filter((p: interfaces.data.IProject) => !this.isObsoletePath(p.fullPath));
|
|
227
267
|
logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'preview');
|
|
228
268
|
|
|
229
|
-
const mappings = projects.map((project) => {
|
|
269
|
+
const mappings = projects.map((project: interfaces.data.IProject) => {
|
|
230
270
|
const targetFullPath = this.computeTargetFullPath(
|
|
231
271
|
project.fullPath, sourceConn.groupFilter, config.targetGroupOffset,
|
|
232
272
|
);
|
|
@@ -238,14 +278,14 @@ export class SyncManager {
|
|
|
238
278
|
if (config.enforceDelete) {
|
|
239
279
|
logger.syncLog('info', 'Computing repo deletions (enforce-delete enabled)...', 'preview');
|
|
240
280
|
const expectedTargetPaths = new Set(
|
|
241
|
-
mappings.map((m) => m.targetFullPath.toLowerCase()),
|
|
281
|
+
mappings.map((m: { sourceFullPath: string; targetFullPath: string }) => this.computeProviderProjectFullPath(targetConn, m.targetFullPath).toLowerCase()),
|
|
242
282
|
);
|
|
243
283
|
const scopePrefix = config.targetGroupOffset;
|
|
244
284
|
const targetProjects = await targetProvider.getProjects();
|
|
245
285
|
|
|
246
286
|
for (const tp of targetProjects) {
|
|
247
287
|
if (this.isObsoletePath(tp.fullPath)) continue;
|
|
248
|
-
if (
|
|
288
|
+
if (!this.isTargetProjectInScope(targetConn, tp.fullPath, scopePrefix)) {
|
|
249
289
|
continue;
|
|
250
290
|
}
|
|
251
291
|
if (!expectedTargetPaths.has(tp.fullPath.toLowerCase())) {
|
|
@@ -326,29 +366,68 @@ export class SyncManager {
|
|
|
326
366
|
// Get all projects from source
|
|
327
367
|
const sourceProvider = this.connectionManager.getProvider(config.sourceConnectionId);
|
|
328
368
|
logger.syncLog('info', `Fetching source projects from "${sourceConn.name}"...`, 'api');
|
|
329
|
-
const allProjects = await sourceProvider.getProjects();
|
|
330
|
-
const projects = allProjects.filter(p => !this.isObsoletePath(p.fullPath));
|
|
369
|
+
const allProjects: interfaces.data.IProject[] = await sourceProvider.getProjects();
|
|
370
|
+
const projects = allProjects.filter((p: interfaces.data.IProject) => !this.isObsoletePath(p.fullPath));
|
|
331
371
|
logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'api');
|
|
332
372
|
|
|
333
373
|
let synced = 0;
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
374
|
+
let failed = 0;
|
|
375
|
+
let releasesSynced = 0;
|
|
376
|
+
let imagesSynced = 0;
|
|
377
|
+
const concurrency = config.syncReleases || config.syncContainerImages
|
|
378
|
+
? ARTIFACT_REPO_SYNC_CONCURRENCY
|
|
379
|
+
: REPO_SYNC_CONCURRENCY;
|
|
380
|
+
for (let i = 0; i < projects.length; i += concurrency) {
|
|
381
|
+
const batch = projects.slice(i, i + concurrency);
|
|
382
|
+
await Promise.all(batch.map(async (project: interfaces.data.IProject) => {
|
|
383
|
+
const targetFullPath = this.computeTargetFullPath(
|
|
384
|
+
project.fullPath, sourceConn.groupFilter, config.targetGroupOffset,
|
|
385
|
+
);
|
|
338
386
|
try {
|
|
339
387
|
logger.syncLog('info', `Syncing ${project.fullPath}...`, 'sync');
|
|
340
|
-
await this.syncRepo(config, project, sourceConn, targetConn);
|
|
388
|
+
const syncResult = await this.syncRepo(config, project, sourceConn, targetConn);
|
|
389
|
+
releasesSynced += syncResult.releasesSynced || 0;
|
|
390
|
+
imagesSynced += syncResult.imagesSynced || 0;
|
|
391
|
+
const artifactErrors = [syncResult.releaseSyncError, syncResult.imageSyncError].filter(Boolean);
|
|
392
|
+
if (artifactErrors.length > 0) {
|
|
393
|
+
failed++;
|
|
394
|
+
const errMsg = artifactErrors.join('; ');
|
|
395
|
+
await this.updateRepoStatus(config.id, project.fullPath, {
|
|
396
|
+
status: 'error',
|
|
397
|
+
targetFullPath: syncResult.targetFullPath,
|
|
398
|
+
sourceProjectId: project.id,
|
|
399
|
+
targetProjectId: syncResult.targetProjectId,
|
|
400
|
+
releasesSynced: syncResult.releasesSynced,
|
|
401
|
+
imagesSynced: syncResult.imagesSynced,
|
|
402
|
+
releaseSyncError: syncResult.releaseSyncError,
|
|
403
|
+
imageSyncError: syncResult.imageSyncError,
|
|
404
|
+
lastSyncAt: Date.now(),
|
|
405
|
+
lastSyncError: errMsg,
|
|
406
|
+
});
|
|
407
|
+
logger.syncLog('error', `Artifact sync failed for ${project.fullPath}: ${errMsg}`, 'sync');
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
341
410
|
synced++;
|
|
342
411
|
await this.updateRepoStatus(config.id, project.fullPath, {
|
|
343
412
|
status: 'synced',
|
|
413
|
+
targetFullPath: syncResult.targetFullPath,
|
|
414
|
+
sourceProjectId: project.id,
|
|
415
|
+
targetProjectId: syncResult.targetProjectId,
|
|
416
|
+
releasesSynced: syncResult.releasesSynced,
|
|
417
|
+
imagesSynced: syncResult.imagesSynced,
|
|
418
|
+
releaseSyncError: syncResult.releaseSyncError,
|
|
419
|
+
imageSyncError: syncResult.imageSyncError,
|
|
344
420
|
lastSyncAt: Date.now(),
|
|
345
421
|
lastSyncError: undefined,
|
|
346
422
|
});
|
|
347
423
|
logger.syncLog('success', `Synced ${project.fullPath}`, 'sync');
|
|
348
424
|
} catch (err) {
|
|
425
|
+
failed++;
|
|
349
426
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
350
427
|
await this.updateRepoStatus(config.id, project.fullPath, {
|
|
351
428
|
status: 'error',
|
|
429
|
+
targetFullPath,
|
|
430
|
+
sourceProjectId: project.id,
|
|
352
431
|
lastSyncError: errMsg,
|
|
353
432
|
lastSyncAt: Date.now(),
|
|
354
433
|
});
|
|
@@ -371,12 +450,20 @@ export class SyncManager {
|
|
|
371
450
|
|
|
372
451
|
config.lastSyncAt = Date.now();
|
|
373
452
|
config.reposSynced = synced;
|
|
453
|
+
config.releasesSynced = releasesSynced;
|
|
454
|
+
config.imagesSynced = imagesSynced;
|
|
374
455
|
config.lastSyncDurationMs = Date.now() - startTime;
|
|
375
|
-
|
|
376
|
-
|
|
456
|
+
if (failed > 0) {
|
|
457
|
+
config.lastSyncError = `Sync completed with ${failed}/${projects.length} repo error(s)`;
|
|
458
|
+
config.status = 'error';
|
|
459
|
+
} else {
|
|
460
|
+
config.lastSyncError = undefined;
|
|
461
|
+
if (config.status === 'error') config.status = 'active';
|
|
462
|
+
}
|
|
377
463
|
await this.persistConfig(config);
|
|
378
464
|
|
|
379
|
-
|
|
465
|
+
const completionLevel = failed > 0 ? 'error' : 'success';
|
|
466
|
+
logger.syncLog(completionLevel, `Sync complete for "${config.name}": ${synced}/${projects.length} repos synced, ${failed} failed in ${config.lastSyncDurationMs}ms`, 'sync');
|
|
380
467
|
} catch (err) {
|
|
381
468
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
382
469
|
config.lastSyncError = errMsg;
|
|
@@ -399,14 +486,22 @@ export class SyncManager {
|
|
|
399
486
|
project: interfaces.data.IProject,
|
|
400
487
|
sourceConn: interfaces.data.IProviderConnection,
|
|
401
488
|
targetConn: interfaces.data.IProviderConnection,
|
|
402
|
-
): Promise<
|
|
489
|
+
): Promise<{
|
|
490
|
+
targetFullPath: string;
|
|
491
|
+
targetProjectId?: string;
|
|
492
|
+
releasesSynced?: number;
|
|
493
|
+
imagesSynced?: number;
|
|
494
|
+
releaseSyncError?: string;
|
|
495
|
+
imageSyncError?: string;
|
|
496
|
+
}> {
|
|
403
497
|
const targetFullPath = this.computeTargetFullPath(
|
|
404
498
|
project.fullPath, sourceConn.groupFilter, config.targetGroupOffset,
|
|
405
499
|
);
|
|
500
|
+
const targetProviderFullPath = this.computeProviderProjectFullPath(targetConn, targetFullPath);
|
|
406
501
|
|
|
407
502
|
// Build authenticated git URLs
|
|
408
503
|
const sourceUrl = this.buildAuthUrl(sourceConn, project.fullPath);
|
|
409
|
-
const targetUrl = this.buildAuthUrl(targetConn,
|
|
504
|
+
const targetUrl = this.buildAuthUrl(targetConn, targetProviderFullPath);
|
|
410
505
|
|
|
411
506
|
// Mirror directory for this repo
|
|
412
507
|
const mirrorDir = plugins.path.join(
|
|
@@ -415,6 +510,8 @@ export class SyncManager {
|
|
|
415
510
|
this.sanitizePath(project.fullPath),
|
|
416
511
|
);
|
|
417
512
|
|
|
513
|
+
await this.reconcileProjectMapping(config, project, sourceConn, targetConn, targetFullPath);
|
|
514
|
+
|
|
418
515
|
// Ensure target group/project hierarchy exists
|
|
419
516
|
await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset);
|
|
420
517
|
|
|
@@ -422,12 +519,27 @@ export class SyncManager {
|
|
|
422
519
|
const sourceProvider = this.connectionManager.getProvider(sourceConn.id);
|
|
423
520
|
const targetProvider = this.connectionManager.getProvider(targetConn.id);
|
|
424
521
|
const apiRefsMatch = await this.refsMatchViaApi(
|
|
425
|
-
sourceProvider, targetProvider, project.fullPath,
|
|
522
|
+
sourceProvider, targetProvider, project.fullPath, targetProviderFullPath,
|
|
426
523
|
);
|
|
427
524
|
if (apiRefsMatch === true) {
|
|
428
525
|
logger.syncLog('info', `Refs match via API for ${project.fullPath}, skipping git`, 'api');
|
|
429
|
-
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath,
|
|
430
|
-
|
|
526
|
+
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetProviderFullPath);
|
|
527
|
+
const targetProject = await this.fetchProjectRaw(targetConn, targetProviderFullPath);
|
|
528
|
+
await this.persistProjectMapping(config, project, sourceConn, targetConn, targetFullPath, targetProject);
|
|
529
|
+
const artifactResult = await this.syncProjectArtifacts(
|
|
530
|
+
config,
|
|
531
|
+
sourceConn,
|
|
532
|
+
targetConn,
|
|
533
|
+
sourceProvider,
|
|
534
|
+
targetProvider,
|
|
535
|
+
project.fullPath,
|
|
536
|
+
targetFullPath,
|
|
537
|
+
);
|
|
538
|
+
return {
|
|
539
|
+
targetFullPath,
|
|
540
|
+
targetProjectId: targetProject?.id ? String(targetProject.id) : undefined,
|
|
541
|
+
...artifactResult,
|
|
542
|
+
};
|
|
431
543
|
}
|
|
432
544
|
|
|
433
545
|
// Clone or fetch from source
|
|
@@ -454,7 +566,22 @@ export class SyncManager {
|
|
|
454
566
|
const msg = err instanceof Error ? err.message : String(err);
|
|
455
567
|
if (msg.includes("couldn't find remote ref HEAD")) {
|
|
456
568
|
logger.syncLog('warn', `Skipping empty repo ${project.fullPath} (no HEAD ref)`, 'git');
|
|
457
|
-
|
|
569
|
+
const targetProject = await this.fetchProjectRaw(targetConn, targetProviderFullPath);
|
|
570
|
+
await this.persistProjectMapping(config, project, sourceConn, targetConn, targetFullPath, targetProject);
|
|
571
|
+
const artifactResult = await this.syncProjectArtifacts(
|
|
572
|
+
config,
|
|
573
|
+
sourceConn,
|
|
574
|
+
targetConn,
|
|
575
|
+
sourceProvider,
|
|
576
|
+
targetProvider,
|
|
577
|
+
project.fullPath,
|
|
578
|
+
targetFullPath,
|
|
579
|
+
);
|
|
580
|
+
return {
|
|
581
|
+
targetFullPath,
|
|
582
|
+
targetProjectId: targetProject?.id ? String(targetProject.id) : undefined,
|
|
583
|
+
...artifactResult,
|
|
584
|
+
};
|
|
458
585
|
}
|
|
459
586
|
throw err;
|
|
460
587
|
}
|
|
@@ -471,7 +598,7 @@ export class SyncManager {
|
|
|
471
598
|
const isUnrelated = await this.checkUnrelatedHistory(mirrorDir);
|
|
472
599
|
if (isUnrelated) {
|
|
473
600
|
logger.syncLog('warn', `Target "${targetFullPath}" has unrelated history — moving to obsolete`, 'git');
|
|
474
|
-
await this.moveToObsolete(targetConn,
|
|
601
|
+
await this.moveToObsolete(targetConn, targetProviderFullPath, config.targetGroupOffset);
|
|
475
602
|
// Re-create fresh target
|
|
476
603
|
await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset);
|
|
477
604
|
this.actionLog.append({
|
|
@@ -498,10 +625,10 @@ export class SyncManager {
|
|
|
498
625
|
], mirrorDir);
|
|
499
626
|
|
|
500
627
|
// Phase 2: sync default_branch now that all branches exist on target
|
|
501
|
-
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath,
|
|
628
|
+
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetProviderFullPath);
|
|
502
629
|
|
|
503
630
|
// Phase 2b: unprotect stale branches on target so --prune can delete them
|
|
504
|
-
await this.unprotectStaleBranches(targetConn,
|
|
631
|
+
await this.unprotectStaleBranches(targetConn, targetProviderFullPath, mirrorDir);
|
|
505
632
|
|
|
506
633
|
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
|
|
507
634
|
await this.runGit([
|
|
@@ -513,7 +640,754 @@ export class SyncManager {
|
|
|
513
640
|
}
|
|
514
641
|
|
|
515
642
|
// Sync project metadata (description, visibility, topics, default_branch, avatar)
|
|
516
|
-
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath,
|
|
643
|
+
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetProviderFullPath);
|
|
644
|
+
const targetProject = await this.fetchProjectRaw(targetConn, targetProviderFullPath);
|
|
645
|
+
await this.persistProjectMapping(config, project, sourceConn, targetConn, targetFullPath, targetProject);
|
|
646
|
+
const artifactResult = await this.syncProjectArtifacts(
|
|
647
|
+
config,
|
|
648
|
+
sourceConn,
|
|
649
|
+
targetConn,
|
|
650
|
+
sourceProvider,
|
|
651
|
+
targetProvider,
|
|
652
|
+
project.fullPath,
|
|
653
|
+
targetFullPath,
|
|
654
|
+
);
|
|
655
|
+
return {
|
|
656
|
+
targetFullPath,
|
|
657
|
+
targetProjectId: targetProject?.id ? String(targetProject.id) : undefined,
|
|
658
|
+
...artifactResult,
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
private async syncProjectArtifacts(
|
|
663
|
+
config: interfaces.data.ISyncConfig,
|
|
664
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
665
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
666
|
+
sourceProvider: BaseProvider,
|
|
667
|
+
targetProvider: BaseProvider,
|
|
668
|
+
sourceFullPath: string,
|
|
669
|
+
targetFullPath: string,
|
|
670
|
+
): Promise<{
|
|
671
|
+
releasesSynced?: number;
|
|
672
|
+
imagesSynced?: number;
|
|
673
|
+
releaseSyncError?: string;
|
|
674
|
+
imageSyncError?: string;
|
|
675
|
+
}> {
|
|
676
|
+
const result: {
|
|
677
|
+
releasesSynced?: number;
|
|
678
|
+
imagesSynced?: number;
|
|
679
|
+
releaseSyncError?: string;
|
|
680
|
+
imageSyncError?: string;
|
|
681
|
+
} = {};
|
|
682
|
+
|
|
683
|
+
if (config.syncReleases) {
|
|
684
|
+
try {
|
|
685
|
+
result.releasesSynced = await this.syncReleasesForProject(
|
|
686
|
+
sourceProvider,
|
|
687
|
+
targetProvider,
|
|
688
|
+
sourceFullPath,
|
|
689
|
+
this.computeProviderProjectFullPath(targetConn, targetFullPath),
|
|
690
|
+
);
|
|
691
|
+
} catch (err) {
|
|
692
|
+
result.releasesSynced = 0;
|
|
693
|
+
result.releaseSyncError = err instanceof Error ? err.message : String(err);
|
|
694
|
+
logger.syncLog('error', `Release sync failed for ${sourceFullPath}: ${result.releaseSyncError}`, 'api');
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
if (config.syncContainerImages) {
|
|
699
|
+
try {
|
|
700
|
+
result.imagesSynced = await this.syncContainerImagesForProject(
|
|
701
|
+
config,
|
|
702
|
+
sourceConn,
|
|
703
|
+
targetConn,
|
|
704
|
+
sourceProvider,
|
|
705
|
+
targetProvider,
|
|
706
|
+
sourceFullPath,
|
|
707
|
+
targetFullPath,
|
|
708
|
+
);
|
|
709
|
+
} catch (err) {
|
|
710
|
+
result.imagesSynced = 0;
|
|
711
|
+
result.imageSyncError = err instanceof Error ? err.message : String(err);
|
|
712
|
+
logger.syncLog('error', `Container image sync failed for ${sourceFullPath}: ${result.imageSyncError}`, 'api');
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
return result;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
private async syncReleasesForProject(
|
|
720
|
+
sourceProvider: BaseProvider,
|
|
721
|
+
targetProvider: BaseProvider,
|
|
722
|
+
sourceFullPath: string,
|
|
723
|
+
targetFullPath: string,
|
|
724
|
+
): Promise<number> {
|
|
725
|
+
const [sourceReleases, targetReleases] = await Promise.all([
|
|
726
|
+
sourceProvider.getReleases(sourceFullPath),
|
|
727
|
+
targetProvider.getReleases(targetFullPath),
|
|
728
|
+
]);
|
|
729
|
+
const targetByTag = new Map<string, interfaces.data.IRelease>(
|
|
730
|
+
targetReleases.map((release: interfaces.data.IRelease) => [release.tagName, release]),
|
|
731
|
+
);
|
|
732
|
+
const sourceTags = new Set<string>(
|
|
733
|
+
sourceReleases.map((release: interfaces.data.IRelease) => release.tagName),
|
|
734
|
+
);
|
|
735
|
+
|
|
736
|
+
let synced = 0;
|
|
737
|
+
for (const sourceRelease of sourceReleases) {
|
|
738
|
+
logger.syncLog('info', `Syncing release ${sourceRelease.tagName} for ${targetFullPath}`, 'api');
|
|
739
|
+
const previousTargetRelease = targetByTag.get(sourceRelease.tagName);
|
|
740
|
+
const targetRelease = await targetProvider.upsertRelease(targetFullPath, sourceRelease);
|
|
741
|
+
const previousTargetAssets = previousTargetRelease?.assets || targetRelease.assets || [];
|
|
742
|
+
const previousAssetsByName = new Map<string, interfaces.data.IReleaseAsset>();
|
|
743
|
+
for (const targetAsset of previousTargetAssets) {
|
|
744
|
+
if (!previousAssetsByName.has(targetAsset.name)) {
|
|
745
|
+
previousAssetsByName.set(targetAsset.name, targetAsset);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
const retainedAssetIds = new Set<string>();
|
|
749
|
+
const syncedAssetIds = new Set<string>();
|
|
750
|
+
const deletedAssetIds = new Set<string>();
|
|
751
|
+
|
|
752
|
+
for (const sourceAsset of sourceRelease.assets) {
|
|
753
|
+
if (sourceAsset.kind === 'source') continue;
|
|
754
|
+
const previousAsset = previousAssetsByName.get(sourceAsset.name);
|
|
755
|
+
if (previousAsset && this.canReuseTargetReleaseAsset(sourceRelease, sourceAsset, previousAsset)) {
|
|
756
|
+
if (previousAsset.id) retainedAssetIds.add(previousAsset.id);
|
|
757
|
+
continue;
|
|
758
|
+
}
|
|
759
|
+
if (sourceAsset.size && sourceAsset.size > RELEASE_ASSET_MAX_BYTES) {
|
|
760
|
+
throw new Error(`Release asset ${sourceAsset.name} is ${sourceAsset.size} bytes, exceeding the ${RELEASE_ASSET_MAX_BYTES} byte limit`);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
const syncAsset = async (): Promise<interfaces.data.IReleaseAsset> => {
|
|
764
|
+
const transfer = this.createTransferAbortSignal(
|
|
765
|
+
RELEASE_ASSET_TRANSFER_TIMEOUT_MS,
|
|
766
|
+
`Release asset transfer ${sourceFullPath}@${sourceRelease.tagName}/${sourceAsset.name}`,
|
|
767
|
+
);
|
|
768
|
+
try {
|
|
769
|
+
const transferOptions = {
|
|
770
|
+
signal: transfer.signal,
|
|
771
|
+
maxBytes: RELEASE_ASSET_MAX_BYTES,
|
|
772
|
+
};
|
|
773
|
+
const payload = await sourceProvider.downloadReleaseAsset(
|
|
774
|
+
sourceFullPath,
|
|
775
|
+
sourceRelease,
|
|
776
|
+
sourceAsset,
|
|
777
|
+
transferOptions,
|
|
778
|
+
);
|
|
779
|
+
if (payload) {
|
|
780
|
+
return await targetProvider.uploadReleaseAsset(
|
|
781
|
+
targetFullPath,
|
|
782
|
+
targetRelease,
|
|
783
|
+
sourceAsset,
|
|
784
|
+
payload,
|
|
785
|
+
transferOptions,
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
return await targetProvider.createReleaseAssetLink(targetFullPath, targetRelease, sourceAsset);
|
|
789
|
+
} finally {
|
|
790
|
+
transfer.cleanup();
|
|
791
|
+
}
|
|
792
|
+
};
|
|
793
|
+
|
|
794
|
+
let syncedAsset: interfaces.data.IReleaseAsset;
|
|
795
|
+
try {
|
|
796
|
+
syncedAsset = await syncAsset();
|
|
797
|
+
} catch (err) {
|
|
798
|
+
if (!previousAsset || !this.isReleaseAssetDuplicateError(err)) throw err;
|
|
799
|
+
await targetProvider.deleteReleaseAsset(targetFullPath, previousTargetRelease || targetRelease, previousAsset);
|
|
800
|
+
if (previousAsset.id) deletedAssetIds.add(previousAsset.id);
|
|
801
|
+
syncedAsset = await syncAsset();
|
|
802
|
+
}
|
|
803
|
+
if (syncedAsset.id) syncedAssetIds.add(syncedAsset.id);
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
for (const targetAsset of previousTargetAssets) {
|
|
807
|
+
if (targetAsset.id && deletedAssetIds.has(targetAsset.id)) {
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
if (targetAsset.id && (retainedAssetIds.has(targetAsset.id) || syncedAssetIds.has(targetAsset.id))) {
|
|
811
|
+
continue;
|
|
812
|
+
}
|
|
813
|
+
await targetProvider.deleteReleaseAsset(targetFullPath, previousTargetRelease || targetRelease, targetAsset);
|
|
814
|
+
}
|
|
815
|
+
synced++;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
for (const targetRelease of targetReleases) {
|
|
819
|
+
if (!sourceTags.has(targetRelease.tagName)) {
|
|
820
|
+
logger.syncLog('warn', `Deleting stale release ${targetRelease.tagName} from ${targetFullPath}`, 'api');
|
|
821
|
+
await targetProvider.deleteRelease(targetFullPath, targetRelease.tagName);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
return synced;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
private canReuseTargetReleaseAsset(
|
|
829
|
+
_release: interfaces.data.IRelease,
|
|
830
|
+
sourceAsset: interfaces.data.IReleaseAsset,
|
|
831
|
+
targetAsset: interfaces.data.IReleaseAsset,
|
|
832
|
+
): boolean {
|
|
833
|
+
if (sourceAsset.name !== targetAsset.name) return false;
|
|
834
|
+
if (sourceAsset.url && targetAsset.url && sourceAsset.url === targetAsset.url) return true;
|
|
835
|
+
if (sourceAsset.size !== undefined && targetAsset.size !== undefined) {
|
|
836
|
+
return sourceAsset.size === targetAsset.size;
|
|
837
|
+
}
|
|
838
|
+
return false;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
private isReleaseAssetDuplicateError(err: unknown): boolean {
|
|
842
|
+
const errMsg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
843
|
+
return errMsg.includes('already')
|
|
844
|
+
|| errMsg.includes('exists')
|
|
845
|
+
|| errMsg.includes('duplicate')
|
|
846
|
+
|| errMsg.includes('409');
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
private async syncContainerImagesForProject(
|
|
850
|
+
config: interfaces.data.ISyncConfig,
|
|
851
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
852
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
853
|
+
sourceProvider: BaseProvider,
|
|
854
|
+
targetProvider: BaseProvider,
|
|
855
|
+
sourceFullPath: string,
|
|
856
|
+
targetFullPath: string,
|
|
857
|
+
): Promise<number> {
|
|
858
|
+
await this.assertSkopeoAvailable();
|
|
859
|
+
|
|
860
|
+
const sourceRepositories = await sourceProvider.getContainerImageRepositories(sourceFullPath);
|
|
861
|
+
const targetProviderFullPath = this.computeProviderProjectFullPath(targetConn, targetFullPath);
|
|
862
|
+
const expectedTargetTags = new Set<string>();
|
|
863
|
+
let copiedTags = 0;
|
|
864
|
+
|
|
865
|
+
for (const sourceRepository of sourceRepositories) {
|
|
866
|
+
const targetImagePath = this.computeProviderContainerImagePath(
|
|
867
|
+
targetConn,
|
|
868
|
+
targetFullPath,
|
|
869
|
+
syncpath.computeTargetContainerImagePath({
|
|
870
|
+
sourceFullPath,
|
|
871
|
+
sourceGroupFilter: sourceConn.groupFilter,
|
|
872
|
+
targetGroupOffset: config.targetGroupOffset,
|
|
873
|
+
sourceImagePath: sourceRepository.path,
|
|
874
|
+
}),
|
|
875
|
+
);
|
|
876
|
+
const targetLocation = this.buildRegistryImageLocation(targetConn, targetImagePath);
|
|
877
|
+
|
|
878
|
+
for (const tag of sourceRepository.tags) {
|
|
879
|
+
expectedTargetTags.add(this.getContainerTagKey(targetImagePath, tag.name));
|
|
880
|
+
await this.copyContainerImageTag(
|
|
881
|
+
sourceConn,
|
|
882
|
+
targetConn,
|
|
883
|
+
`${sourceRepository.location}:${tag.name}`,
|
|
884
|
+
`${targetLocation}:${tag.name}`,
|
|
885
|
+
);
|
|
886
|
+
copiedTags++;
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
const targetRepositories = await targetProvider.getContainerImageRepositories(targetProviderFullPath);
|
|
891
|
+
for (const targetRepository of targetRepositories) {
|
|
892
|
+
for (const tag of targetRepository.tags) {
|
|
893
|
+
const tagKey = this.getContainerTagKey(targetRepository.path, tag.name);
|
|
894
|
+
if (!expectedTargetTags.has(tagKey)) {
|
|
895
|
+
logger.syncLog('warn', `Deleting stale image tag ${targetRepository.path}:${tag.name}`, 'api');
|
|
896
|
+
await targetProvider.deleteContainerImageTag(targetProviderFullPath, targetRepository.id, tag.name);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
return copiedTags;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
private computeProviderProjectFullPath(
|
|
905
|
+
conn: interfaces.data.IProviderConnection,
|
|
906
|
+
fullPath: string,
|
|
907
|
+
): string {
|
|
908
|
+
const normalizedPath = syncpath.normalizeSyncPath(fullPath);
|
|
909
|
+
if (conn.providerType !== 'gitea') return normalizedPath;
|
|
910
|
+
const [orgName, ...repoSegments] = normalizedPath.split('/');
|
|
911
|
+
if (!orgName) return normalizedPath;
|
|
912
|
+
if (repoSegments.length === 0) return `${conn.groupFilter || 'default'}/${orgName}`;
|
|
913
|
+
return `${orgName}/${repoSegments.join('-')}`;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
private isTargetProjectInScope(
|
|
917
|
+
conn: interfaces.data.IProviderConnection,
|
|
918
|
+
providerFullPath: string,
|
|
919
|
+
logicalScopePrefix?: string,
|
|
920
|
+
): boolean {
|
|
921
|
+
if (!logicalScopePrefix) return true;
|
|
922
|
+
const normalizedProviderPath = syncpath.normalizeSyncPath(providerFullPath).toLowerCase();
|
|
923
|
+
const normalizedScope = syncpath.normalizeSyncPath(logicalScopePrefix).toLowerCase();
|
|
924
|
+
if (conn.providerType !== 'gitea') {
|
|
925
|
+
return normalizedProviderPath === normalizedScope
|
|
926
|
+
|| normalizedProviderPath.startsWith(`${normalizedScope}/`);
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
const [scopeOwner, ...scopeRepoSegments] = normalizedScope.split('/');
|
|
930
|
+
if (!scopeOwner) return true;
|
|
931
|
+
if (scopeRepoSegments.length === 0) {
|
|
932
|
+
return normalizedProviderPath === scopeOwner
|
|
933
|
+
|| normalizedProviderPath.startsWith(`${scopeOwner}/`);
|
|
934
|
+
}
|
|
935
|
+
const repoPrefix = scopeRepoSegments.join('-');
|
|
936
|
+
return normalizedProviderPath === `${scopeOwner}/${repoPrefix}`
|
|
937
|
+
|| normalizedProviderPath.startsWith(`${scopeOwner}/${repoPrefix}-`)
|
|
938
|
+
|| normalizedProviderPath.startsWith(`${scopeOwner}/${repoPrefix}/`);
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
private computeProviderContainerImagePath(
|
|
942
|
+
conn: interfaces.data.IProviderConnection,
|
|
943
|
+
targetFullPath: string,
|
|
944
|
+
targetImagePath: string,
|
|
945
|
+
): string {
|
|
946
|
+
if (conn.providerType !== 'gitea') return syncpath.normalizeSyncPath(targetImagePath);
|
|
947
|
+
const normalizedTargetFullPath = syncpath.normalizeSyncPath(targetFullPath);
|
|
948
|
+
const normalizedImagePath = syncpath.normalizeSyncPath(targetImagePath);
|
|
949
|
+
const providerProjectPath = this.computeProviderProjectFullPath(conn, normalizedTargetFullPath);
|
|
950
|
+
if (normalizedImagePath === normalizedTargetFullPath) return providerProjectPath;
|
|
951
|
+
if (normalizedImagePath.startsWith(`${normalizedTargetFullPath}/`)) {
|
|
952
|
+
return `${providerProjectPath}/${normalizedImagePath.substring(normalizedTargetFullPath.length + 1)}`;
|
|
953
|
+
}
|
|
954
|
+
return this.computeProviderProjectFullPath(conn, normalizedImagePath);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
private getContainerTagKey(imagePath: string, tagName: string): string {
|
|
958
|
+
return `${syncpath.normalizeSyncPath(imagePath).toLowerCase()}:${tagName}`;
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
private buildRegistryImageLocation(
|
|
962
|
+
conn: interfaces.data.IProviderConnection,
|
|
963
|
+
imagePath: string,
|
|
964
|
+
): string {
|
|
965
|
+
return `${this.getRegistryHost(conn)}/${syncpath.normalizeSyncPath(imagePath)}`;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
private getRegistryHost(conn: interfaces.data.IProviderConnection): string {
|
|
969
|
+
return (conn.registryUrl || new URL(conn.baseUrl).host)
|
|
970
|
+
.replace(/^https?:\/\//, '')
|
|
971
|
+
.replace(/\/+$/, '');
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
private getRegistryAuth(conn: interfaces.data.IProviderConnection): { registry: string; username: string; password: string } | undefined {
|
|
975
|
+
const token = conn.registryToken || conn.token;
|
|
976
|
+
const username = conn.registryUsername || (conn.providerType === 'gitlab' ? 'oauth2' : undefined);
|
|
977
|
+
if (!username || !token || token === '***') return undefined;
|
|
978
|
+
return {
|
|
979
|
+
registry: this.getRegistryHost(conn),
|
|
980
|
+
username,
|
|
981
|
+
password: token,
|
|
982
|
+
};
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
private async createRegistryAuthFile(conn: interfaces.data.IProviderConnection): Promise<string | undefined> {
|
|
986
|
+
const auth = this.getRegistryAuth(conn);
|
|
987
|
+
if (!auth) return undefined;
|
|
988
|
+
const authDir = await plugins.fs.mkdtemp(plugins.path.join(plugins.os.tmpdir(), 'gitops-skopeo-auth-'));
|
|
989
|
+
const authFile = plugins.path.join(authDir, 'auth.json');
|
|
990
|
+
const authString = plugins.Buffer.from(`${auth.username}:${auth.password}`, 'utf8').toString('base64');
|
|
991
|
+
try {
|
|
992
|
+
await plugins.fs.writeFile(
|
|
993
|
+
authFile,
|
|
994
|
+
JSON.stringify({ auths: { [auth.registry]: { auth: authString } } }),
|
|
995
|
+
{ mode: 0o600 },
|
|
996
|
+
);
|
|
997
|
+
return authFile;
|
|
998
|
+
} catch (err) {
|
|
999
|
+
await plugins.fs.rm(authDir, { recursive: true, force: true });
|
|
1000
|
+
throw err;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
private async removeRegistryAuthFile(authFile: string | undefined): Promise<void> {
|
|
1005
|
+
if (!authFile) return;
|
|
1006
|
+
await plugins.fs.rm(plugins.path.dirname(authFile), { recursive: true, force: true });
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
private async assertSkopeoAvailable(): Promise<void> {
|
|
1010
|
+
if (this.skopeoAvailable === true) return;
|
|
1011
|
+
if (this.skopeoAvailable === false) {
|
|
1012
|
+
throw new Error('Container image sync requires skopeo, but the skopeo command is not available');
|
|
1013
|
+
}
|
|
1014
|
+
try {
|
|
1015
|
+
await this.runCommand('skopeo', ['--version'], undefined, { timeoutMs: SKOPEO_VERSION_TIMEOUT_MS });
|
|
1016
|
+
this.skopeoAvailable = true;
|
|
1017
|
+
} catch {
|
|
1018
|
+
this.skopeoAvailable = false;
|
|
1019
|
+
throw new Error('Container image sync requires skopeo, but the skopeo command is not available');
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
private async copyContainerImageTag(
|
|
1024
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1025
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
1026
|
+
sourceReference: string,
|
|
1027
|
+
targetReference: string,
|
|
1028
|
+
): Promise<void> {
|
|
1029
|
+
logger.syncLog('info', `Copying image ${sourceReference} -> ${targetReference}`, 'api');
|
|
1030
|
+
let sourceAuthFile: string | undefined;
|
|
1031
|
+
let targetAuthFile: string | undefined;
|
|
1032
|
+
try {
|
|
1033
|
+
sourceAuthFile = await this.createRegistryAuthFile(sourceConn);
|
|
1034
|
+
targetAuthFile = await this.createRegistryAuthFile(targetConn);
|
|
1035
|
+
const args = ['copy', '--all', '--image-parallel-copies', '3'];
|
|
1036
|
+
if (sourceAuthFile) args.push('--src-authfile', sourceAuthFile);
|
|
1037
|
+
if (targetAuthFile) args.push('--dest-authfile', targetAuthFile);
|
|
1038
|
+
args.push(`docker://${sourceReference}`, `docker://${targetReference}`);
|
|
1039
|
+
await this.withImageCopySlot(() => this.runCommand('skopeo', args, undefined, {
|
|
1040
|
+
timeoutMs: SKOPEO_COPY_TIMEOUT_MS,
|
|
1041
|
+
}));
|
|
1042
|
+
} finally {
|
|
1043
|
+
await Promise.all([
|
|
1044
|
+
this.removeRegistryAuthFile(sourceAuthFile),
|
|
1045
|
+
this.removeRegistryAuthFile(targetAuthFile),
|
|
1046
|
+
]);
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
private async withImageCopySlot<T>(fn: () => Promise<T>): Promise<T> {
|
|
1051
|
+
while (this.activeImageCopies >= IMAGE_COPY_CONCURRENCY) {
|
|
1052
|
+
if (this.stopping) {
|
|
1053
|
+
throw new Error('SyncManager is stopping');
|
|
1054
|
+
}
|
|
1055
|
+
await new Promise<void>((resolve) => this.imageCopyWaiters.push(resolve));
|
|
1056
|
+
}
|
|
1057
|
+
if (this.stopping) {
|
|
1058
|
+
throw new Error('SyncManager is stopping');
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
this.activeImageCopies++;
|
|
1062
|
+
try {
|
|
1063
|
+
return await fn();
|
|
1064
|
+
} finally {
|
|
1065
|
+
this.activeImageCopies--;
|
|
1066
|
+
const wake = this.imageCopyWaiters.shift();
|
|
1067
|
+
if (wake) wake();
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
private createTransferAbortSignal(
|
|
1072
|
+
timeoutMs: number,
|
|
1073
|
+
label: string,
|
|
1074
|
+
): { signal: AbortSignal; cleanup: () => void } {
|
|
1075
|
+
const controller = new AbortController();
|
|
1076
|
+
const abortFromStop = (): void => {
|
|
1077
|
+
if (!controller.signal.aborted) {
|
|
1078
|
+
controller.abort(new Error('SyncManager is stopping'));
|
|
1079
|
+
}
|
|
1080
|
+
};
|
|
1081
|
+
if (this.abortController.signal.aborted) {
|
|
1082
|
+
abortFromStop();
|
|
1083
|
+
} else {
|
|
1084
|
+
this.abortController.signal.addEventListener('abort', abortFromStop, { once: true });
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
const timeout = setTimeout(() => {
|
|
1088
|
+
if (!controller.signal.aborted) {
|
|
1089
|
+
controller.abort(new Error(`${label} timed out after ${timeoutMs}ms`));
|
|
1090
|
+
}
|
|
1091
|
+
}, timeoutMs);
|
|
1092
|
+
unrefTimer(timeout);
|
|
1093
|
+
|
|
1094
|
+
return {
|
|
1095
|
+
signal: controller.signal,
|
|
1096
|
+
cleanup: () => {
|
|
1097
|
+
clearTimeout(timeout);
|
|
1098
|
+
this.abortController.signal.removeEventListener('abort', abortFromStop);
|
|
1099
|
+
},
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
// ============================================================================
|
|
1104
|
+
// Stable Project Mapping
|
|
1105
|
+
// ============================================================================
|
|
1106
|
+
|
|
1107
|
+
private getProjectMappingId(
|
|
1108
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1109
|
+
sourceProjectId: string,
|
|
1110
|
+
): string {
|
|
1111
|
+
return encodeURIComponent(`${sourceConn.id}:${sourceProjectId}`);
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
private getProjectMappingKey(
|
|
1115
|
+
configId: string,
|
|
1116
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1117
|
+
sourceProjectId: string,
|
|
1118
|
+
): string {
|
|
1119
|
+
return `${SYNC_MAPPING_PREFIX}${configId}/${this.getProjectMappingId(sourceConn, sourceProjectId)}.json`;
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
private getRepoStatusKey(syncConfigId: string, sourceFullPath: string): string {
|
|
1123
|
+
const hash = this.sanitizePath(sourceFullPath).replace(/\//g, '__');
|
|
1124
|
+
return `${SYNC_STATUS_PREFIX}${syncConfigId}/${hash}.json`;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
private isApiNotFoundError(err: unknown): boolean {
|
|
1128
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1129
|
+
return errMsg.includes(': 404 -') || errMsg.includes(': 404 ');
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
private async fetchProjectRawOptional(
|
|
1133
|
+
conn: interfaces.data.IProviderConnection,
|
|
1134
|
+
fullPath: string,
|
|
1135
|
+
): Promise<any | null> {
|
|
1136
|
+
try {
|
|
1137
|
+
return await this.fetchProjectRaw(conn, fullPath);
|
|
1138
|
+
} catch (err) {
|
|
1139
|
+
if (this.isApiNotFoundError(err)) return null;
|
|
1140
|
+
throw err;
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
private getRawProjectId(rawProject: any): string | undefined {
|
|
1145
|
+
return rawProject?.id ? String(rawProject.id) : undefined;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
private async getProjectMapping(
|
|
1149
|
+
config: interfaces.data.ISyncConfig,
|
|
1150
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1151
|
+
project: interfaces.data.IProject,
|
|
1152
|
+
): Promise<interfaces.data.ISyncProjectMapping | null> {
|
|
1153
|
+
const key = this.getProjectMappingKey(config.id, sourceConn, project.id);
|
|
1154
|
+
const direct = await this.storageManager.getJSON<interfaces.data.ISyncProjectMapping>(key);
|
|
1155
|
+
if (direct) return direct;
|
|
1156
|
+
|
|
1157
|
+
const statusFromCurrentPath = await this.storageManager.getJSON<interfaces.data.ISyncRepoStatus>(
|
|
1158
|
+
this.getRepoStatusKey(config.id, project.fullPath),
|
|
1159
|
+
);
|
|
1160
|
+
if (statusFromCurrentPath?.targetFullPath) {
|
|
1161
|
+
return {
|
|
1162
|
+
id: this.getProjectMappingId(sourceConn, project.id),
|
|
1163
|
+
syncConfigId: config.id,
|
|
1164
|
+
sourceConnectionId: sourceConn.id,
|
|
1165
|
+
sourceProjectId: project.id,
|
|
1166
|
+
sourceFullPath: project.fullPath,
|
|
1167
|
+
targetConnectionId: config.targetConnectionId,
|
|
1168
|
+
targetFullPath: statusFromCurrentPath.targetFullPath,
|
|
1169
|
+
targetProjectId: statusFromCurrentPath.targetProjectId,
|
|
1170
|
+
updatedAt: Date.now(),
|
|
1171
|
+
};
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
const statusKeys = await this.storageManager.list(`${SYNC_STATUS_PREFIX}${config.id}/`);
|
|
1175
|
+
for (const statusKey of statusKeys) {
|
|
1176
|
+
const status = await this.storageManager.getJSON<interfaces.data.ISyncRepoStatus>(statusKey);
|
|
1177
|
+
if (status?.sourceProjectId === project.id && status.targetFullPath) {
|
|
1178
|
+
return {
|
|
1179
|
+
id: this.getProjectMappingId(sourceConn, project.id),
|
|
1180
|
+
syncConfigId: config.id,
|
|
1181
|
+
sourceConnectionId: sourceConn.id,
|
|
1182
|
+
sourceProjectId: project.id,
|
|
1183
|
+
sourceFullPath: status.sourceFullPath,
|
|
1184
|
+
targetConnectionId: config.targetConnectionId,
|
|
1185
|
+
targetFullPath: status.targetFullPath,
|
|
1186
|
+
targetProjectId: status.targetProjectId,
|
|
1187
|
+
updatedAt: Date.now(),
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
return null;
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
private async reconcileProjectMapping(
|
|
1196
|
+
config: interfaces.data.ISyncConfig,
|
|
1197
|
+
project: interfaces.data.IProject,
|
|
1198
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1199
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
1200
|
+
targetFullPath: string,
|
|
1201
|
+
): Promise<void> {
|
|
1202
|
+
const mapping = await this.getProjectMapping(config, sourceConn, project);
|
|
1203
|
+
if (!mapping) return;
|
|
1204
|
+
|
|
1205
|
+
const previousTargetPath = syncpath.normalizeSyncPath(mapping.targetFullPath);
|
|
1206
|
+
const nextTargetPath = syncpath.normalizeSyncPath(targetFullPath);
|
|
1207
|
+
if (previousTargetPath === nextTargetPath) return;
|
|
1208
|
+
|
|
1209
|
+
logger.syncLog(
|
|
1210
|
+
'info',
|
|
1211
|
+
`Project mapping moved for source ID ${project.id}: ${mapping.targetFullPath} -> ${targetFullPath}`,
|
|
1212
|
+
'sync',
|
|
1213
|
+
);
|
|
1214
|
+
|
|
1215
|
+
const targetProjectId = await this.moveMappedProjectTarget(
|
|
1216
|
+
config,
|
|
1217
|
+
project,
|
|
1218
|
+
sourceConn,
|
|
1219
|
+
targetConn,
|
|
1220
|
+
mapping,
|
|
1221
|
+
targetFullPath,
|
|
1222
|
+
);
|
|
1223
|
+
|
|
1224
|
+
await this.persistProjectMapping(config, project, sourceConn, targetConn, targetFullPath, {
|
|
1225
|
+
id: targetProjectId,
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
private async moveMappedProjectTarget(
|
|
1230
|
+
config: interfaces.data.ISyncConfig,
|
|
1231
|
+
project: interfaces.data.IProject,
|
|
1232
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1233
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
1234
|
+
mapping: interfaces.data.ISyncProjectMapping,
|
|
1235
|
+
targetFullPath: string,
|
|
1236
|
+
): Promise<string | undefined> {
|
|
1237
|
+
const nextProviderFullPath = this.computeProviderProjectFullPath(targetConn, targetFullPath);
|
|
1238
|
+
const previousProviderFullPath = this.computeProviderProjectFullPath(targetConn, mapping.targetFullPath);
|
|
1239
|
+
const existingTarget = await this.fetchProjectRawOptional(targetConn, nextProviderFullPath);
|
|
1240
|
+
const existingTargetId = this.getRawProjectId(existingTarget);
|
|
1241
|
+
if (existingTarget) {
|
|
1242
|
+
if (mapping.targetProjectId && existingTargetId === mapping.targetProjectId) {
|
|
1243
|
+
return existingTargetId;
|
|
1244
|
+
}
|
|
1245
|
+
throw new Error(
|
|
1246
|
+
`Cannot move mapped project ${project.fullPath}: target path "${targetFullPath}" already exists`,
|
|
1247
|
+
);
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
const previousTarget = await this.fetchProjectRawOptional(targetConn, previousProviderFullPath);
|
|
1251
|
+
const previousTargetId = this.getRawProjectId(previousTarget);
|
|
1252
|
+
if (!previousTarget) {
|
|
1253
|
+
logger.syncLog(
|
|
1254
|
+
'warn',
|
|
1255
|
+
`Stored target path "${mapping.targetFullPath}" no longer exists for source ID ${project.id}; creating fresh target`,
|
|
1256
|
+
'sync',
|
|
1257
|
+
);
|
|
1258
|
+
return undefined;
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
if (mapping.targetProjectId && previousTargetId && previousTargetId !== mapping.targetProjectId) {
|
|
1262
|
+
throw new Error(
|
|
1263
|
+
`Cannot move mapped project ${project.fullPath}: stored target path "${mapping.targetFullPath}" belongs to a different project`,
|
|
1264
|
+
);
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
await this.moveProjectPath(
|
|
1268
|
+
config,
|
|
1269
|
+
project,
|
|
1270
|
+
sourceConn,
|
|
1271
|
+
targetConn,
|
|
1272
|
+
previousTarget,
|
|
1273
|
+
mapping.targetFullPath,
|
|
1274
|
+
targetFullPath,
|
|
1275
|
+
);
|
|
1276
|
+
return previousTargetId;
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
private async moveProjectPath(
|
|
1280
|
+
config: interfaces.data.ISyncConfig,
|
|
1281
|
+
project: interfaces.data.IProject,
|
|
1282
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1283
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
1284
|
+
rawProject: any,
|
|
1285
|
+
previousTargetFullPath: string,
|
|
1286
|
+
nextTargetFullPath: string,
|
|
1287
|
+
): Promise<void> {
|
|
1288
|
+
const segments = nextTargetFullPath.split('/');
|
|
1289
|
+
const projectPath = segments.pop()!;
|
|
1290
|
+
const groupSegments = segments;
|
|
1291
|
+
|
|
1292
|
+
if (targetConn.providerType === 'gitlab') {
|
|
1293
|
+
const namespaceId = await this.ensureGitLabGroupPath(
|
|
1294
|
+
targetConn,
|
|
1295
|
+
groupSegments,
|
|
1296
|
+
sourceConn,
|
|
1297
|
+
sourceConn.groupFilter,
|
|
1298
|
+
config.targetGroupOffset,
|
|
1299
|
+
);
|
|
1300
|
+
const rawProjectId = this.getRawProjectId(rawProject);
|
|
1301
|
+
if (!rawProjectId) {
|
|
1302
|
+
throw new Error(`Cannot move ${previousTargetFullPath}: missing target project ID`);
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
if (namespaceId && String(rawProject.namespace?.id) !== String(namespaceId)) {
|
|
1306
|
+
await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${rawProjectId}/transfer`, {
|
|
1307
|
+
namespace: namespaceId,
|
|
1308
|
+
});
|
|
1309
|
+
}
|
|
1310
|
+
await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${rawProjectId}`, {
|
|
1311
|
+
name: project.name,
|
|
1312
|
+
path: projectPath,
|
|
1313
|
+
});
|
|
1314
|
+
logger.syncLog('success', `Moved GitLab project ${previousTargetFullPath} to ${nextTargetFullPath}`, 'api');
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
const previousRepo = this.getGiteaRepoCoordinates(previousTargetFullPath, targetConn);
|
|
1319
|
+
const nextRepo = this.getGiteaRepoCoordinates(nextTargetFullPath, targetConn);
|
|
1320
|
+
const client = new plugins.giteaClient.GiteaClient(targetConn.baseUrl, targetConn.token);
|
|
1321
|
+
|
|
1322
|
+
try {
|
|
1323
|
+
await client.getOrg(nextRepo.owner);
|
|
1324
|
+
} catch {
|
|
1325
|
+
await client.createOrg(nextRepo.owner, { visibility: 'private' });
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
if (previousRepo.owner !== nextRepo.owner) {
|
|
1329
|
+
await this.rawApiCall(
|
|
1330
|
+
targetConn,
|
|
1331
|
+
'POST',
|
|
1332
|
+
`/api/v1/repos/${encodeURIComponent(previousRepo.owner)}/${encodeURIComponent(previousRepo.repo)}/transfer`,
|
|
1333
|
+
{ new_owner: nextRepo.owner },
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
if (previousRepo.repo !== nextRepo.repo) {
|
|
1338
|
+
await this.rawApiCall(
|
|
1339
|
+
targetConn,
|
|
1340
|
+
'PATCH',
|
|
1341
|
+
`/api/v1/repos/${encodeURIComponent(nextRepo.owner)}/${encodeURIComponent(previousRepo.repo)}`,
|
|
1342
|
+
{ name: nextRepo.repo },
|
|
1343
|
+
);
|
|
1344
|
+
}
|
|
1345
|
+
logger.syncLog('success', `Moved Gitea repo ${previousTargetFullPath} to ${nextTargetFullPath}`, 'api');
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
private getGiteaRepoCoordinates(
|
|
1349
|
+
fullPath: string,
|
|
1350
|
+
conn: interfaces.data.IProviderConnection,
|
|
1351
|
+
): { owner: string; repo: string } {
|
|
1352
|
+
const segments = fullPath.split('/');
|
|
1353
|
+
const projectName = segments.pop()!;
|
|
1354
|
+
const groupSegments = segments;
|
|
1355
|
+
return {
|
|
1356
|
+
owner: groupSegments[0] || conn.groupFilter || 'default',
|
|
1357
|
+
repo: groupSegments.length > 1
|
|
1358
|
+
? [...groupSegments.slice(1), projectName].join('-')
|
|
1359
|
+
: projectName,
|
|
1360
|
+
};
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
private async persistProjectMapping(
|
|
1364
|
+
config: interfaces.data.ISyncConfig,
|
|
1365
|
+
project: interfaces.data.IProject,
|
|
1366
|
+
sourceConn: interfaces.data.IProviderConnection,
|
|
1367
|
+
targetConn: interfaces.data.IProviderConnection,
|
|
1368
|
+
targetFullPath: string,
|
|
1369
|
+
targetRawProject?: any,
|
|
1370
|
+
): Promise<void> {
|
|
1371
|
+
const previousMapping = await this.getProjectMapping(config, sourceConn, project);
|
|
1372
|
+
if (previousMapping && previousMapping.sourceFullPath !== project.fullPath) {
|
|
1373
|
+
await this.storageManager.delete(this.getRepoStatusKey(config.id, previousMapping.sourceFullPath));
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
const mapping: interfaces.data.ISyncProjectMapping = {
|
|
1377
|
+
id: this.getProjectMappingId(sourceConn, project.id),
|
|
1378
|
+
syncConfigId: config.id,
|
|
1379
|
+
sourceConnectionId: sourceConn.id,
|
|
1380
|
+
sourceProjectId: project.id,
|
|
1381
|
+
sourceFullPath: project.fullPath,
|
|
1382
|
+
targetConnectionId: targetConn.id,
|
|
1383
|
+
targetFullPath,
|
|
1384
|
+
targetProjectId: this.getRawProjectId(targetRawProject),
|
|
1385
|
+
updatedAt: Date.now(),
|
|
1386
|
+
};
|
|
1387
|
+
await this.storageManager.setJSON(
|
|
1388
|
+
this.getProjectMappingKey(config.id, sourceConn, project.id),
|
|
1389
|
+
mapping,
|
|
1390
|
+
);
|
|
517
1391
|
}
|
|
518
1392
|
|
|
519
1393
|
// ============================================================================
|
|
@@ -549,8 +1423,51 @@ export class SyncManager {
|
|
|
549
1423
|
targetGroupOffset?: string,
|
|
550
1424
|
): Promise<void> {
|
|
551
1425
|
const client = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token);
|
|
1426
|
+
const parentId = await this.ensureGitLabGroupPath(
|
|
1427
|
+
conn,
|
|
1428
|
+
groupSegments,
|
|
1429
|
+
sourceConn,
|
|
1430
|
+
sourceGroupFilter,
|
|
1431
|
+
targetGroupOffset,
|
|
1432
|
+
);
|
|
1433
|
+
|
|
1434
|
+
// Create the project if it doesn't exist
|
|
1435
|
+
const projectPath = groupSegments.length > 0
|
|
1436
|
+
? `${groupSegments.join('/')}/${projectName}`
|
|
1437
|
+
: projectName;
|
|
1438
|
+
|
|
1439
|
+
try {
|
|
1440
|
+
// Check if project exists by path
|
|
1441
|
+
await client.getGroup(projectPath);
|
|
1442
|
+
// If this succeeds, it's actually a group, not a project... unlikely but handle
|
|
1443
|
+
} catch {
|
|
1444
|
+
// Project doesn't exist as a group path; try creating it
|
|
1445
|
+
try {
|
|
1446
|
+
await client.createProject(projectName, {
|
|
1447
|
+
path: projectName,
|
|
1448
|
+
namespaceId: parentId,
|
|
1449
|
+
description: sourceProject.description,
|
|
1450
|
+
visibility: sourceProject.visibility || 'private',
|
|
1451
|
+
});
|
|
1452
|
+
logger.info(`Created GitLab project: ${projectPath}`);
|
|
1453
|
+
} catch (createErr: any) {
|
|
1454
|
+
// Already exists is fine
|
|
1455
|
+
if (!String(createErr).includes('409') && !String(createErr).includes('already been taken')) {
|
|
1456
|
+
throw createErr;
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
private async ensureGitLabGroupPath(
|
|
1463
|
+
conn: interfaces.data.IProviderConnection,
|
|
1464
|
+
groupSegments: string[],
|
|
1465
|
+
sourceConn?: interfaces.data.IProviderConnection,
|
|
1466
|
+
sourceGroupFilter?: string,
|
|
1467
|
+
targetGroupOffset?: string,
|
|
1468
|
+
): Promise<number | undefined> {
|
|
1469
|
+
const client = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token);
|
|
552
1470
|
|
|
553
|
-
// Walk group hierarchy top-down, creating each if needed
|
|
554
1471
|
let parentId: number | undefined = undefined;
|
|
555
1472
|
let currentPath = '';
|
|
556
1473
|
|
|
@@ -560,13 +1477,11 @@ export class SyncManager {
|
|
|
560
1477
|
const group = await client.getGroup(currentPath);
|
|
561
1478
|
parentId = group.id;
|
|
562
1479
|
} catch {
|
|
563
|
-
// Group doesn't exist — create it
|
|
564
1480
|
try {
|
|
565
1481
|
const newGroup = await client.createGroup(segment, segment, parentId);
|
|
566
1482
|
parentId = newGroup.id;
|
|
567
1483
|
logger.info(`Created GitLab group: ${currentPath}`);
|
|
568
1484
|
} catch (createErr: any) {
|
|
569
|
-
// 409 = already exists (race condition), try fetching again
|
|
570
1485
|
if (String(createErr).includes('409') || String(createErr).includes('already')) {
|
|
571
1486
|
const group = await client.getGroup(currentPath);
|
|
572
1487
|
parentId = group.id;
|
|
@@ -576,7 +1491,6 @@ export class SyncManager {
|
|
|
576
1491
|
}
|
|
577
1492
|
}
|
|
578
1493
|
|
|
579
|
-
// Sync group metadata from source (once per group per sync cycle)
|
|
580
1494
|
if (sourceConn && !this.syncedGroupMeta.has(currentPath)) {
|
|
581
1495
|
const sourceGroupPath = this.reverseTargetGroupPath(currentPath, sourceGroupFilter, targetGroupOffset);
|
|
582
1496
|
if (sourceGroupPath) {
|
|
@@ -586,32 +1500,7 @@ export class SyncManager {
|
|
|
586
1500
|
}
|
|
587
1501
|
}
|
|
588
1502
|
|
|
589
|
-
|
|
590
|
-
const projectPath = groupSegments.length > 0
|
|
591
|
-
? `${groupSegments.join('/')}/${projectName}`
|
|
592
|
-
: projectName;
|
|
593
|
-
|
|
594
|
-
try {
|
|
595
|
-
// Check if project exists by path
|
|
596
|
-
await client.getGroup(projectPath);
|
|
597
|
-
// If this succeeds, it's actually a group, not a project... unlikely but handle
|
|
598
|
-
} catch {
|
|
599
|
-
// Project doesn't exist as a group path; try creating it
|
|
600
|
-
try {
|
|
601
|
-
await client.createProject(projectName, {
|
|
602
|
-
path: projectName,
|
|
603
|
-
namespaceId: parentId,
|
|
604
|
-
description: sourceProject.description,
|
|
605
|
-
visibility: sourceProject.visibility || 'private',
|
|
606
|
-
});
|
|
607
|
-
logger.info(`Created GitLab project: ${projectPath}`);
|
|
608
|
-
} catch (createErr: any) {
|
|
609
|
-
// Already exists is fine
|
|
610
|
-
if (!String(createErr).includes('409') && !String(createErr).includes('already been taken')) {
|
|
611
|
-
throw createErr;
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
}
|
|
1503
|
+
return parentId;
|
|
615
1504
|
}
|
|
616
1505
|
|
|
617
1506
|
private async ensureGiteaTarget(
|
|
@@ -686,7 +1575,7 @@ export class SyncManager {
|
|
|
686
1575
|
const targetFullPath = this.computeTargetFullPath(
|
|
687
1576
|
project.fullPath, sourceConn.groupFilter, config.targetGroupOffset,
|
|
688
1577
|
);
|
|
689
|
-
expectedTargetPaths.add(targetFullPath.toLowerCase());
|
|
1578
|
+
expectedTargetPaths.add(this.computeProviderProjectFullPath(targetConn, targetFullPath).toLowerCase());
|
|
690
1579
|
}
|
|
691
1580
|
|
|
692
1581
|
// Scope prefix — only delete repos under this prefix
|
|
@@ -700,7 +1589,7 @@ export class SyncManager {
|
|
|
700
1589
|
for (const targetProject of targetProjects) {
|
|
701
1590
|
if (this.isObsoletePath(targetProject.fullPath)) continue;
|
|
702
1591
|
// Skip repos outside our managed prefix
|
|
703
|
-
if (
|
|
1592
|
+
if (!this.isTargetProjectInScope(targetConn, targetProject.fullPath, scopePrefix)) {
|
|
704
1593
|
continue;
|
|
705
1594
|
}
|
|
706
1595
|
if (!expectedTargetPaths.has(targetProject.fullPath.toLowerCase())) {
|
|
@@ -857,7 +1746,8 @@ export class SyncManager {
|
|
|
857
1746
|
const allRepos: any[] = [];
|
|
858
1747
|
let page = 1;
|
|
859
1748
|
const perPage = 50;
|
|
860
|
-
|
|
1749
|
+
const maxPages = 1000;
|
|
1750
|
+
while (page <= maxPages) {
|
|
861
1751
|
const repos = await this.rawApiCall(
|
|
862
1752
|
targetConn, 'GET',
|
|
863
1753
|
`/api/v1/orgs/${encodeURIComponent(orgName)}/repos?page=${page}&limit=${perPage}`,
|
|
@@ -867,6 +1757,9 @@ export class SyncManager {
|
|
|
867
1757
|
if (repoList.length < perPage) break;
|
|
868
1758
|
page++;
|
|
869
1759
|
}
|
|
1760
|
+
if (page > maxPages) {
|
|
1761
|
+
throw new Error(`Gitea org repo pagination exceeded ${maxPages} pages for ${orgName}`);
|
|
1762
|
+
}
|
|
870
1763
|
|
|
871
1764
|
// Move each repo to obsolete
|
|
872
1765
|
for (const repo of allRepos) {
|
|
@@ -914,11 +1807,7 @@ export class SyncManager {
|
|
|
914
1807
|
}
|
|
915
1808
|
|
|
916
1809
|
private computeRelativePath(fullPath: string, groupFilter?: string): string {
|
|
917
|
-
|
|
918
|
-
if (fullPath.startsWith(groupFilter + '/')) {
|
|
919
|
-
return fullPath.substring(groupFilter.length + 1);
|
|
920
|
-
}
|
|
921
|
-
return fullPath;
|
|
1810
|
+
return syncpath.computeRelativePath(fullPath, groupFilter);
|
|
922
1811
|
}
|
|
923
1812
|
|
|
924
1813
|
/**
|
|
@@ -930,27 +1819,7 @@ export class SyncManager {
|
|
|
930
1819
|
sourceGroupFilter?: string,
|
|
931
1820
|
targetGroupOffset?: string,
|
|
932
1821
|
): string | null {
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
// Strip the target offset prefix
|
|
936
|
-
if (targetGroupOffset) {
|
|
937
|
-
if (targetGroupPath === targetGroupOffset) {
|
|
938
|
-
// This IS the offset group itself, not a source-derived group
|
|
939
|
-
return null;
|
|
940
|
-
}
|
|
941
|
-
if (targetGroupPath.startsWith(targetGroupOffset + '/')) {
|
|
942
|
-
relativePath = targetGroupPath.substring(targetGroupOffset.length + 1);
|
|
943
|
-
} else {
|
|
944
|
-
// Target path is not under the offset — can't reverse-map
|
|
945
|
-
return null;
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
// Re-add the source group filter prefix
|
|
950
|
-
if (sourceGroupFilter) {
|
|
951
|
-
return `${sourceGroupFilter}/${relativePath}`;
|
|
952
|
-
}
|
|
953
|
-
return relativePath;
|
|
1822
|
+
return syncpath.reverseTargetGroupPath(targetGroupPath, sourceGroupFilter, targetGroupOffset);
|
|
954
1823
|
}
|
|
955
1824
|
|
|
956
1825
|
private computeTargetFullPath(
|
|
@@ -958,8 +1827,7 @@ export class SyncManager {
|
|
|
958
1827
|
sourceGroupFilter?: string,
|
|
959
1828
|
targetGroupOffset?: string,
|
|
960
1829
|
): string {
|
|
961
|
-
|
|
962
|
-
return targetGroupOffset ? `${targetGroupOffset}/${relativePath}` : relativePath;
|
|
1830
|
+
return syncpath.computeTargetFullPath({ sourceFullPath, sourceGroupFilter, targetGroupOffset });
|
|
963
1831
|
}
|
|
964
1832
|
|
|
965
1833
|
/**
|
|
@@ -1019,12 +1887,20 @@ export class SyncManager {
|
|
|
1019
1887
|
} else {
|
|
1020
1888
|
headers['Authorization'] = `token ${conn.token}`;
|
|
1021
1889
|
}
|
|
1022
|
-
const
|
|
1023
|
-
|
|
1024
|
-
await
|
|
1025
|
-
|
|
1890
|
+
const apiSignal = this.createTransferAbortSignal(RAW_API_TIMEOUT_MS, `GET ${url}`);
|
|
1891
|
+
try {
|
|
1892
|
+
const resp = await fetch(url, {
|
|
1893
|
+
headers,
|
|
1894
|
+
signal: apiSignal.signal,
|
|
1895
|
+
});
|
|
1896
|
+
if (!resp.ok) {
|
|
1897
|
+
await resp.body?.cancel();
|
|
1898
|
+
return null;
|
|
1899
|
+
}
|
|
1900
|
+
return new Uint8Array(await resp.arrayBuffer());
|
|
1901
|
+
} finally {
|
|
1902
|
+
apiSignal.cleanup();
|
|
1026
1903
|
}
|
|
1027
|
-
return new Uint8Array(await resp.arrayBuffer());
|
|
1028
1904
|
} catch {
|
|
1029
1905
|
return null;
|
|
1030
1906
|
}
|
|
@@ -1048,15 +1924,25 @@ export class SyncManager {
|
|
|
1048
1924
|
headers['Authorization'] = `token ${conn.token}`;
|
|
1049
1925
|
}
|
|
1050
1926
|
// Do NOT set Content-Type — let fetch set the multipart boundary
|
|
1051
|
-
const
|
|
1052
|
-
if (!resp.ok) {
|
|
1053
|
-
const text = await resp.text();
|
|
1054
|
-
throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`);
|
|
1055
|
-
}
|
|
1927
|
+
const apiSignal = this.createTransferAbortSignal(RAW_API_TIMEOUT_MS, `${method} ${apiPath}`);
|
|
1056
1928
|
try {
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1929
|
+
const resp = await fetch(url, {
|
|
1930
|
+
method,
|
|
1931
|
+
headers,
|
|
1932
|
+
body: formData,
|
|
1933
|
+
signal: apiSignal.signal,
|
|
1934
|
+
});
|
|
1935
|
+
if (!resp.ok) {
|
|
1936
|
+
const text = await resp.text();
|
|
1937
|
+
throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`);
|
|
1938
|
+
}
|
|
1939
|
+
try {
|
|
1940
|
+
return await resp.json();
|
|
1941
|
+
} catch {
|
|
1942
|
+
return undefined;
|
|
1943
|
+
}
|
|
1944
|
+
} finally {
|
|
1945
|
+
apiSignal.cleanup();
|
|
1060
1946
|
}
|
|
1061
1947
|
}
|
|
1062
1948
|
|
|
@@ -1669,19 +2555,26 @@ export class SyncManager {
|
|
|
1669
2555
|
} else {
|
|
1670
2556
|
headers['Authorization'] = `token ${conn.token}`;
|
|
1671
2557
|
}
|
|
1672
|
-
const
|
|
1673
|
-
|
|
1674
|
-
headers,
|
|
1675
|
-
body: body ? JSON.stringify(body) : undefined,
|
|
1676
|
-
});
|
|
1677
|
-
if (!resp.ok) {
|
|
1678
|
-
const text = await resp.text();
|
|
1679
|
-
throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`);
|
|
1680
|
-
}
|
|
2558
|
+
const apiSignal = this.createTransferAbortSignal(RAW_API_TIMEOUT_MS, `${method} ${apiPath}`);
|
|
2559
|
+
let resp!: Response;
|
|
1681
2560
|
try {
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
2561
|
+
resp = await fetch(url, {
|
|
2562
|
+
method,
|
|
2563
|
+
headers,
|
|
2564
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
2565
|
+
signal: apiSignal.signal,
|
|
2566
|
+
});
|
|
2567
|
+
if (!resp.ok) {
|
|
2568
|
+
const text = await resp.text();
|
|
2569
|
+
throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`);
|
|
2570
|
+
}
|
|
2571
|
+
try {
|
|
2572
|
+
return await resp.json();
|
|
2573
|
+
} catch {
|
|
2574
|
+
return undefined;
|
|
2575
|
+
}
|
|
2576
|
+
} finally {
|
|
2577
|
+
apiSignal.cleanup();
|
|
1685
2578
|
}
|
|
1686
2579
|
}
|
|
1687
2580
|
|
|
@@ -1873,8 +2766,8 @@ export class SyncManager {
|
|
|
1873
2766
|
provider.getTags(fullPath),
|
|
1874
2767
|
]);
|
|
1875
2768
|
return {
|
|
1876
|
-
branches: new Map(branches.map((b) => [b.name, b.commitSha])),
|
|
1877
|
-
tags: new Map(tags.map((t) => [t.name, t.commitSha])),
|
|
2769
|
+
branches: new Map(branches.map((b: interfaces.data.IBranch) => [b.name, b.commitSha])),
|
|
2770
|
+
tags: new Map(tags.map((t: interfaces.data.ITag) => [t.name, t.commitSha])),
|
|
1878
2771
|
};
|
|
1879
2772
|
} catch {
|
|
1880
2773
|
return null;
|
|
@@ -2011,7 +2904,7 @@ export class SyncManager {
|
|
|
2011
2904
|
|
|
2012
2905
|
child.stdout.on('data', (chunk: Uint8Array) => stdoutChunks.push(chunk));
|
|
2013
2906
|
child.stderr.on('data', (chunk: Uint8Array) => stderrChunks.push(chunk));
|
|
2014
|
-
child.on('error', (err) => finish(() => reject(err)));
|
|
2907
|
+
child.on('error', (err: Error) => finish(() => reject(err)));
|
|
2015
2908
|
child.on('close', (code: number | null, signal: NodeJS.Signals | null) => finish(() => {
|
|
2016
2909
|
const stderr = plugins.Buffer.concat(stderrChunks).toString('utf8');
|
|
2017
2910
|
if (code !== 0) {
|
|
@@ -2024,6 +2917,89 @@ export class SyncManager {
|
|
|
2024
2917
|
});
|
|
2025
2918
|
}
|
|
2026
2919
|
|
|
2920
|
+
private async runCommand(
|
|
2921
|
+
command: string,
|
|
2922
|
+
args: string[],
|
|
2923
|
+
cwd?: string,
|
|
2924
|
+
options: IRunCommandOptions = {},
|
|
2925
|
+
): Promise<string> {
|
|
2926
|
+
if (this.stopping) {
|
|
2927
|
+
throw new Error('SyncManager is stopping');
|
|
2928
|
+
}
|
|
2929
|
+
|
|
2930
|
+
return await new Promise<string>((resolve, reject) => {
|
|
2931
|
+
const child = plugins.childProcess.spawn(command, args, {
|
|
2932
|
+
cwd,
|
|
2933
|
+
env: { ...process.env, ...options.env },
|
|
2934
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
2935
|
+
});
|
|
2936
|
+
this.activeGitChildren.add(child);
|
|
2937
|
+
const stdoutChunks: Uint8Array[] = [];
|
|
2938
|
+
const stderrChunks: Uint8Array[] = [];
|
|
2939
|
+
let stdoutBytes = 0;
|
|
2940
|
+
let stderrBytes = 0;
|
|
2941
|
+
let settled = false;
|
|
2942
|
+
let timedOut = false;
|
|
2943
|
+
let timeoutTimer: ReturnType<typeof setTimeout> | undefined;
|
|
2944
|
+
let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
|
|
2945
|
+
|
|
2946
|
+
const appendChunk = (chunks: Uint8Array[], chunk: Uint8Array, currentBytes: number): number => {
|
|
2947
|
+
const nextBytes = currentBytes + chunk.byteLength;
|
|
2948
|
+
const remainingBytes = MAX_COMMAND_OUTPUT_BYTES - currentBytes;
|
|
2949
|
+
if (remainingBytes > 0) {
|
|
2950
|
+
chunks.push(chunk.byteLength <= remainingBytes ? chunk : chunk.subarray(0, remainingBytes));
|
|
2951
|
+
}
|
|
2952
|
+
return nextBytes;
|
|
2953
|
+
};
|
|
2954
|
+
|
|
2955
|
+
const finish = (callback: () => void): void => {
|
|
2956
|
+
if (settled) return;
|
|
2957
|
+
settled = true;
|
|
2958
|
+
if (timeoutTimer) clearTimeout(timeoutTimer);
|
|
2959
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
2960
|
+
this.activeGitChildren.delete(child);
|
|
2961
|
+
callback();
|
|
2962
|
+
};
|
|
2963
|
+
|
|
2964
|
+
if (options.timeoutMs) {
|
|
2965
|
+
timeoutTimer = setTimeout(() => {
|
|
2966
|
+
timedOut = true;
|
|
2967
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
2968
|
+
child.kill('SIGTERM');
|
|
2969
|
+
forceKillTimer = setTimeout(() => {
|
|
2970
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
2971
|
+
child.kill('SIGKILL');
|
|
2972
|
+
}
|
|
2973
|
+
}, 5000);
|
|
2974
|
+
unrefTimer(forceKillTimer);
|
|
2975
|
+
}
|
|
2976
|
+
}, options.timeoutMs);
|
|
2977
|
+
unrefTimer(timeoutTimer);
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
child.stdout.on('data', (chunk: Uint8Array) => {
|
|
2981
|
+
stdoutBytes = appendChunk(stdoutChunks, chunk, stdoutBytes);
|
|
2982
|
+
});
|
|
2983
|
+
child.stderr.on('data', (chunk: Uint8Array) => {
|
|
2984
|
+
stderrBytes = appendChunk(stderrChunks, chunk, stderrBytes);
|
|
2985
|
+
});
|
|
2986
|
+
child.on('error', (err: Error) => finish(() => reject(err)));
|
|
2987
|
+
child.on('close', (code: number | null, signal: NodeJS.Signals | null) => finish(() => {
|
|
2988
|
+
const stderr = plugins.Buffer.concat(stderrChunks).toString('utf8');
|
|
2989
|
+
if (timedOut) {
|
|
2990
|
+
reject(new Error(`${command} timed out after ${options.timeoutMs}ms: ${stderr.trim()}`));
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
if (code !== 0) {
|
|
2994
|
+
const exitInfo = signal ? `signal ${signal}` : `code ${code}`;
|
|
2995
|
+
reject(new Error(`${command} failed with ${exitInfo}: ${stderr.trim()}`));
|
|
2996
|
+
return;
|
|
2997
|
+
}
|
|
2998
|
+
resolve(plugins.Buffer.concat(stdoutChunks).toString('utf8'));
|
|
2999
|
+
}));
|
|
3000
|
+
});
|
|
3001
|
+
}
|
|
3002
|
+
|
|
2027
3003
|
// ============================================================================
|
|
2028
3004
|
// Persistence
|
|
2029
3005
|
// ============================================================================
|
|
@@ -2060,7 +3036,7 @@ export class SyncManager {
|
|
|
2060
3036
|
updates: Partial<interfaces.data.ISyncRepoStatus>,
|
|
2061
3037
|
): Promise<void> {
|
|
2062
3038
|
const hash = this.sanitizePath(sourceFullPath).replace(/\//g, '__');
|
|
2063
|
-
const key =
|
|
3039
|
+
const key = this.getRepoStatusKey(syncConfigId, sourceFullPath);
|
|
2064
3040
|
let status = await this.storageManager.getJSON<interfaces.data.ISyncRepoStatus>(key);
|
|
2065
3041
|
if (!status) {
|
|
2066
3042
|
status = {
|