ghcr-cleanup-manager 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +356 -0
- package/LICENSE +21 -0
- package/README.md +262 -0
- package/dist/cleanup-summary/_cleanup-summary-markdown.d.ts +5 -0
- package/dist/cleanup-summary/_cleanup-summary-markdown.js +224 -0
- package/dist/cleanup-summary/_cleanup-summary.d.ts +56 -0
- package/dist/cleanup-summary/_cleanup-summary.js +59 -0
- package/dist/cleanup-summary/index.d.ts +2 -0
- package/dist/cleanup-summary/index.js +2 -0
- package/dist/cli/_args.d.ts +7 -0
- package/dist/cli/_args.js +41 -0
- package/dist/cli/_cleanup-command.d.ts +1 -0
- package/dist/cli/_cleanup-command.js +142 -0
- package/dist/cli/_db-merge-command.d.ts +1 -0
- package/dist/cli/_db-merge-command.js +41 -0
- package/dist/cli/_github-output.d.ts +10 -0
- package/dist/cli/_github-output.js +13 -0
- package/dist/cli/_json-output.d.ts +1 -0
- package/dist/cli/_json-output.js +11 -0
- package/dist/cli/_logger.d.ts +10 -0
- package/dist/cli/_logger.js +26 -0
- package/dist/cli/_older-than.d.ts +5 -0
- package/dist/cli/_older-than.js +42 -0
- package/dist/cli/_planner-options.d.ts +20 -0
- package/dist/cli/_planner-options.js +83 -0
- package/dist/cli/_scan-command.d.ts +1 -0
- package/dist/cli/_scan-command.js +39 -0
- package/dist/cli/_tag-selector-resolver.d.ts +3 -0
- package/dist/cli/_tag-selector-resolver.js +132 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +52 -0
- package/dist/config/_service-constants.d.ts +3 -0
- package/dist/config/_service-constants.js +3 -0
- package/dist/config/index.d.ts +9 -0
- package/dist/config/index.js +9 -0
- package/dist/core/_digest-tag.d.ts +2 -0
- package/dist/core/_digest-tag.js +12 -0
- package/dist/core/_github-package-owner.d.ts +11 -0
- package/dist/core/_github-package-owner.js +45 -0
- package/dist/core/_http-error.d.ts +6 -0
- package/dist/core/_http-error.js +33 -0
- package/dist/core/_types.d.ts +54 -0
- package/dist/core/_types.js +8 -0
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.js +4 -0
- package/dist/db/_cleanup-run-writer.d.ts +10 -0
- package/dist/db/_cleanup-run-writer.js +113 -0
- package/dist/db/_db-merge-cleanup-copy.d.ts +7 -0
- package/dist/db/_db-merge-cleanup-copy.js +172 -0
- package/dist/db/_db-merge-history.d.ts +2 -0
- package/dist/db/_db-merge-history.js +15 -0
- package/dist/db/_db-merge-repository.d.ts +8 -0
- package/dist/db/_db-merge-repository.js +95 -0
- package/dist/db/_db-merge-scan-copy.d.ts +10 -0
- package/dist/db/_db-merge-scan-copy.js +70 -0
- package/dist/db/_db-merge-types.d.ts +44 -0
- package/dist/db/_db-merge-types.js +1 -0
- package/dist/db/_github-actions-run-url.d.ts +1 -0
- package/dist/db/_github-actions-run-url.js +9 -0
- package/dist/db/_manifest-reachability.d.ts +2 -0
- package/dist/db/_manifest-reachability.js +158 -0
- package/dist/db/_scan-writer.d.ts +20 -0
- package/dist/db/_scan-writer.js +184 -0
- package/dist/db/_schema.d.ts +2 -0
- package/dist/db/_schema.js +19 -0
- package/dist/db/_snapshot-repository.d.ts +24 -0
- package/dist/db/_snapshot-repository.js +86 -0
- package/dist/db/_sql-placeholders.d.ts +2 -0
- package/dist/db/_sql-placeholders.js +16 -0
- package/dist/db/index.d.ts +10 -0
- package/dist/db/index.js +13 -0
- package/dist/db/planner/_planner-direct-target-root-options.d.ts +11 -0
- package/dist/db/planner/_planner-direct-target-root-options.js +1 -0
- package/dist/db/planner/_planner-direct-target-root-tag-filters.d.ts +7 -0
- package/dist/db/planner/_planner-direct-target-root-tag-filters.js +41 -0
- package/dist/db/planner/_planner-direct-target-roots-combined-sql.d.ts +7 -0
- package/dist/db/planner/_planner-direct-target-roots-combined-sql.js +189 -0
- package/dist/db/planner/_planner-direct-target-roots-combined.d.ts +4 -0
- package/dist/db/planner/_planner-direct-target-roots-combined.js +10 -0
- package/dist/db/planner/_planner-direct-target-roots-tagged.d.ts +4 -0
- package/dist/db/planner/_planner-direct-target-roots-tagged.js +117 -0
- package/dist/db/planner/_planner-direct-target-roots.d.ts +8 -0
- package/dist/db/planner/_planner-direct-target-roots.js +17 -0
- package/dist/db/planner/_planner-direct-target-tags.d.ts +6 -0
- package/dist/db/planner/_planner-direct-target-tags.js +54 -0
- package/dist/db/planner/_planner-latest-scan.d.ts +10 -0
- package/dist/db/planner/_planner-latest-scan.js +20 -0
- package/dist/db/planner/_planner-output.d.ts +5 -0
- package/dist/db/planner/_planner-output.js +111 -0
- package/dist/db/planner/_planner-plan-artifacts-blocked-roots-sql.d.ts +1 -0
- package/dist/db/planner/_planner-plan-artifacts-blocked-roots-sql.js +110 -0
- package/dist/db/planner/_planner-plan-artifacts-closure-sql.d.ts +1 -0
- package/dist/db/planner/_planner-plan-artifacts-closure-sql.js +188 -0
- package/dist/db/planner/_planner-plan-artifacts-supported-untag-only-sql.d.ts +1 -0
- package/dist/db/planner/_planner-plan-artifacts-supported-untag-only-sql.js +86 -0
- package/dist/db/planner/_planner-plan-artifacts.d.ts +7 -0
- package/dist/db/planner/_planner-plan-artifacts.js +109 -0
- package/dist/db/planner/_planner-repository.d.ts +44 -0
- package/dist/db/planner/_planner-repository.js +105 -0
- package/dist/db/planner/_planner-sql.d.ts +12 -0
- package/dist/db/planner/_planner-sql.js +46 -0
- package/dist/db/planner/_planner-tag-selectors.d.ts +8 -0
- package/dist/db/planner/_planner-tag-selectors.js +57 -0
- package/dist/db/planner/_planner-types.d.ts +144 -0
- package/dist/db/planner/_planner-types.js +49 -0
- package/dist/db/planner/index.d.ts +3 -0
- package/dist/db/planner/index.js +2 -0
- package/dist/execute/_http.d.ts +7 -0
- package/dist/execute/_http.js +48 -0
- package/dist/execute/_manifest-detach.d.ts +4 -0
- package/dist/execute/_manifest-detach.js +31 -0
- package/dist/execute/_package-version-delete-client.d.ts +4 -0
- package/dist/execute/_package-version-delete-client.js +34 -0
- package/dist/execute/_package-version-page-client.d.ts +14 -0
- package/dist/execute/_package-version-page-client.js +64 -0
- package/dist/execute/_package-version-tag-source-client.d.ts +12 -0
- package/dist/execute/_package-version-tag-source-client.js +65 -0
- package/dist/execute/_plan-executor.d.ts +3 -0
- package/dist/execute/_plan-executor.js +69 -0
- package/dist/execute/_registry-manifest-client.d.ts +12 -0
- package/dist/execute/_registry-manifest-client.js +79 -0
- package/dist/execute/_registry-token-client.d.ts +4 -0
- package/dist/execute/_registry-token-client.js +37 -0
- package/dist/execute/_types.d.ts +34 -0
- package/dist/execute/_types.js +1 -0
- package/dist/execute/_untag-client.d.ts +2 -0
- package/dist/execute/_untag-client.js +30 -0
- package/dist/execute/index.d.ts +5 -0
- package/dist/execute/index.js +3 -0
- package/dist/ingest/github/_manifest-client.d.ts +14 -0
- package/dist/ingest/github/_manifest-client.js +108 -0
- package/dist/ingest/github/_manifest-ingest.d.ts +3 -0
- package/dist/ingest/github/_manifest-ingest.js +90 -0
- package/dist/ingest/github/_manifest-kind.d.ts +26 -0
- package/dist/ingest/github/_manifest-kind.js +65 -0
- package/dist/ingest/github/_package-metadata-load.d.ts +5 -0
- package/dist/ingest/github/_package-metadata-load.js +45 -0
- package/dist/ingest/github/_package-version-page-load.d.ts +13 -0
- package/dist/ingest/github/_package-version-page-load.js +55 -0
- package/dist/ingest/github/_packages-client.d.ts +10 -0
- package/dist/ingest/github/_packages-client.js +76 -0
- package/dist/ingest/github/_parallel-paginated-ingest.d.ts +12 -0
- package/dist/ingest/github/_parallel-paginated-ingest.js +49 -0
- package/dist/ingest/github/_registry-token-client.d.ts +6 -0
- package/dist/ingest/github/_registry-token-client.js +67 -0
- package/dist/ingest/github/_shared.d.ts +28 -0
- package/dist/ingest/github/_shared.js +70 -0
- package/dist/ingest/github/index.d.ts +11 -0
- package/dist/ingest/github/index.js +29 -0
- package/package.json +78 -0
- package/resources/sql/schema/001_schema.sql +222 -0
- package/resources/sql/views/001_v_latest_scan_per_package.sql +27 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export function resolveOlderThan(rawValue, now) {
|
|
2
|
+
const normalized = rawValue.trim().toLowerCase();
|
|
3
|
+
const match = normalized.match(/^(\d+)\s+(minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)$/);
|
|
4
|
+
if (!match) {
|
|
5
|
+
throw new Error(`invalid older-than interval: ${rawValue}`);
|
|
6
|
+
}
|
|
7
|
+
const amount = Number(match[1]);
|
|
8
|
+
const unit = match[2];
|
|
9
|
+
const cutoff = new Date(now.getTime());
|
|
10
|
+
switch (unit) {
|
|
11
|
+
case "minute":
|
|
12
|
+
case "minutes":
|
|
13
|
+
cutoff.setUTCMinutes(cutoff.getUTCMinutes() - amount);
|
|
14
|
+
break;
|
|
15
|
+
case "hour":
|
|
16
|
+
case "hours":
|
|
17
|
+
cutoff.setUTCHours(cutoff.getUTCHours() - amount);
|
|
18
|
+
break;
|
|
19
|
+
case "day":
|
|
20
|
+
case "days":
|
|
21
|
+
cutoff.setUTCDate(cutoff.getUTCDate() - amount);
|
|
22
|
+
break;
|
|
23
|
+
case "week":
|
|
24
|
+
case "weeks":
|
|
25
|
+
cutoff.setUTCDate(cutoff.getUTCDate() - amount * 7);
|
|
26
|
+
break;
|
|
27
|
+
case "month":
|
|
28
|
+
case "months":
|
|
29
|
+
cutoff.setUTCMonth(cutoff.getUTCMonth() - amount);
|
|
30
|
+
break;
|
|
31
|
+
case "year":
|
|
32
|
+
case "years":
|
|
33
|
+
cutoff.setUTCFullYear(cutoff.getUTCFullYear() - amount);
|
|
34
|
+
break;
|
|
35
|
+
default:
|
|
36
|
+
throw new Error(`invalid older-than unit: ${unit}`);
|
|
37
|
+
}
|
|
38
|
+
return {
|
|
39
|
+
olderThan: normalized,
|
|
40
|
+
cutoffTimestamp: cutoff.toISOString()
|
|
41
|
+
};
|
|
42
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { DeletePlan, PlannerRepository } from "../db/index.js";
|
|
2
|
+
export interface PlanCommandInputs {
|
|
3
|
+
databasePath: string;
|
|
4
|
+
owner: string;
|
|
5
|
+
packageName: string;
|
|
6
|
+
deleteTags: string[];
|
|
7
|
+
deleteTagsRequested: boolean;
|
|
8
|
+
deleteGhostImages: boolean;
|
|
9
|
+
deletePartialImages: boolean;
|
|
10
|
+
deleteOrphanedImages: boolean;
|
|
11
|
+
excludeTags: string[];
|
|
12
|
+
deleteUntagged: boolean;
|
|
13
|
+
useRegex: boolean;
|
|
14
|
+
keepNTagged?: number;
|
|
15
|
+
keepNUntagged?: number;
|
|
16
|
+
olderThan?: string;
|
|
17
|
+
cutoffTimestamp?: string;
|
|
18
|
+
}
|
|
19
|
+
export declare function resolvePlanCommandInputs(args: string[]): PlanCommandInputs;
|
|
20
|
+
export declare function loadDeletePlan(repository: PlannerRepository, inputs: PlanCommandInputs): DeletePlan;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { collectRepeatedOption, hasFlag, requireOption } from "./_args.js";
|
|
2
|
+
import { resolveOlderThan } from "./_older-than.js";
|
|
3
|
+
export function resolvePlanCommandInputs(args) {
|
|
4
|
+
const databasePath = requireOption(args, "--db");
|
|
5
|
+
const owner = requireOption(args, "--owner");
|
|
6
|
+
const packageName = requireOption(args, "--package");
|
|
7
|
+
const deleteTags = collectRepeatedOption(args, "--delete-tag");
|
|
8
|
+
const deleteGhostImages = hasFlag(args, "--delete-ghost-images");
|
|
9
|
+
const deletePartialImages = hasFlag(args, "--delete-partial-images");
|
|
10
|
+
const deleteOrphanedImages = hasFlag(args, "--delete-orphaned-images");
|
|
11
|
+
const excludeTags = collectRepeatedOption(args, "--exclude-tag");
|
|
12
|
+
const deleteUntagged = hasFlag(args, "--delete-untagged");
|
|
13
|
+
const useRegex = hasFlag(args, "--use-regex");
|
|
14
|
+
const keepNTaggedRaw = collectRepeatedOption(args, "--keep-n-tagged");
|
|
15
|
+
const keepNUntaggedRaw = collectRepeatedOption(args, "--keep-n-untagged");
|
|
16
|
+
const olderThanRaw = collectRepeatedOption(args, "--older-than");
|
|
17
|
+
if (keepNTaggedRaw.length > 1) {
|
|
18
|
+
throw new Error("--keep-n-tagged may only be provided once");
|
|
19
|
+
}
|
|
20
|
+
if (keepNUntaggedRaw.length > 1) {
|
|
21
|
+
throw new Error("--keep-n-untagged may only be provided once");
|
|
22
|
+
}
|
|
23
|
+
const keepNTagged = keepNTaggedRaw[0] ? resolveKeepCount("--keep-n-tagged", keepNTaggedRaw[0]) : undefined;
|
|
24
|
+
const keepNUntagged = keepNUntaggedRaw[0] ? resolveKeepCount("--keep-n-untagged", keepNUntaggedRaw[0]) : undefined;
|
|
25
|
+
const taggedSelectorActive = deleteTags.length > 0 ||
|
|
26
|
+
deleteGhostImages ||
|
|
27
|
+
deletePartialImages ||
|
|
28
|
+
deleteOrphanedImages ||
|
|
29
|
+
keepNTagged !== undefined;
|
|
30
|
+
const hasAnySelector = deleteUntagged || taggedSelectorActive || keepNUntagged !== undefined;
|
|
31
|
+
if (deleteUntagged && keepNUntagged !== undefined) {
|
|
32
|
+
throw new Error("--delete-untagged and --keep-n-untagged cannot be combined");
|
|
33
|
+
}
|
|
34
|
+
if (!hasAnySelector) {
|
|
35
|
+
throw new Error("missing required cleanup selector: --delete-untagged, --delete-tag, --delete-ghost-images, --delete-partial-images, --delete-orphaned-images, --keep-n-tagged, or --keep-n-untagged");
|
|
36
|
+
}
|
|
37
|
+
if (!taggedSelectorActive && excludeTags.length > 0) {
|
|
38
|
+
throw new Error("--exclude-tag is only supported with tagged selector families");
|
|
39
|
+
}
|
|
40
|
+
if (olderThanRaw.length > 1) {
|
|
41
|
+
throw new Error("--older-than may only be provided once");
|
|
42
|
+
}
|
|
43
|
+
const olderThan = olderThanRaw[0] ? resolveOlderThan(olderThanRaw[0], new Date()) : undefined;
|
|
44
|
+
return {
|
|
45
|
+
databasePath,
|
|
46
|
+
owner,
|
|
47
|
+
packageName,
|
|
48
|
+
deleteTags,
|
|
49
|
+
deleteTagsRequested: deleteTags.length > 0 || deleteGhostImages || deletePartialImages || deleteOrphanedImages,
|
|
50
|
+
deleteGhostImages,
|
|
51
|
+
deletePartialImages,
|
|
52
|
+
deleteOrphanedImages,
|
|
53
|
+
excludeTags,
|
|
54
|
+
deleteUntagged,
|
|
55
|
+
useRegex,
|
|
56
|
+
keepNTagged,
|
|
57
|
+
keepNUntagged,
|
|
58
|
+
olderThan: olderThan?.olderThan,
|
|
59
|
+
cutoffTimestamp: olderThan?.cutoffTimestamp
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
export function loadDeletePlan(repository, inputs) {
|
|
63
|
+
return repository.getCleanupPlanWithCutoff(inputs.owner, inputs.packageName, {
|
|
64
|
+
deleteTags: inputs.deleteTags,
|
|
65
|
+
deleteTagsRequested: inputs.deleteTagsRequested,
|
|
66
|
+
deleteGhostImages: inputs.deleteGhostImages,
|
|
67
|
+
deletePartialImages: inputs.deletePartialImages,
|
|
68
|
+
deleteOrphanedImages: inputs.deleteOrphanedImages,
|
|
69
|
+
excludeTags: inputs.excludeTags,
|
|
70
|
+
deleteUntagged: inputs.deleteUntagged,
|
|
71
|
+
useRegex: inputs.useRegex,
|
|
72
|
+
keepNTagged: inputs.keepNTagged,
|
|
73
|
+
keepNUntagged: inputs.keepNUntagged,
|
|
74
|
+
olderThan: inputs.olderThan,
|
|
75
|
+
cutoffTimestamp: inputs.cutoffTimestamp
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
function resolveKeepCount(optionName, rawValue) {
|
|
79
|
+
if (!/^\d+$/.test(rawValue)) {
|
|
80
|
+
throw new Error(`${optionName} must be a non-negative integer`);
|
|
81
|
+
}
|
|
82
|
+
return Number.parseInt(rawValue, 10);
|
|
83
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function handleScan(args: string[]): Promise<number>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { ScanWriter, SnapshotRepository, openDatabase } from "../db/index.js";
|
|
2
|
+
import { importGitHubScan } from "../ingest/github/index.js";
|
|
3
|
+
import { findOption, requireOption, resolveLogLevel, resolveToken } from "./_args.js";
|
|
4
|
+
import { writeGitHubScanOutputs } from "./_github-output.js";
|
|
5
|
+
import { createLogger } from "./_logger.js";
|
|
6
|
+
export async function handleScan(args) {
|
|
7
|
+
const databasePath = requireOption(args, "--db");
|
|
8
|
+
const owner = requireOption(args, "--owner");
|
|
9
|
+
const packageName = requireOption(args, "--package");
|
|
10
|
+
const githubOutputPath = findOption(args, "--github-output");
|
|
11
|
+
const token = resolveToken(args);
|
|
12
|
+
const logger = createLogger(resolveLogLevel(args));
|
|
13
|
+
const database = openDatabase(databasePath);
|
|
14
|
+
const repository = new SnapshotRepository(database);
|
|
15
|
+
const writer = new ScanWriter(database);
|
|
16
|
+
await importGitHubScan({
|
|
17
|
+
owner,
|
|
18
|
+
packageName,
|
|
19
|
+
token,
|
|
20
|
+
logger
|
|
21
|
+
}, writer, repository);
|
|
22
|
+
const scanId = writer.getActiveScanId();
|
|
23
|
+
const metadata = repository.getPackageMetadata(scanId);
|
|
24
|
+
const summary = {
|
|
25
|
+
owner: metadata.owner,
|
|
26
|
+
packageName: metadata.packageName,
|
|
27
|
+
scanCompletedAt: metadata.scanCompletedAt,
|
|
28
|
+
packageVersions: repository.countPackageVersions(scanId),
|
|
29
|
+
tags: repository.countTags(scanId),
|
|
30
|
+
manifests: repository.countManifests(scanId),
|
|
31
|
+
manifestEdges: repository.countManifestEdges(scanId)
|
|
32
|
+
};
|
|
33
|
+
if (githubOutputPath) {
|
|
34
|
+
writeGitHubScanOutputs(githubOutputPath, summary);
|
|
35
|
+
}
|
|
36
|
+
console.log(JSON.stringify(summary));
|
|
37
|
+
database.close();
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
const BrokenIndexModes = {
|
|
2
|
+
allMissing: "all-missing",
|
|
3
|
+
someMissing: "some-missing"
|
|
4
|
+
};
|
|
5
|
+
export function resolveTagSelectors(database, inputs) {
|
|
6
|
+
if (!inputs.deleteGhostImages && !inputs.deletePartialImages && !inputs.deleteOrphanedImages) {
|
|
7
|
+
return inputs;
|
|
8
|
+
}
|
|
9
|
+
return {
|
|
10
|
+
...inputs,
|
|
11
|
+
deleteTags: inputs.deleteGhostImages
|
|
12
|
+
? _listLatestGhostTags(database, inputs.owner, inputs.packageName, inputs.cutoffTimestamp)
|
|
13
|
+
: inputs.deletePartialImages
|
|
14
|
+
? _listLatestPartialTags(database, inputs.owner, inputs.packageName, inputs.cutoffTimestamp)
|
|
15
|
+
: inputs.deleteOrphanedImages
|
|
16
|
+
? _listLatestOrphanedTags(database, inputs.owner, inputs.packageName, inputs.cutoffTimestamp)
|
|
17
|
+
: inputs.deleteTags
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
function _listLatestGhostTags(database, owner, packageName, cutoffTimestamp) {
|
|
21
|
+
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, BrokenIndexModes.allMissing);
|
|
22
|
+
}
|
|
23
|
+
function _listLatestPartialTags(database, owner, packageName, cutoffTimestamp) {
|
|
24
|
+
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, BrokenIndexModes.someMissing);
|
|
25
|
+
}
|
|
26
|
+
function _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, mode) {
|
|
27
|
+
const havingClause = mode === BrokenIndexModes.allMissing
|
|
28
|
+
? "COUNT(*) > 0 AND COUNT(child.digest) = 0"
|
|
29
|
+
: "COUNT(child.digest) > 0 AND COUNT(child.digest) < COUNT(*)";
|
|
30
|
+
const rows = database
|
|
31
|
+
.prepare(`
|
|
32
|
+
WITH latest_scan AS (
|
|
33
|
+
SELECT scan_id
|
|
34
|
+
FROM v_latest_scan_per_package
|
|
35
|
+
WHERE owner = ?
|
|
36
|
+
AND package_name = ?
|
|
37
|
+
),
|
|
38
|
+
ghost_roots AS (
|
|
39
|
+
SELECT
|
|
40
|
+
m.scan_id,
|
|
41
|
+
m.version_id
|
|
42
|
+
FROM latest_scan ls
|
|
43
|
+
JOIN manifests m
|
|
44
|
+
ON m.scan_id = ls.scan_id
|
|
45
|
+
JOIN package_versions pv
|
|
46
|
+
ON pv.scan_id = m.scan_id
|
|
47
|
+
AND pv.version_id = m.version_id
|
|
48
|
+
JOIN tags root_tags
|
|
49
|
+
ON root_tags.scan_id = m.scan_id
|
|
50
|
+
AND root_tags.version_id = m.version_id
|
|
51
|
+
JOIN manifest_descriptors md
|
|
52
|
+
ON md.scan_id = m.scan_id
|
|
53
|
+
AND md.parent_digest = m.digest
|
|
54
|
+
LEFT JOIN manifests child
|
|
55
|
+
ON child.scan_id = md.scan_id
|
|
56
|
+
AND child.digest = md.child_digest
|
|
57
|
+
WHERE m.media_type IN (
|
|
58
|
+
'application/vnd.oci.image.index.v1+json',
|
|
59
|
+
'application/vnd.docker.distribution.manifest.list.v2+json'
|
|
60
|
+
)
|
|
61
|
+
AND NOT EXISTS (
|
|
62
|
+
SELECT 1
|
|
63
|
+
FROM manifest_reachability mr
|
|
64
|
+
WHERE mr.scan_id = m.scan_id
|
|
65
|
+
AND mr.descendant_digest = m.digest
|
|
66
|
+
AND mr.min_distance > 0
|
|
67
|
+
)
|
|
68
|
+
AND (? IS NULL OR pv.created_at < ?)
|
|
69
|
+
GROUP BY m.scan_id, m.version_id
|
|
70
|
+
HAVING ${havingClause}
|
|
71
|
+
)
|
|
72
|
+
SELECT DISTINCT t.tag
|
|
73
|
+
FROM ghost_roots gr
|
|
74
|
+
JOIN tags t
|
|
75
|
+
ON t.scan_id = gr.scan_id
|
|
76
|
+
AND t.version_id = gr.version_id
|
|
77
|
+
ORDER BY t.tag
|
|
78
|
+
`)
|
|
79
|
+
.all(owner, packageName, cutoffTimestamp ?? null, cutoffTimestamp ?? null);
|
|
80
|
+
return rows.map((row) => row.tag);
|
|
81
|
+
}
|
|
82
|
+
// Some OCI tooling publishes companion artifacts such as signatures or attestations under
|
|
83
|
+
// digest tags in the same repository, for example `sha256-<digest>.sig`, while the
|
|
84
|
+
// actual relationship is the artifact's subject/referrer link to the parent digest.
|
|
85
|
+
//
|
|
86
|
+
// Public references:
|
|
87
|
+
// - Sigstore Cosign example pushing `sha256-<digest>.sig`:
|
|
88
|
+
// https://docs.sigstore.dev/cosign/signing/other_types/
|
|
89
|
+
// - OCI referrers / subject model:
|
|
90
|
+
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md
|
|
91
|
+
//
|
|
92
|
+
// This resolver intentionally mirrors the `delete-orphaned-images` behavior from
|
|
93
|
+
// `dataaxiom/ghcr-cleanup-action`, but keeps the check narrow and local to the current package
|
|
94
|
+
// scan: derive the parent digest from the tag, then treat the tag as orphaned only when that
|
|
95
|
+
// digest is absent from the scanned manifests for the same package.
|
|
96
|
+
function _listLatestOrphanedTags(database, owner, packageName, cutoffTimestamp) {
|
|
97
|
+
const rows = database
|
|
98
|
+
.prepare(`
|
|
99
|
+
WITH latest_scan AS (
|
|
100
|
+
SELECT scan_id
|
|
101
|
+
FROM v_latest_scan_per_package
|
|
102
|
+
WHERE owner = ?
|
|
103
|
+
AND package_name = ?
|
|
104
|
+
),
|
|
105
|
+
digest_tag_artifacts AS (
|
|
106
|
+
SELECT
|
|
107
|
+
t.tag,
|
|
108
|
+
t.scan_id,
|
|
109
|
+
t.version_id AS artifact_version_id,
|
|
110
|
+
'sha256:' || SUBSTR(t.tag, 8, 64) AS parent_digest
|
|
111
|
+
FROM latest_scan ls
|
|
112
|
+
JOIN tags t
|
|
113
|
+
ON t.scan_id = ls.scan_id
|
|
114
|
+
WHERE t.is_digest_tag = 1
|
|
115
|
+
)
|
|
116
|
+
SELECT DISTINCT dta.tag
|
|
117
|
+
FROM digest_tag_artifacts dta
|
|
118
|
+
JOIN package_versions pv
|
|
119
|
+
ON pv.scan_id = dta.scan_id
|
|
120
|
+
AND pv.version_id = dta.artifact_version_id
|
|
121
|
+
WHERE NOT EXISTS (
|
|
122
|
+
SELECT 1
|
|
123
|
+
FROM manifests parent
|
|
124
|
+
WHERE parent.scan_id = dta.scan_id
|
|
125
|
+
AND parent.digest = dta.parent_digest
|
|
126
|
+
)
|
|
127
|
+
AND (? IS NULL OR pv.created_at < ?)
|
|
128
|
+
ORDER BY dta.tag
|
|
129
|
+
`)
|
|
130
|
+
.all(owner, packageName, cutoffTimestamp ?? null, cutoffTimestamp ?? null);
|
|
131
|
+
return rows.map((row) => row.tag);
|
|
132
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { realpathSync } from "node:fs";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { handleCleanup } from "./_cleanup-command.js";
|
|
5
|
+
import { handleDbMerge } from "./_db-merge-command.js";
|
|
6
|
+
import { handleScan } from "./_scan-command.js";
|
|
7
|
+
export async function main(argv) {
|
|
8
|
+
const [command, ...rest] = argv;
|
|
9
|
+
if (!command) {
|
|
10
|
+
printUsage();
|
|
11
|
+
return 1;
|
|
12
|
+
}
|
|
13
|
+
switch (command) {
|
|
14
|
+
case "cleanup":
|
|
15
|
+
return handleCleanup(rest);
|
|
16
|
+
case "db-merge":
|
|
17
|
+
return handleDbMerge(rest);
|
|
18
|
+
case "scan":
|
|
19
|
+
return handleScan(rest);
|
|
20
|
+
default:
|
|
21
|
+
throw new Error(`unknown command: ${command}`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function printUsage() {
|
|
25
|
+
console.error(`Usage:
|
|
26
|
+
ghcr-cleanup-manager cleanup --db <path> [--log-level <trace|debug|info|warn|error|silent>] [--dry-run] [--summary-json-path <path>] --owner <org> --package <name> [--token <token>] <cleanup selectors...> [--exclude-tag <tag> ...] [--use-regex] [--older-than <interval>]
|
|
27
|
+
ghcr-cleanup-manager db-merge --db <target-path> --source-db <path> [--source-db <path> ...]
|
|
28
|
+
ghcr-cleanup-manager scan --db <path> [--log-level <trace|debug|info|warn|error|silent>] [--github-output <path>] --owner <org> --package <name> --token <token>
|
|
29
|
+
|
|
30
|
+
Cleanup selectors:
|
|
31
|
+
--delete-untagged
|
|
32
|
+
--delete-ghost-images
|
|
33
|
+
--delete-partial-images
|
|
34
|
+
--delete-orphaned-images
|
|
35
|
+
--delete-tag <tag> [--delete-tag <tag> ...]
|
|
36
|
+
--keep-n-tagged <count>
|
|
37
|
+
--keep-n-untagged <count>
|
|
38
|
+
|
|
39
|
+
Notes:
|
|
40
|
+
- Tagged selector families may be combined with --delete-untagged.
|
|
41
|
+
- --exclude-tag requires at least one tagged selector family.
|
|
42
|
+
- --delete-untagged and --keep-n-untagged cannot be combined.`);
|
|
43
|
+
}
|
|
44
|
+
const _entryPath = process.argv[1];
|
|
45
|
+
const _isDirectExecution = realpathSync(_entryPath) === realpathSync(fileURLToPath(import.meta.url));
|
|
46
|
+
if (_isDirectExecution) {
|
|
47
|
+
main(process.argv.slice(2)).catch((error) => {
|
|
48
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
49
|
+
console.error(message);
|
|
50
|
+
process.exitCode = 1;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const packageVersionPageFetchConcurrency = 4;
|
|
2
|
+
export declare const manifestFetchConcurrency = 16;
|
|
3
|
+
export declare const ingestRequestRetryCount = 3;
|
|
4
|
+
export declare const ingestRequestRetryDelayMs = 1000;
|
|
5
|
+
export declare const executeRequestRetryCount = 3;
|
|
6
|
+
export declare const executeRequestRetryDelayMs = 1000;
|
|
7
|
+
export declare const paginatedIngestProgressIntervalPages = 10;
|
|
8
|
+
export declare const manifestIngestProgressStepRatio = 0.05;
|
|
9
|
+
export { ghcrRegistryBaseUrl, githubApiBaseUrl, githubApiVersion } from "./_service-constants.js";
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export const packageVersionPageFetchConcurrency = 4;
|
|
2
|
+
export const manifestFetchConcurrency = 16;
|
|
3
|
+
export const ingestRequestRetryCount = 3;
|
|
4
|
+
export const ingestRequestRetryDelayMs = 1000;
|
|
5
|
+
export const executeRequestRetryCount = 3;
|
|
6
|
+
export const executeRequestRetryDelayMs = 1000;
|
|
7
|
+
export const paginatedIngestProgressIntervalPages = 10;
|
|
8
|
+
export const manifestIngestProgressStepRatio = 0.05;
|
|
9
|
+
export { ghcrRegistryBaseUrl, githubApiBaseUrl, githubApiVersion } from "./_service-constants.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export function isDigestTag(tag) {
|
|
2
|
+
return tag.startsWith("sha256-") && tag.length >= 71 && !_containsNonHex(tag.slice(7, 71));
|
|
3
|
+
}
|
|
4
|
+
export function digestFromDigestTag(tag) {
|
|
5
|
+
if (!isDigestTag(tag)) {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
return `sha256:${tag.slice(7, 71)}`;
|
|
9
|
+
}
|
|
10
|
+
function _containsNonHex(value) {
|
|
11
|
+
return /[^0-9A-Fa-f]/.test(value);
|
|
12
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
interface _FetchLikeResponse {
|
|
2
|
+
ok: boolean;
|
|
3
|
+
status: number;
|
|
4
|
+
headers: Headers;
|
|
5
|
+
json(): Promise<unknown>;
|
|
6
|
+
}
|
|
7
|
+
interface _Logger {
|
|
8
|
+
warn(message: string): void;
|
|
9
|
+
}
|
|
10
|
+
export declare function getOwnerURIComponent(fetchImpl: (input: string, init?: RequestInit) => Promise<_FetchLikeResponse>, owner: string, token: string, logger: _Logger): Promise<string>;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { githubApiBaseUrl, githubApiVersion, ingestRequestRetryCount, ingestRequestRetryDelayMs } from "../config/index.js";
|
|
2
|
+
import { buildHttpErrorMessage } from "./_http-error.js";
|
|
3
|
+
const _ownerUriComponentByOwner = new Map();
|
|
4
|
+
export async function getOwnerURIComponent(fetchImpl, owner, token, logger) {
|
|
5
|
+
const cachedOwnerURIComponent = _ownerUriComponentByOwner.get(owner);
|
|
6
|
+
if (cachedOwnerURIComponent) {
|
|
7
|
+
return cachedOwnerURIComponent;
|
|
8
|
+
}
|
|
9
|
+
const url = new URL(`/users/${encodeURIComponent(owner)}`, githubApiBaseUrl).toString();
|
|
10
|
+
for (let attempt = 1;; attempt += 1) {
|
|
11
|
+
const response = await fetchImpl(url, {
|
|
12
|
+
headers: {
|
|
13
|
+
Accept: "application/vnd.github+json",
|
|
14
|
+
Authorization: `Bearer ${token}`,
|
|
15
|
+
"User-Agent": "ghcr-cleanup-manager",
|
|
16
|
+
"X-GitHub-Api-Version": githubApiVersion
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
if (response.ok) {
|
|
20
|
+
const payload = (await response.json());
|
|
21
|
+
if (payload.type === "Organization") {
|
|
22
|
+
const ownerURIComponent = `orgs/${encodeURIComponent(owner)}`;
|
|
23
|
+
_ownerUriComponentByOwner.set(owner, ownerURIComponent);
|
|
24
|
+
return ownerURIComponent;
|
|
25
|
+
}
|
|
26
|
+
if (payload.type === "User") {
|
|
27
|
+
const ownerURIComponent = `users/${encodeURIComponent(owner)}`;
|
|
28
|
+
_ownerUriComponentByOwner.set(owner, ownerURIComponent);
|
|
29
|
+
return ownerURIComponent;
|
|
30
|
+
}
|
|
31
|
+
throw new Error(`GitHub owner lookup did not include a supported type`);
|
|
32
|
+
}
|
|
33
|
+
if (!_isRetryableStatus(response.status) || attempt > ingestRequestRetryCount) {
|
|
34
|
+
throw new Error(await buildHttpErrorMessage(response, "GitHub owner lookup failed"));
|
|
35
|
+
}
|
|
36
|
+
logger.warn(`GitHub owner lookup failed on attempt ${attempt}/${ingestRequestRetryCount + 1}; retrying in ${ingestRequestRetryDelayMs}ms - ${await buildHttpErrorMessage(response, "GitHub owner lookup failed")}`);
|
|
37
|
+
await _sleep(ingestRequestRetryDelayMs);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function _isRetryableStatus(status) {
|
|
41
|
+
return status === 429 || status === 502 || status === 503 || status === 504;
|
|
42
|
+
}
|
|
43
|
+
function _sleep(delayMs) {
|
|
44
|
+
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
45
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export async function buildHttpErrorMessage(response, fallback) {
|
|
2
|
+
const details = [fallback, `status ${response.status}`];
|
|
3
|
+
const body = await _readJsonErrorBody(response);
|
|
4
|
+
const message = typeof body?.message === "string" ? body.message : undefined;
|
|
5
|
+
const documentationUrl = typeof body?.documentation_url === "string" ? body.documentation_url : undefined;
|
|
6
|
+
const authenticateHeader = response.headers.get("www-authenticate") ?? undefined;
|
|
7
|
+
if (message) {
|
|
8
|
+
details.push(message);
|
|
9
|
+
}
|
|
10
|
+
if (documentationUrl) {
|
|
11
|
+
details.push(documentationUrl);
|
|
12
|
+
}
|
|
13
|
+
if (authenticateHeader) {
|
|
14
|
+
details.push(`www-authenticate: ${authenticateHeader}`);
|
|
15
|
+
}
|
|
16
|
+
return details.join(" - ");
|
|
17
|
+
}
|
|
18
|
+
async function _readJsonErrorBody(response) {
|
|
19
|
+
const contentType = response.headers.get("content-type")?.split(";")[0];
|
|
20
|
+
if (contentType && contentType !== "application/json" && !contentType.endsWith("+json")) {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
try {
|
|
24
|
+
const body = await response.json();
|
|
25
|
+
if (body && typeof body === "object") {
|
|
26
|
+
return body;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export declare const ManifestKinds: {
|
|
2
|
+
readonly indexManifest: "index_manifest";
|
|
3
|
+
readonly multiArchManifest: "multi_arch_manifest";
|
|
4
|
+
readonly imageManifest: "image_manifest";
|
|
5
|
+
readonly artifactManifest: "artifact_manifest";
|
|
6
|
+
readonly attestationManifest: "attestation_manifest";
|
|
7
|
+
readonly signatureManifest: "signature_manifest";
|
|
8
|
+
};
|
|
9
|
+
export type ManifestKind = (typeof ManifestKinds)[keyof typeof ManifestKinds];
|
|
10
|
+
export type ManifestEdgeKind = "image-child" | "referrer" | "digest-tag-referrer";
|
|
11
|
+
export interface PackageVersionRecord {
|
|
12
|
+
versionId: number;
|
|
13
|
+
createdAt: string;
|
|
14
|
+
updatedAt: string;
|
|
15
|
+
metadata?: Record<string, unknown>;
|
|
16
|
+
}
|
|
17
|
+
export interface TagRecord {
|
|
18
|
+
tag: string;
|
|
19
|
+
versionId: number;
|
|
20
|
+
}
|
|
21
|
+
export interface ManifestRecord {
|
|
22
|
+
versionId: number;
|
|
23
|
+
digest: string;
|
|
24
|
+
mediaType: string;
|
|
25
|
+
artifactType?: string;
|
|
26
|
+
configMediaType?: string;
|
|
27
|
+
subjectDigest?: string;
|
|
28
|
+
annotations?: Record<string, unknown>;
|
|
29
|
+
manifestKind?: ManifestKind;
|
|
30
|
+
}
|
|
31
|
+
export interface ManifestDescriptorRecord {
|
|
32
|
+
parentDigest: string;
|
|
33
|
+
childDigest: string;
|
|
34
|
+
mediaType: string;
|
|
35
|
+
artifactType?: string;
|
|
36
|
+
platform?: {
|
|
37
|
+
architecture?: string;
|
|
38
|
+
os?: string;
|
|
39
|
+
variant?: string;
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export interface ManifestEdgeRecord {
|
|
43
|
+
parentDigest: string;
|
|
44
|
+
childDigest: string;
|
|
45
|
+
edgeKind: ManifestEdgeKind;
|
|
46
|
+
}
|
|
47
|
+
export interface PackageSnapshot {
|
|
48
|
+
packageName: string;
|
|
49
|
+
scanCompletedAt: string;
|
|
50
|
+
packageVersions: PackageVersionRecord[];
|
|
51
|
+
tags: TagRecord[];
|
|
52
|
+
manifests: ManifestRecord[];
|
|
53
|
+
manifestEdges: ManifestEdgeRecord[];
|
|
54
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const ManifestKinds = {
|
|
2
|
+
indexManifest: "index_manifest",
|
|
3
|
+
multiArchManifest: "multi_arch_manifest",
|
|
4
|
+
imageManifest: "image_manifest",
|
|
5
|
+
artifactManifest: "artifact_manifest",
|
|
6
|
+
attestationManifest: "attestation_manifest",
|
|
7
|
+
signatureManifest: "signature_manifest"
|
|
8
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { ManifestEdgeKind, ManifestEdgeRecord, ManifestDescriptorRecord, ManifestKind, ManifestRecord, PackageSnapshot, PackageVersionRecord, TagRecord } from "./_types.js";
|
|
2
|
+
export { ManifestKinds } from "./_types.js";
|
|
3
|
+
export type { HttpErrorResponse } from "./_http-error.js";
|
|
4
|
+
export { buildHttpErrorMessage } from "./_http-error.js";
|
|
5
|
+
export { getOwnerURIComponent } from "./_github-package-owner.js";
|
|
6
|
+
export { digestFromDigestTag, isDigestTag } from "./_digest-tag.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import { type DeletePlan } from "./planner/index.js";
|
|
3
|
+
export declare class CleanupRunWriter {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(database: Database.Database);
|
|
6
|
+
persistCleanupRun(scanId: number, plan: DeletePlan, options: {
|
|
7
|
+
dryRun: boolean;
|
|
8
|
+
cleanupStartedAt: string;
|
|
9
|
+
}): number;
|
|
10
|
+
}
|