ghcr-manager 0.9.5 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +14 -13
- package/dist/cleanup-summary/_cleanup-summary-markdown.d.ts +0 -1
- package/dist/cleanup-summary/_cleanup-summary-markdown.js +146 -32
- package/dist/cleanup-summary/_cleanup-summary.d.ts +26 -9
- package/dist/cleanup-summary/_cleanup-summary.js +27 -7
- package/dist/cleanup-summary/index.d.ts +1 -1
- package/dist/cli/_cleanup-command.js +84 -5
- package/dist/cli/_json-output.d.ts +1 -0
- package/dist/cli/_json-output.js +11 -0
- package/dist/cli/_tag-selector-resolver.js +9 -6
- package/dist/cli/_untag-command.js +2 -1
- package/dist/cli/index.js +2 -2
- package/dist/core/_digest-tag.d.ts +2 -0
- package/dist/core/_digest-tag.js +12 -0
- package/dist/core/_types.d.ts +11 -2
- package/dist/core/_types.js +8 -1
- package/dist/core/index.d.ts +3 -1
- package/dist/core/index.js +2 -0
- package/dist/db/_cleanup-run-writer.d.ts +1 -1
- package/dist/db/_cleanup-run-writer.js +90 -50
- package/dist/db/_db-merge-cleanup-copy.js +130 -80
- package/dist/db/_db-merge-scan-copy.js +1 -1
- package/dist/db/_manifest-kind-refinement.d.ts +2 -0
- package/dist/db/_manifest-kind-refinement.js +43 -0
- package/dist/db/_manifest-reachability.js +25 -0
- package/dist/db/_scan-writer.js +122 -117
- package/dist/db/index.d.ts +2 -1
- package/dist/db/index.js +1 -0
- package/dist/db/planner/_planner-direct-target-roots.d.ts +1 -0
- package/dist/db/planner/_planner-direct-target-roots.js +24 -11
- package/dist/db/planner/_planner-direct-target-tags.d.ts +1 -1
- package/dist/db/planner/_planner-direct-target-tags.js +8 -1
- package/dist/db/planner/_planner-output.d.ts +1 -1
- package/dist/db/planner/_planner-output.js +7 -17
- package/dist/db/planner/_planner-repository.d.ts +2 -1
- package/dist/db/planner/_planner-repository.js +3 -1
- package/dist/db/planner/_planner-types.d.ts +33 -26
- package/dist/db/planner/_planner-types.js +13 -3
- package/dist/db/planner/index.d.ts +2 -1
- package/dist/db/planner/index.js +1 -0
- package/dist/execute/_plan-executor.d.ts +1 -1
- package/dist/execute/_plan-executor.js +35 -9
- package/dist/ingest/github/_manifest-kind.d.ts +1 -1
- package/dist/ingest/github/_manifest-kind.js +6 -5
- package/package.json +1 -1
- package/resources/sql/schema/001_schema.sql +24 -1
- package/resources/sql/views/{003_v_scan_root_manifests.sql → 002_v_scan_root_manifests.sql} +1 -0
- package/resources/sql/views/{004_v_digest_derived_tag_relations.sql → 003_v_digest_tag_relations.sql} +7 -8
- package/resources/sql/views/{005_v_cleanup_root_closure_members.sql → 004_v_cleanup_root_closure_members.sql} +2 -1
- package/resources/sql/views/006_v_cleanup_root_decision_readable.sql +67 -0
- package/resources/sql/views/002_v_missing_digests.sql +0 -32
- /package/resources/sql/views/{006_v_cleanup_blocking_overlaps.sql → 005_v_cleanup_blocking_overlaps.sql} +0 -0
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
const BrokenIndexModes = {
|
|
2
|
+
allMissing: "all-missing",
|
|
3
|
+
someMissing: "some-missing"
|
|
4
|
+
};
|
|
1
5
|
export function resolveTagSelectors(database, inputs) {
|
|
2
6
|
if (!inputs.deleteGhostImages && !inputs.deletePartialImages && !inputs.deleteOrphanedImages) {
|
|
3
7
|
return inputs;
|
|
@@ -14,13 +18,13 @@ export function resolveTagSelectors(database, inputs) {
|
|
|
14
18
|
};
|
|
15
19
|
}
|
|
16
20
|
function _listLatestGhostTags(database, owner, packageName, cutoffTimestamp) {
|
|
17
|
-
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp,
|
|
21
|
+
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, BrokenIndexModes.allMissing);
|
|
18
22
|
}
|
|
19
23
|
function _listLatestPartialTags(database, owner, packageName, cutoffTimestamp) {
|
|
20
|
-
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp,
|
|
24
|
+
return _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, BrokenIndexModes.someMissing);
|
|
21
25
|
}
|
|
22
26
|
function _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestamp, mode) {
|
|
23
|
-
const havingClause = mode ===
|
|
27
|
+
const havingClause = mode === BrokenIndexModes.allMissing
|
|
24
28
|
? "COUNT(*) > 0 AND COUNT(child.digest) = 0"
|
|
25
29
|
: "COUNT(child.digest) > 0 AND COUNT(child.digest) < COUNT(*)";
|
|
26
30
|
const rows = database
|
|
@@ -30,7 +34,6 @@ function _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestam
|
|
|
30
34
|
FROM v_latest_scan_per_package
|
|
31
35
|
WHERE owner = ?
|
|
32
36
|
AND package_name = ?
|
|
33
|
-
LIMIT 1
|
|
34
37
|
),
|
|
35
38
|
ghost_roots AS (
|
|
36
39
|
SELECT
|
|
@@ -77,7 +80,7 @@ function _listLatestBrokenIndexTags(database, owner, packageName, cutoffTimestam
|
|
|
77
80
|
return rows.map((row) => row.tag);
|
|
78
81
|
}
|
|
79
82
|
// Some OCI tooling publishes companion artifacts such as signatures or attestations under
|
|
80
|
-
// digest
|
|
83
|
+
// digest tags in the same repository, for example `sha256-<digest>.sig`, while the
|
|
81
84
|
// actual relationship is the artifact's subject/referrer link to the parent digest.
|
|
82
85
|
//
|
|
83
86
|
// Public references:
|
|
@@ -94,7 +97,7 @@ function _listLatestOrphanedTags(database, owner, packageName, cutoffTimestamp)
|
|
|
94
97
|
const rows = database
|
|
95
98
|
.prepare(`
|
|
96
99
|
SELECT DISTINCT dtr.tag
|
|
97
|
-
FROM
|
|
100
|
+
FROM v_digest_tag_relations dtr
|
|
98
101
|
INNER JOIN package_versions pv
|
|
99
102
|
ON pv.scan_id = dtr.scan_id
|
|
100
103
|
AND pv.version_id = dtr.artifact_version_id
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { listPackageVersionTagSources, untagRootTags } from "../execute/index.js";
|
|
2
2
|
import { collectRepeatedOption, hasFlag, requireOption, resolveLogLevel, resolveToken } from "./_args.js";
|
|
3
|
+
import { writeJsonOutput } from "./_json-output.js";
|
|
3
4
|
import { createLogger } from "./_logger.js";
|
|
4
5
|
export async function handleUntag(args) {
|
|
5
6
|
const owner = requireOption(args, "--owner");
|
|
@@ -35,7 +36,7 @@ export async function handleUntag(args) {
|
|
|
35
36
|
roots,
|
|
36
37
|
untaggedTags
|
|
37
38
|
};
|
|
38
|
-
|
|
39
|
+
writeJsonOutput(args, "--summary-json-path", summary);
|
|
39
40
|
return 0;
|
|
40
41
|
}
|
|
41
42
|
function _groupTagSources(tagSources) {
|
package/dist/cli/index.js
CHANGED
|
@@ -26,10 +26,10 @@ export async function main(argv) {
|
|
|
26
26
|
}
|
|
27
27
|
function printUsage() {
|
|
28
28
|
console.error(`Usage:
|
|
29
|
-
ghcr-manager cleanup --db <path> [--log-level <trace|debug|info|warn|error|silent>] [--dry-run] --owner <org> --package <name> [--token <token>] <cleanup selectors...> [--exclude-tag <tag> ...] [--use-regex] [--older-than <interval>]
|
|
29
|
+
ghcr-manager cleanup --db <path> [--log-level <trace|debug|info|warn|error|silent>] [--dry-run] [--summary-json-path <path>] --owner <org> --package <name> [--token <token>] <cleanup selectors...> [--exclude-tag <tag> ...] [--use-regex] [--older-than <interval>]
|
|
30
30
|
ghcr-manager db-merge --db <target-path> --source-db <path> [--source-db <path> ...]
|
|
31
31
|
ghcr-manager scan --db <path> [--log-level <trace|debug|info|warn|error|silent>] [--github-output <path>] --owner <org> --package <name> --token <token>
|
|
32
|
-
ghcr-manager untag [--log-level <trace|debug|info|warn|error|silent>] [--dry-run] --owner <org> --package <name> --token <token> --tag <tag> [--tag <tag> ...]
|
|
32
|
+
ghcr-manager untag [--log-level <trace|debug|info|warn|error|silent>] [--dry-run] [--summary-json-path <path>] --owner <org> --package <name> --token <token> --tag <tag> [--tag <tag> ...]
|
|
33
33
|
|
|
34
34
|
Cleanup selectors:
|
|
35
35
|
--delete-untagged
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export function isDigestTag(tag) {
|
|
2
|
+
return tag.startsWith("sha256-") && tag.length >= 71 && !_containsNonHex(tag.slice(7, 71));
|
|
3
|
+
}
|
|
4
|
+
export function digestFromDigestTag(tag) {
|
|
5
|
+
if (!isDigestTag(tag)) {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
return `sha256:${tag.slice(7, 71)}`;
|
|
9
|
+
}
|
|
10
|
+
function _containsNonHex(value) {
|
|
11
|
+
return /[^0-9A-Fa-f]/.test(value);
|
|
12
|
+
}
|
package/dist/core/_types.d.ts
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
export
|
|
1
|
+
export declare const ManifestKinds: {
|
|
2
|
+
readonly indexManifest: "index_manifest";
|
|
3
|
+
readonly crossArchManifest: "cross_arch_manifest";
|
|
4
|
+
readonly imageManifest: "image_manifest";
|
|
5
|
+
readonly artifactManifest: "artifact_manifest";
|
|
6
|
+
readonly attestationManifest: "attestation_manifest";
|
|
7
|
+
readonly signatureManifest: "signature_manifest";
|
|
8
|
+
};
|
|
9
|
+
export type ManifestKind = (typeof ManifestKinds)[keyof typeof ManifestKinds];
|
|
10
|
+
export type ManifestEdgeKind = "image-child" | "referrer" | "digest-tag-referrer";
|
|
2
11
|
export interface PackageVersionRecord {
|
|
3
12
|
versionId: number;
|
|
4
13
|
createdAt: string;
|
|
@@ -38,7 +47,7 @@ export interface ManifestDescriptorRecord {
|
|
|
38
47
|
export interface ManifestEdgeRecord {
|
|
39
48
|
parentDigest: string;
|
|
40
49
|
childDigest: string;
|
|
41
|
-
edgeKind:
|
|
50
|
+
edgeKind: ManifestEdgeKind;
|
|
42
51
|
}
|
|
43
52
|
export interface PackageSnapshot {
|
|
44
53
|
packageName: string;
|
package/dist/core/_types.js
CHANGED
|
@@ -1 +1,8 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export const ManifestKinds = {
|
|
2
|
+
indexManifest: "index_manifest",
|
|
3
|
+
crossArchManifest: "cross_arch_manifest",
|
|
4
|
+
imageManifest: "image_manifest",
|
|
5
|
+
artifactManifest: "artifact_manifest",
|
|
6
|
+
attestationManifest: "attestation_manifest",
|
|
7
|
+
signatureManifest: "signature_manifest"
|
|
8
|
+
};
|
package/dist/core/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
export type { ManifestEdgeRecord, ManifestDescriptorRecord, ManifestKind, ManifestRecord, PackageSnapshot, PackageVersionRecord, TagRecord } from "./_types.js";
|
|
1
|
+
export type { ManifestEdgeKind, ManifestEdgeRecord, ManifestDescriptorRecord, ManifestKind, ManifestRecord, PackageSnapshot, PackageVersionRecord, TagRecord } from "./_types.js";
|
|
2
|
+
export { ManifestKinds } from "./_types.js";
|
|
2
3
|
export type { HttpErrorResponse } from "./_http-error.js";
|
|
3
4
|
export { buildHttpErrorMessage } from "./_http-error.js";
|
|
4
5
|
export { getOwnerURIComponent } from "./_github-package-owner.js";
|
|
6
|
+
export { digestFromDigestTag, isDigestTag } from "./_digest-tag.js";
|
package/dist/core/index.js
CHANGED
|
@@ -1,73 +1,113 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { resolveGitHubActionsRunUrl } from "./_github-actions-run-url.js";
|
|
3
|
+
import { DeletePlanValidationStatuses } from "./planner/index.js";
|
|
3
4
|
export class CleanupRunWriter {
|
|
4
5
|
#database;
|
|
6
|
+
#insertSelectedTagStatement;
|
|
7
|
+
#updateSelectedTagsDeletedStatement;
|
|
8
|
+
#insertRootDecisionStatement;
|
|
9
|
+
#insertProtectedRootBlockStatement;
|
|
10
|
+
#insertCleanupRunStatement;
|
|
5
11
|
constructor(database) {
|
|
6
12
|
this.#database = database;
|
|
13
|
+
this.#insertSelectedTagStatement = this.#database.prepare(`
|
|
14
|
+
INSERT INTO cleanup_selected_tags(
|
|
15
|
+
cleanup_run_id,
|
|
16
|
+
scan_id,
|
|
17
|
+
tag,
|
|
18
|
+
is_deleted
|
|
19
|
+
)
|
|
20
|
+
VALUES(?, ?, ?, 0)
|
|
21
|
+
`);
|
|
22
|
+
this.#updateSelectedTagsDeletedStatement = this.#database.prepare(`
|
|
23
|
+
UPDATE cleanup_selected_tags
|
|
24
|
+
SET is_deleted = 1
|
|
25
|
+
FROM cleanup_root_decisions decision
|
|
26
|
+
JOIN manifests manifest
|
|
27
|
+
ON manifest.scan_id = decision.scan_id
|
|
28
|
+
AND manifest.digest = decision.digest
|
|
29
|
+
JOIN tags
|
|
30
|
+
ON tags.scan_id = manifest.scan_id
|
|
31
|
+
AND tags.version_id = manifest.version_id
|
|
32
|
+
WHERE cleanup_selected_tags.cleanup_run_id = ?
|
|
33
|
+
AND cleanup_selected_tags.scan_id = ?
|
|
34
|
+
AND decision.cleanup_run_id = cleanup_selected_tags.cleanup_run_id
|
|
35
|
+
AND decision.scan_id = cleanup_selected_tags.scan_id
|
|
36
|
+
AND decision.validation_status != 'blocked'
|
|
37
|
+
AND tags.tag = cleanup_selected_tags.tag
|
|
38
|
+
`);
|
|
39
|
+
this.#insertRootDecisionStatement = this.#database.prepare(`
|
|
40
|
+
INSERT INTO cleanup_root_decisions(
|
|
41
|
+
cleanup_run_id,
|
|
42
|
+
scan_id,
|
|
43
|
+
digest,
|
|
44
|
+
selection_mode,
|
|
45
|
+
selection_reason,
|
|
46
|
+
validation_status,
|
|
47
|
+
validation_reason_code,
|
|
48
|
+
validation_reason,
|
|
49
|
+
blocking_digest,
|
|
50
|
+
overlap_digest
|
|
51
|
+
)
|
|
52
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
53
|
+
`);
|
|
54
|
+
this.#insertProtectedRootBlockStatement = this.#database.prepare(`
|
|
55
|
+
INSERT INTO cleanup_protected_root_blocks(
|
|
56
|
+
cleanup_run_id,
|
|
57
|
+
scan_id,
|
|
58
|
+
protected_digest,
|
|
59
|
+
blocked_digest,
|
|
60
|
+
block_reason_code,
|
|
61
|
+
overlap_digest
|
|
62
|
+
)
|
|
63
|
+
VALUES(?, ?, ?, ?, ?, ?)
|
|
64
|
+
`);
|
|
65
|
+
this.#insertCleanupRunStatement = this.#database.prepare(`
|
|
66
|
+
INSERT INTO cleanup_runs(
|
|
67
|
+
scan_id,
|
|
68
|
+
cleanup_uuid,
|
|
69
|
+
cleanup_started_at,
|
|
70
|
+
github_actions_run_url,
|
|
71
|
+
dry_run,
|
|
72
|
+
planner_inputs_json,
|
|
73
|
+
direct_target_tag_count,
|
|
74
|
+
direct_target_root_count,
|
|
75
|
+
delete_root_candidate_count,
|
|
76
|
+
untag_only_root_count,
|
|
77
|
+
fully_deletable_root_count,
|
|
78
|
+
blocked_delete_root_count,
|
|
79
|
+
protected_root_count
|
|
80
|
+
)
|
|
81
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
82
|
+
`);
|
|
7
83
|
}
|
|
8
84
|
persistCleanupRun(scanId, plan, options) {
|
|
9
85
|
return this.#database.transaction(() => {
|
|
10
86
|
const cleanupRunId = this.#insertCleanupRun(scanId, plan, options);
|
|
11
87
|
for (const rootDecision of plan.rootDecisions) {
|
|
12
|
-
this.#
|
|
13
|
-
.prepare(`
|
|
14
|
-
INSERT INTO cleanup_root_decisions(
|
|
15
|
-
cleanup_run_id,
|
|
16
|
-
scan_id,
|
|
17
|
-
digest,
|
|
18
|
-
selection_mode,
|
|
19
|
-
selection_reason,
|
|
20
|
-
validation_status,
|
|
21
|
-
validation_reason_code,
|
|
22
|
-
validation_reason,
|
|
23
|
-
blocking_digest,
|
|
24
|
-
overlap_digest
|
|
25
|
-
)
|
|
26
|
-
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
27
|
-
`)
|
|
28
|
-
.run(cleanupRunId, scanId, rootDecision.digest, rootDecision.selectionMode, rootDecision.selectionReason, rootDecision.validationStatus, rootDecision.validationReasonCode, rootDecision.validationReason, rootDecision.blockingDigest ?? null, rootDecision.overlapDigest ?? null);
|
|
88
|
+
this.#insertRootDecisionStatement.run(cleanupRunId, scanId, rootDecision.digest, rootDecision.selectionMode, rootDecision.selectionReason, rootDecision.validationStatus, rootDecision.validationReasonCode, rootDecision.validationReason, rootDecision.blockingDigest ?? null, rootDecision.overlapDigest ?? null);
|
|
29
89
|
}
|
|
30
90
|
for (const protectedRoot of plan.protectedRoots) {
|
|
31
91
|
for (const block of protectedRoot.blocks) {
|
|
32
|
-
this.#
|
|
33
|
-
.prepare(`
|
|
34
|
-
INSERT INTO cleanup_protected_root_blocks(
|
|
35
|
-
cleanup_run_id,
|
|
36
|
-
scan_id,
|
|
37
|
-
protected_digest,
|
|
38
|
-
blocked_digest,
|
|
39
|
-
block_reason_code,
|
|
40
|
-
overlap_digest
|
|
41
|
-
)
|
|
42
|
-
VALUES(?, ?, ?, ?, ?, ?)
|
|
43
|
-
`)
|
|
44
|
-
.run(cleanupRunId, scanId, protectedRoot.digest, block.blockedDigest, block.blockReasonCode, block.overlapDigest);
|
|
92
|
+
this.#insertProtectedRootBlockStatement.run(cleanupRunId, scanId, protectedRoot.digest, block.blockedDigest, block.blockReasonCode, block.overlapDigest);
|
|
45
93
|
}
|
|
46
94
|
}
|
|
95
|
+
for (const tag of plan.directTargetTags) {
|
|
96
|
+
this.#insertSelectedTagStatement.run(cleanupRunId, scanId, tag);
|
|
97
|
+
}
|
|
98
|
+
this.#updateSelectedTagsDeletedStatement.run(cleanupRunId, scanId);
|
|
47
99
|
return cleanupRunId;
|
|
48
100
|
})();
|
|
49
101
|
}
|
|
50
102
|
#insertCleanupRun(scanId, plan, options) {
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
planner_inputs_json,
|
|
60
|
-
direct_target_tag_count,
|
|
61
|
-
direct_target_root_count,
|
|
62
|
-
delete_root_candidate_count,
|
|
63
|
-
untag_only_root_count,
|
|
64
|
-
fully_deletable_root_count,
|
|
65
|
-
blocked_delete_root_count,
|
|
66
|
-
protected_root_count
|
|
67
|
-
)
|
|
68
|
-
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
69
|
-
`)
|
|
70
|
-
.run(scanId, randomUUID(), options.cleanupStartedAt, resolveGitHubActionsRunUrl(), options.dryRun ? 1 : 0, JSON.stringify(plan.plannerInputs), plan.validationSummary.directTargetTagCount, plan.validationSummary.directTargetRootCount, plan.validationSummary.deleteRootCandidateCount, plan.validationSummary.untagOnlyRootCount, plan.validationSummary.fullyDeletableRootCount, plan.validationSummary.blockedDeleteRootCount, plan.validationSummary.protectedRootCount);
|
|
103
|
+
const directTargetTagCount = plan.directTargetTags.length;
|
|
104
|
+
const directTargetRootCount = plan.directTargetRoots.length;
|
|
105
|
+
const deleteRootCandidateCount = plan.directTargetRoots.filter((root) => root.selectionMode === "delete-root").length;
|
|
106
|
+
const untagOnlyRootCount = plan.rootDecisions.filter((decision) => decision.validationStatus === DeletePlanValidationStatuses.untagOnly).length;
|
|
107
|
+
const fullyDeletableRootCount = plan.fullyDeletableRoots.length;
|
|
108
|
+
const blockedDeleteRootCount = plan.rootDecisions.filter((decision) => decision.validationStatus === DeletePlanValidationStatuses.blocked).length;
|
|
109
|
+
const protectedRootCount = plan.protectedRoots.length;
|
|
110
|
+
const result = this.#insertCleanupRunStatement.run(scanId, randomUUID(), options.cleanupStartedAt, resolveGitHubActionsRunUrl(), options.dryRun ? 1 : 0, JSON.stringify(plan.plannerInputs), directTargetTagCount, directTargetRootCount, deleteRootCandidateCount, untagOnlyRootCount, fullyDeletableRootCount, blockedDeleteRootCount, protectedRootCount);
|
|
71
111
|
return Number(result.lastInsertRowid);
|
|
72
112
|
}
|
|
73
113
|
}
|
|
@@ -1,7 +1,30 @@
|
|
|
1
1
|
export class DbMergeCleanupCopy {
|
|
2
2
|
#database;
|
|
3
|
+
#insertCleanupRunStatement;
|
|
4
|
+
#copyRootDecisionsStatementByAttachName = new Map();
|
|
5
|
+
#copySelectedTagsStatementByAttachName = new Map();
|
|
6
|
+
#copyProtectedRootBlocksStatementByAttachName = new Map();
|
|
7
|
+
#listCleanupUuidsStatementByTableName = new Map();
|
|
3
8
|
constructor(database) {
|
|
4
9
|
this.#database = database;
|
|
10
|
+
this.#insertCleanupRunStatement = this.#database.prepare(`
|
|
11
|
+
INSERT INTO cleanup_runs(
|
|
12
|
+
scan_id,
|
|
13
|
+
cleanup_uuid,
|
|
14
|
+
cleanup_started_at,
|
|
15
|
+
github_actions_run_url,
|
|
16
|
+
dry_run,
|
|
17
|
+
planner_inputs_json,
|
|
18
|
+
direct_target_tag_count,
|
|
19
|
+
direct_target_root_count,
|
|
20
|
+
delete_root_candidate_count,
|
|
21
|
+
untag_only_root_count,
|
|
22
|
+
fully_deletable_root_count,
|
|
23
|
+
blocked_delete_root_count,
|
|
24
|
+
protected_root_count
|
|
25
|
+
)
|
|
26
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
27
|
+
`);
|
|
5
28
|
}
|
|
6
29
|
copyCleanupRuns(attachName, sourceScanId, targetScanId, existingCleanupUuids) {
|
|
7
30
|
const rows = this.#database
|
|
@@ -31,92 +54,119 @@ export class DbMergeCleanupCopy {
|
|
|
31
54
|
if (knownCleanupUuids.has(row.cleanup_uuid)) {
|
|
32
55
|
continue;
|
|
33
56
|
}
|
|
34
|
-
const cleanupRunId = Number(this.#
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
cleanup_uuid,
|
|
39
|
-
cleanup_started_at,
|
|
40
|
-
github_actions_run_url,
|
|
41
|
-
dry_run,
|
|
42
|
-
planner_inputs_json,
|
|
43
|
-
direct_target_tag_count,
|
|
44
|
-
direct_target_root_count,
|
|
45
|
-
delete_root_candidate_count,
|
|
46
|
-
untag_only_root_count,
|
|
47
|
-
fully_deletable_root_count,
|
|
48
|
-
blocked_delete_root_count,
|
|
49
|
-
protected_root_count
|
|
50
|
-
)
|
|
51
|
-
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
52
|
-
`)
|
|
53
|
-
.run(targetScanId, row.cleanup_uuid, row.cleanup_started_at, row.github_actions_run_url, row.dry_run, row.planner_inputs_json, row.direct_target_tag_count, row.direct_target_root_count, row.delete_root_candidate_count, row.untag_only_root_count, row.fully_deletable_root_count, row.blocked_delete_root_count, row.protected_root_count).lastInsertRowid);
|
|
54
|
-
this.#database
|
|
55
|
-
.prepare(`
|
|
56
|
-
INSERT INTO cleanup_root_decisions(
|
|
57
|
-
cleanup_run_id,
|
|
58
|
-
scan_id,
|
|
59
|
-
digest,
|
|
60
|
-
selection_mode,
|
|
61
|
-
selection_reason,
|
|
62
|
-
validation_status,
|
|
63
|
-
validation_reason_code,
|
|
64
|
-
validation_reason,
|
|
65
|
-
blocking_digest,
|
|
66
|
-
overlap_digest
|
|
67
|
-
)
|
|
68
|
-
SELECT
|
|
69
|
-
?,
|
|
70
|
-
?,
|
|
71
|
-
digest,
|
|
72
|
-
selection_mode,
|
|
73
|
-
selection_reason,
|
|
74
|
-
validation_status,
|
|
75
|
-
validation_reason_code,
|
|
76
|
-
validation_reason,
|
|
77
|
-
blocking_digest,
|
|
78
|
-
overlap_digest
|
|
79
|
-
FROM ${attachName}.cleanup_root_decisions
|
|
80
|
-
WHERE cleanup_run_id = ?
|
|
81
|
-
AND scan_id = ?
|
|
82
|
-
`)
|
|
83
|
-
.run(cleanupRunId, targetScanId, row.cleanup_run_id, sourceScanId);
|
|
84
|
-
this.#database
|
|
85
|
-
.prepare(`
|
|
86
|
-
INSERT INTO cleanup_protected_root_blocks(
|
|
87
|
-
cleanup_run_id,
|
|
88
|
-
scan_id,
|
|
89
|
-
protected_digest,
|
|
90
|
-
blocked_digest,
|
|
91
|
-
block_reason_code,
|
|
92
|
-
overlap_digest
|
|
93
|
-
)
|
|
94
|
-
SELECT
|
|
95
|
-
?,
|
|
96
|
-
?,
|
|
97
|
-
protected_digest,
|
|
98
|
-
blocked_digest,
|
|
99
|
-
block_reason_code,
|
|
100
|
-
overlap_digest
|
|
101
|
-
FROM ${attachName}.cleanup_protected_root_blocks
|
|
102
|
-
WHERE cleanup_run_id = ?
|
|
103
|
-
AND scan_id = ?
|
|
104
|
-
`)
|
|
105
|
-
.run(cleanupRunId, targetScanId, row.cleanup_run_id, sourceScanId);
|
|
57
|
+
const cleanupRunId = Number(this.#insertCleanupRunStatement.run(targetScanId, row.cleanup_uuid, row.cleanup_started_at, row.github_actions_run_url, row.dry_run, row.planner_inputs_json, row.direct_target_tag_count, row.direct_target_root_count, row.delete_root_candidate_count, row.untag_only_root_count, row.fully_deletable_root_count, row.blocked_delete_root_count, row.protected_root_count).lastInsertRowid);
|
|
58
|
+
this.#copyRootDecisionsStatement(attachName).run(cleanupRunId, targetScanId, row.cleanup_run_id, sourceScanId);
|
|
59
|
+
this.#copySelectedTagsStatement(attachName).run(cleanupRunId, targetScanId, row.cleanup_run_id, sourceScanId);
|
|
60
|
+
this.#copyProtectedRootBlocksStatement(attachName).run(cleanupRunId, targetScanId, row.cleanup_run_id, sourceScanId);
|
|
106
61
|
knownCleanupUuids.add(row.cleanup_uuid);
|
|
107
62
|
importedCleanupRunCount += 1;
|
|
108
63
|
}
|
|
109
64
|
return importedCleanupRunCount;
|
|
110
65
|
}
|
|
111
66
|
listCleanupUuids(tableName, scanId) {
|
|
112
|
-
const rows = this.#
|
|
113
|
-
.prepare(`
|
|
114
|
-
SELECT cleanup_uuid
|
|
115
|
-
FROM ${tableName}
|
|
116
|
-
WHERE scan_id = ?
|
|
117
|
-
ORDER BY cleanup_run_id
|
|
118
|
-
`)
|
|
119
|
-
.all(scanId);
|
|
67
|
+
const rows = this.#listCleanupUuidsStatement(tableName).all(scanId);
|
|
120
68
|
return rows.map((row) => row.cleanup_uuid);
|
|
121
69
|
}
|
|
70
|
+
#copyRootDecisionsStatement(attachName) {
|
|
71
|
+
const cached = this.#copyRootDecisionsStatementByAttachName.get(attachName);
|
|
72
|
+
if (cached) {
|
|
73
|
+
return cached;
|
|
74
|
+
}
|
|
75
|
+
const statement = this.#database.prepare(`
|
|
76
|
+
INSERT INTO cleanup_root_decisions(
|
|
77
|
+
cleanup_run_id,
|
|
78
|
+
scan_id,
|
|
79
|
+
digest,
|
|
80
|
+
selection_mode,
|
|
81
|
+
selection_reason,
|
|
82
|
+
validation_status,
|
|
83
|
+
validation_reason_code,
|
|
84
|
+
validation_reason,
|
|
85
|
+
blocking_digest,
|
|
86
|
+
overlap_digest
|
|
87
|
+
)
|
|
88
|
+
SELECT
|
|
89
|
+
?,
|
|
90
|
+
?,
|
|
91
|
+
digest,
|
|
92
|
+
selection_mode,
|
|
93
|
+
selection_reason,
|
|
94
|
+
validation_status,
|
|
95
|
+
validation_reason_code,
|
|
96
|
+
validation_reason,
|
|
97
|
+
blocking_digest,
|
|
98
|
+
overlap_digest
|
|
99
|
+
FROM ${attachName}.cleanup_root_decisions
|
|
100
|
+
WHERE cleanup_run_id = ?
|
|
101
|
+
AND scan_id = ?
|
|
102
|
+
`);
|
|
103
|
+
this.#copyRootDecisionsStatementByAttachName.set(attachName, statement);
|
|
104
|
+
return statement;
|
|
105
|
+
}
|
|
106
|
+
#copySelectedTagsStatement(attachName) {
|
|
107
|
+
const cached = this.#copySelectedTagsStatementByAttachName.get(attachName);
|
|
108
|
+
if (cached) {
|
|
109
|
+
return cached;
|
|
110
|
+
}
|
|
111
|
+
const statement = this.#database.prepare(`
|
|
112
|
+
INSERT INTO cleanup_selected_tags(
|
|
113
|
+
cleanup_run_id,
|
|
114
|
+
scan_id,
|
|
115
|
+
tag,
|
|
116
|
+
is_deleted
|
|
117
|
+
)
|
|
118
|
+
SELECT
|
|
119
|
+
?,
|
|
120
|
+
?,
|
|
121
|
+
tag,
|
|
122
|
+
is_deleted
|
|
123
|
+
FROM ${attachName}.cleanup_selected_tags
|
|
124
|
+
WHERE cleanup_run_id = ?
|
|
125
|
+
AND scan_id = ?
|
|
126
|
+
`);
|
|
127
|
+
this.#copySelectedTagsStatementByAttachName.set(attachName, statement);
|
|
128
|
+
return statement;
|
|
129
|
+
}
|
|
130
|
+
#copyProtectedRootBlocksStatement(attachName) {
|
|
131
|
+
const cached = this.#copyProtectedRootBlocksStatementByAttachName.get(attachName);
|
|
132
|
+
if (cached) {
|
|
133
|
+
return cached;
|
|
134
|
+
}
|
|
135
|
+
const statement = this.#database.prepare(`
|
|
136
|
+
INSERT INTO cleanup_protected_root_blocks(
|
|
137
|
+
cleanup_run_id,
|
|
138
|
+
scan_id,
|
|
139
|
+
protected_digest,
|
|
140
|
+
blocked_digest,
|
|
141
|
+
block_reason_code,
|
|
142
|
+
overlap_digest
|
|
143
|
+
)
|
|
144
|
+
SELECT
|
|
145
|
+
?,
|
|
146
|
+
?,
|
|
147
|
+
protected_digest,
|
|
148
|
+
blocked_digest,
|
|
149
|
+
block_reason_code,
|
|
150
|
+
overlap_digest
|
|
151
|
+
FROM ${attachName}.cleanup_protected_root_blocks
|
|
152
|
+
WHERE cleanup_run_id = ?
|
|
153
|
+
AND scan_id = ?
|
|
154
|
+
`);
|
|
155
|
+
this.#copyProtectedRootBlocksStatementByAttachName.set(attachName, statement);
|
|
156
|
+
return statement;
|
|
157
|
+
}
|
|
158
|
+
#listCleanupUuidsStatement(tableName) {
|
|
159
|
+
const cached = this.#listCleanupUuidsStatementByTableName.get(tableName);
|
|
160
|
+
if (cached) {
|
|
161
|
+
return cached;
|
|
162
|
+
}
|
|
163
|
+
const statement = this.#database.prepare(`
|
|
164
|
+
SELECT cleanup_uuid
|
|
165
|
+
FROM ${tableName}
|
|
166
|
+
WHERE scan_id = ?
|
|
167
|
+
ORDER BY cleanup_run_id
|
|
168
|
+
`);
|
|
169
|
+
this.#listCleanupUuidsStatementByTableName.set(tableName, statement);
|
|
170
|
+
return statement;
|
|
171
|
+
}
|
|
122
172
|
}
|
|
@@ -25,7 +25,7 @@ export class DbMergeScanCopy {
|
|
|
25
25
|
const copySpecs = [
|
|
26
26
|
"package_versions(scan_id, version_id, created_at, updated_at)",
|
|
27
27
|
"package_version_payloads(scan_id, version_id, raw_json)",
|
|
28
|
-
"tags(scan_id, tag, version_id)",
|
|
28
|
+
"tags(scan_id, tag, version_id, is_digest_tag)",
|
|
29
29
|
"manifests(scan_id, version_id, digest, media_type, artifact_type, config_media_type, subject_digest, annotations_json, platform_os, platform_architecture, platform_variant, manifest_kind)",
|
|
30
30
|
"manifest_descriptors(scan_id, parent_digest, child_digest, media_type, artifact_type, platform_os, platform_architecture, platform_variant)",
|
|
31
31
|
"manifest_payloads(scan_id, digest, raw_json)",
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ManifestKinds } from "../core/index.js";
|
|
2
|
+
const _refineManifestKindsStatementByDatabase = new WeakMap();
|
|
3
|
+
export function refineManifestKinds(database, scanId) {
|
|
4
|
+
_refineManifestKindsStatement(database).run(ManifestKinds.crossArchManifest, scanId, ManifestKinds.indexManifest);
|
|
5
|
+
}
|
|
6
|
+
function _refineManifestKindsStatement(database) {
|
|
7
|
+
const cached = _refineManifestKindsStatementByDatabase.get(database);
|
|
8
|
+
if (cached) {
|
|
9
|
+
return cached;
|
|
10
|
+
}
|
|
11
|
+
const statement = database.prepare(`
|
|
12
|
+
UPDATE manifests AS parent
|
|
13
|
+
SET manifest_kind = ?
|
|
14
|
+
WHERE parent.scan_id = ?
|
|
15
|
+
AND parent.manifest_kind = ?
|
|
16
|
+
AND parent.media_type IN (
|
|
17
|
+
'application/vnd.oci.image.index.v1+json',
|
|
18
|
+
'application/vnd.docker.distribution.manifest.list.v2+json'
|
|
19
|
+
)
|
|
20
|
+
AND NOT EXISTS (
|
|
21
|
+
SELECT 1
|
|
22
|
+
FROM tags helper_tag
|
|
23
|
+
WHERE helper_tag.scan_id = parent.scan_id
|
|
24
|
+
AND helper_tag.version_id = parent.version_id
|
|
25
|
+
AND helper_tag.is_digest_tag = 1
|
|
26
|
+
)
|
|
27
|
+
AND EXISTS (
|
|
28
|
+
SELECT 1
|
|
29
|
+
FROM manifest_descriptors descriptor
|
|
30
|
+
JOIN manifests child
|
|
31
|
+
ON child.scan_id = descriptor.scan_id
|
|
32
|
+
AND child.digest = descriptor.child_digest
|
|
33
|
+
WHERE descriptor.scan_id = parent.scan_id
|
|
34
|
+
AND descriptor.parent_digest = parent.digest
|
|
35
|
+
AND child.media_type IN (
|
|
36
|
+
'application/vnd.oci.image.manifest.v1+json',
|
|
37
|
+
'application/vnd.docker.distribution.manifest.v2+json'
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
`);
|
|
41
|
+
_refineManifestKindsStatementByDatabase.set(database, statement);
|
|
42
|
+
return statement;
|
|
43
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export function rebuildManifestReachability(database, scanId) {
|
|
2
|
+
_refreshDigestTagEdges(database, scanId);
|
|
2
3
|
const manifestDigests = _loadManifestDigests(database, scanId);
|
|
3
4
|
const childDigestsByParent = new Map();
|
|
4
5
|
const parentDigestsByChild = new Map();
|
|
@@ -70,6 +71,30 @@ export function rebuildManifestReachability(database, scanId) {
|
|
|
70
71
|
});
|
|
71
72
|
rebuild();
|
|
72
73
|
}
|
|
74
|
+
function _refreshDigestTagEdges(database, scanId) {
|
|
75
|
+
database.prepare("DELETE FROM manifest_edges WHERE scan_id = ? AND edge_kind = 'digest-tag-referrer'").run(scanId);
|
|
76
|
+
database
|
|
77
|
+
.prepare(`
|
|
78
|
+
INSERT OR IGNORE INTO manifest_edges(scan_id, parent_digest, child_digest, edge_kind)
|
|
79
|
+
SELECT
|
|
80
|
+
t.scan_id,
|
|
81
|
+
'sha256:' || SUBSTR(t.tag, 8, 64) AS parent_digest,
|
|
82
|
+
m.digest AS child_digest,
|
|
83
|
+
'digest-tag-referrer' AS edge_kind
|
|
84
|
+
FROM tags t
|
|
85
|
+
JOIN manifests m
|
|
86
|
+
ON m.scan_id = t.scan_id
|
|
87
|
+
AND m.version_id = t.version_id
|
|
88
|
+
JOIN manifests parent_manifest
|
|
89
|
+
ON parent_manifest.scan_id = t.scan_id
|
|
90
|
+
AND parent_manifest.digest = 'sha256:' || SUBSTR(t.tag, 8, 64)
|
|
91
|
+
WHERE t.scan_id = ?
|
|
92
|
+
AND t.tag LIKE 'sha256-%'
|
|
93
|
+
AND LENGTH(t.tag) >= 71
|
|
94
|
+
AND SUBSTR(t.tag, 8, 64) NOT GLOB '*[^0-9A-Fa-f]*'
|
|
95
|
+
`)
|
|
96
|
+
.run(scanId);
|
|
97
|
+
}
|
|
73
98
|
function _loadManifestDigests(database, scanId) {
|
|
74
99
|
const rows = database
|
|
75
100
|
.prepare("SELECT digest FROM manifests WHERE scan_id = ? ORDER BY digest")
|