ghcr-manager 0.0.4 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -1
- package/README.md +166 -57
- package/dist/cleanup-summary/_cleanup-summary-markdown.d.ts +6 -0
- package/dist/cleanup-summary/_cleanup-summary-markdown.js +113 -0
- package/dist/cleanup-summary/_cleanup-summary.d.ts +40 -0
- package/dist/cleanup-summary/_cleanup-summary.js +40 -0
- package/dist/cleanup-summary/index.d.ts +2 -0
- package/dist/cleanup-summary/index.js +2 -0
- package/dist/cli/_args.d.ts +1 -0
- package/dist/cli/_args.js +3 -0
- package/dist/cli/_cleanup-command.d.ts +1 -0
- package/dist/cli/_cleanup-command.js +63 -0
- package/dist/cli/_db-merge-command.d.ts +1 -0
- package/dist/cli/_db-merge-command.js +41 -0
- package/dist/cli/_github-output.d.ts +10 -0
- package/dist/cli/_github-output.js +13 -0
- package/dist/cli/_logger.d.ts +2 -1
- package/dist/cli/_logger.js +2 -0
- package/dist/cli/_older-than.d.ts +5 -0
- package/dist/cli/_older-than.js +42 -0
- package/dist/cli/_planner-options.d.ts +20 -0
- package/dist/cli/_planner-options.js +101 -0
- package/dist/cli/_scan-command.js +11 -4
- package/dist/cli/_tag-selector-resolver.d.ts +3 -0
- package/dist/cli/_tag-selector-resolver.js +109 -0
- package/dist/cli/_untag-command.d.ts +1 -0
- package/dist/cli/_untag-command.js +57 -0
- package/dist/cli/index.js +19 -1
- package/dist/config/_service-constants.d.ts +3 -0
- package/dist/config/_service-constants.js +3 -0
- package/dist/{tuning → config}/index.d.ts +3 -0
- package/dist/{tuning → config}/index.js +3 -0
- package/dist/core/_github-package-owner.d.ts +11 -0
- package/dist/core/_github-package-owner.js +45 -0
- package/dist/core/_http-error.d.ts +6 -0
- package/dist/core/_http-error.js +33 -0
- package/dist/core/_types.d.ts +3 -2
- package/dist/core/index.d.ts +4 -1
- package/dist/core/index.js +2 -1
- package/dist/db/_cleanup-run-writer.d.ts +10 -0
- package/dist/db/_cleanup-run-writer.js +73 -0
- package/dist/db/_db-merge-cleanup-copy.d.ts +7 -0
- package/dist/db/_db-merge-cleanup-copy.js +122 -0
- package/dist/db/_db-merge-history.d.ts +2 -0
- package/dist/db/_db-merge-history.js +15 -0
- package/dist/db/_db-merge-repository.d.ts +8 -0
- package/dist/db/_db-merge-repository.js +95 -0
- package/dist/db/_db-merge-scan-copy.d.ts +10 -0
- package/dist/db/_db-merge-scan-copy.js +69 -0
- package/dist/db/_db-merge-types.d.ts +44 -0
- package/dist/db/_db-merge-types.js +1 -0
- package/dist/db/_github-actions-run-url.d.ts +1 -0
- package/dist/db/_github-actions-run-url.js +9 -0
- package/dist/db/_scan-writer.d.ts +3 -1
- package/dist/db/_scan-writer.js +28 -13
- package/dist/db/_snapshot-repository.d.ts +9 -9
- package/dist/db/_snapshot-repository.js +37 -49
- package/dist/db/_sql-placeholders.d.ts +2 -0
- package/dist/db/_sql-placeholders.js +16 -0
- package/dist/db/index.d.ts +5 -0
- package/dist/db/index.js +3 -0
- package/dist/db/planner/_planner-delete-tag-root-targets.d.ts +7 -0
- package/dist/db/planner/_planner-delete-tag-root-targets.js +130 -0
- package/dist/db/planner/_planner-direct-target-tags.d.ts +6 -0
- package/dist/db/planner/_planner-direct-target-tags.js +47 -0
- package/dist/db/planner/_planner-keep-tagged-root-targets.d.ts +7 -0
- package/dist/db/planner/_planner-keep-tagged-root-targets.js +74 -0
- package/dist/db/planner/_planner-output.d.ts +5 -0
- package/dist/db/planner/_planner-output.js +101 -0
- package/dist/db/planner/_planner-plan-artifacts.d.ts +7 -0
- package/dist/db/planner/_planner-plan-artifacts.js +211 -0
- package/dist/db/planner/_planner-repository.d.ts +34 -0
- package/dist/db/planner/_planner-repository.js +126 -0
- package/dist/db/planner/_planner-sql.d.ts +12 -0
- package/dist/db/planner/_planner-sql.js +35 -0
- package/dist/db/planner/_planner-tag-selectors.d.ts +8 -0
- package/dist/db/planner/_planner-tag-selectors.js +57 -0
- package/dist/db/planner/_planner-tagged-root-targets.d.ts +15 -0
- package/dist/db/planner/_planner-tagged-root-targets.js +19 -0
- package/dist/db/planner/_planner-tagged-targets.d.ts +8 -0
- package/dist/db/planner/_planner-tagged-targets.js +16 -0
- package/dist/db/planner/_planner-types.d.ts +135 -0
- package/dist/db/planner/_planner-types.js +38 -0
- package/dist/db/planner/_planner-untagged-targets.d.ts +9 -0
- package/dist/db/planner/_planner-untagged-targets.js +91 -0
- package/dist/db/planner/index.d.ts +2 -0
- package/dist/db/planner/index.js +1 -0
- package/dist/execute/_http.d.ts +7 -0
- package/dist/execute/_http.js +48 -0
- package/dist/execute/_manifest-detach.d.ts +4 -0
- package/dist/execute/_manifest-detach.js +31 -0
- package/dist/execute/_package-version-delete-client.d.ts +4 -0
- package/dist/execute/_package-version-delete-client.js +34 -0
- package/dist/execute/_package-version-page-client.d.ts +14 -0
- package/dist/execute/_package-version-page-client.js +64 -0
- package/dist/execute/_package-version-tag-source-client.d.ts +12 -0
- package/dist/execute/_package-version-tag-source-client.js +65 -0
- package/dist/execute/_plan-executor.d.ts +3 -0
- package/dist/execute/_plan-executor.js +47 -0
- package/dist/execute/_registry-manifest-client.d.ts +12 -0
- package/dist/execute/_registry-manifest-client.js +79 -0
- package/dist/execute/_registry-token-client.d.ts +4 -0
- package/dist/execute/_registry-token-client.js +37 -0
- package/dist/execute/_types.d.ts +51 -0
- package/dist/execute/_types.js +1 -0
- package/dist/execute/_untag-client.d.ts +2 -0
- package/dist/execute/_untag-client.js +71 -0
- package/dist/execute/index.d.ts +5 -0
- package/dist/execute/index.js +3 -0
- package/dist/ingest/github/_manifest-client.d.ts +7 -1
- package/dist/ingest/github/_manifest-client.js +8 -0
- package/dist/ingest/github/_manifest-ingest.js +39 -53
- package/dist/ingest/github/_manifest-kind.d.ts +20 -0
- package/dist/ingest/github/_manifest-kind.js +50 -0
- package/dist/ingest/github/_package-metadata-load.d.ts +5 -0
- package/dist/ingest/github/_package-metadata-load.js +45 -0
- package/dist/ingest/github/_package-version-page-load.d.ts +1 -1
- package/dist/ingest/github/_package-version-page-load.js +8 -5
- package/dist/ingest/github/_packages-client.d.ts +1 -1
- package/dist/ingest/github/_packages-client.js +21 -4
- package/dist/ingest/github/_parallel-paginated-ingest.d.ts +1 -0
- package/dist/ingest/github/_parallel-paginated-ingest.js +2 -2
- package/dist/ingest/github/_shared.d.ts +1 -1
- package/dist/ingest/github/_shared.js +2 -34
- package/dist/ingest/github/index.d.ts +4 -0
- package/dist/ingest/github/index.js +8 -5
- package/package.json +7 -5
- package/resources/sql/schema/001_schema.sql +82 -8
- package/resources/sql/views/001_v_latest_scan_per_package.sql +2 -2
- package/resources/sql/views/003_v_scan_root_manifests.sql +43 -0
- package/resources/sql/views/004_v_digest_derived_tag_relations.sql +51 -0
- package/resources/sql/views/005_v_cleanup_root_closure_members.sql +100 -0
- package/resources/sql/views/006_v_cleanup_blocking_overlaps.sql +42 -0
- package/dist/ingest/github/_paginated-ingest.d.ts +0 -11
- package/dist/ingest/github/_paginated-ingest.js +0 -28
- package/resources/sql/views/003_v_missing_digests_related_manifests.sql +0 -78
- package/resources/sql/views/004_v_manifests_related_manifests.sql +0 -142
|
@@ -1,20 +1,23 @@
|
|
|
1
|
+
import { ghcrRegistryBaseUrl } from "../../config/index.js";
|
|
1
2
|
import { ingestManifests } from "./_manifest-ingest.js";
|
|
3
|
+
import { loadPackageMetadata } from "./_package-metadata-load.js";
|
|
2
4
|
import { ingestPackageVersions } from "./_packages-client.js";
|
|
3
5
|
import { defaultFetch } from "./_shared.js";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
+
export { loadPackageMetadata } from "./_package-metadata-load.js";
|
|
7
|
+
export { defaultFetch } from "./_shared.js";
|
|
6
8
|
export async function importGitHubScan(options, writer, repository, runtime) {
|
|
7
9
|
const fetchImpl = runtime?.fetchImpl ?? defaultFetch;
|
|
8
10
|
const scanStartedAt = new Date().toISOString();
|
|
9
11
|
const fullPackageName = `${options.owner}/${options.packageName}`;
|
|
10
|
-
|
|
12
|
+
const packageMetadata = runtime?.packageMetadata ?? (await loadPackageMetadata(fetchImpl, options));
|
|
13
|
+
writer.startScan(options.owner, options.packageName, scanStartedAt, packageMetadata);
|
|
11
14
|
const scanId = writer.getActiveScanId();
|
|
12
15
|
options.logger.info(`Starting GitHub package scan for ${fullPackageName}`);
|
|
13
16
|
try {
|
|
14
17
|
options.logger.info(`Starting remote data pull for ${fullPackageName}`);
|
|
15
|
-
const counts = await ingestPackageVersions(fetchImpl,
|
|
18
|
+
const counts = await ingestPackageVersions(fetchImpl, options, writer);
|
|
16
19
|
options.logger.info(`Loaded ${counts.packageVersions} package versions and ${counts.tags} tags`);
|
|
17
|
-
await ingestManifests(fetchImpl,
|
|
20
|
+
await ingestManifests(fetchImpl, ghcrRegistryBaseUrl, options, writer, repository, scanId);
|
|
18
21
|
options.logger.info(`Completed remote data pull for ${fullPackageName}`);
|
|
19
22
|
writer.markScanCompleted(new Date().toISOString());
|
|
20
23
|
options.logger.info(`Completed GitHub package scan for ${fullPackageName}`);
|
package/package.json
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"type": "git",
|
|
10
10
|
"url": "https://github.com/gh-workflow/ghcr-manager"
|
|
11
11
|
},
|
|
12
|
-
"version": "0.0
|
|
12
|
+
"version": "0.9.0",
|
|
13
13
|
"type": "module",
|
|
14
14
|
"engines": {
|
|
15
15
|
"node": ">=20.0.0"
|
|
@@ -28,11 +28,12 @@
|
|
|
28
28
|
"build": "tsc --project tsconfig.json",
|
|
29
29
|
"coverage": "c8 --src src --reporter=text --reporter=text-summary node --import tsx --test",
|
|
30
30
|
"ghcr-manager": "tsx src/cli/index.ts",
|
|
31
|
+
"ghcr-manager:dist": "node dist/cli/index.js",
|
|
31
32
|
"format": "prettier --write .",
|
|
32
33
|
"format:check": "prettier --check .",
|
|
33
34
|
"lint": "npm run check:file-names && npm run check:test-mapping && npm run typecheck && npm run lint:ts && npm run lint:yaml && npm run lint:markdown && npm run format:check",
|
|
34
|
-
"check:file-names": "node tools/check-src-file-names.mjs",
|
|
35
|
-
"check:test-mapping": "node tools/check-test-mapping.mjs",
|
|
35
|
+
"check:file-names": "node tools/tests/check-src-file-names.mjs",
|
|
36
|
+
"check:test-mapping": "node tools/tests/check-test-mapping.mjs",
|
|
36
37
|
"lint:ts": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
|
|
37
38
|
"lint:yaml": "eslint \".github/**/*.yml\" \"*.yml\"",
|
|
38
39
|
"lint:markdown": "markdownlint-cli2",
|
|
@@ -51,9 +52,10 @@
|
|
|
51
52
|
"eslint-plugin-yml": "^3.3.2",
|
|
52
53
|
"globals": "^17.5.0",
|
|
53
54
|
"markdownlint-cli2": "^0.22.1",
|
|
54
|
-
"prettier": "^3.
|
|
55
|
+
"prettier": "^3.8.3",
|
|
55
56
|
"tsx": "^4.20.6",
|
|
57
|
+
"typescript": "^6.0.3",
|
|
56
58
|
"typescript-eslint": "^8.46.2",
|
|
57
|
-
"
|
|
59
|
+
"yaml": "^2.9.0"
|
|
58
60
|
}
|
|
59
61
|
}
|
|
@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS package_scans (
|
|
|
5
5
|
scan_uuid TEXT NOT NULL UNIQUE,
|
|
6
6
|
owner TEXT NOT NULL,
|
|
7
7
|
package_name TEXT NOT NULL,
|
|
8
|
+
package_metadata_json TEXT NOT NULL,
|
|
9
|
+
github_actions_run_url TEXT,
|
|
8
10
|
scan_started_at TEXT NOT NULL,
|
|
9
11
|
scan_completed_at TEXT,
|
|
10
12
|
status TEXT NOT NULL,
|
|
@@ -14,11 +16,9 @@ CREATE TABLE IF NOT EXISTS package_scans (
|
|
|
14
16
|
CREATE TABLE IF NOT EXISTS package_versions (
|
|
15
17
|
scan_id INTEGER NOT NULL,
|
|
16
18
|
version_id INTEGER NOT NULL,
|
|
17
|
-
digest TEXT NOT NULL,
|
|
18
19
|
created_at TEXT NOT NULL,
|
|
19
20
|
updated_at TEXT NOT NULL,
|
|
20
21
|
PRIMARY KEY(scan_id, version_id),
|
|
21
|
-
UNIQUE(scan_id, version_id, digest),
|
|
22
22
|
FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
|
|
23
23
|
);
|
|
24
24
|
|
|
@@ -33,14 +33,14 @@ CREATE TABLE IF NOT EXISTS package_version_payloads (
|
|
|
33
33
|
CREATE TABLE IF NOT EXISTS tags (
|
|
34
34
|
scan_id INTEGER NOT NULL,
|
|
35
35
|
tag TEXT NOT NULL,
|
|
36
|
-
digest TEXT NOT NULL,
|
|
37
36
|
version_id INTEGER NOT NULL,
|
|
38
37
|
PRIMARY KEY(scan_id, tag),
|
|
39
|
-
FOREIGN KEY(scan_id, version_id
|
|
38
|
+
FOREIGN KEY(scan_id, version_id) REFERENCES package_versions(scan_id, version_id)
|
|
40
39
|
);
|
|
41
40
|
|
|
42
41
|
CREATE TABLE IF NOT EXISTS manifests (
|
|
43
42
|
scan_id INTEGER NOT NULL,
|
|
43
|
+
version_id INTEGER NOT NULL,
|
|
44
44
|
digest TEXT NOT NULL,
|
|
45
45
|
media_type TEXT NOT NULL,
|
|
46
46
|
artifact_type TEXT,
|
|
@@ -50,8 +50,17 @@ CREATE TABLE IF NOT EXISTS manifests (
|
|
|
50
50
|
platform_os TEXT,
|
|
51
51
|
platform_architecture TEXT,
|
|
52
52
|
platform_variant TEXT,
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
manifest_kind TEXT,
|
|
54
|
+
CHECK(manifest_kind IN (
|
|
55
|
+
'image_index',
|
|
56
|
+
'image_manifest',
|
|
57
|
+
'artifact_manifest',
|
|
58
|
+
'attestation_manifest',
|
|
59
|
+
'signature_manifest'
|
|
60
|
+
)),
|
|
61
|
+
PRIMARY KEY(scan_id, version_id),
|
|
62
|
+
UNIQUE(scan_id, digest),
|
|
63
|
+
FOREIGN KEY(scan_id, version_id) REFERENCES package_versions(scan_id, version_id)
|
|
55
64
|
);
|
|
56
65
|
|
|
57
66
|
CREATE TABLE IF NOT EXISTS manifest_descriptors (
|
|
@@ -96,14 +105,79 @@ CREATE TABLE IF NOT EXISTS manifest_reachability (
|
|
|
96
105
|
CHECK(min_distance >= 0)
|
|
97
106
|
);
|
|
98
107
|
|
|
108
|
+
CREATE TABLE IF NOT EXISTS cleanup_runs (
|
|
109
|
+
cleanup_run_id INTEGER PRIMARY KEY,
|
|
110
|
+
scan_id INTEGER NOT NULL,
|
|
111
|
+
cleanup_uuid TEXT NOT NULL,
|
|
112
|
+
cleanup_started_at TEXT NOT NULL,
|
|
113
|
+
github_actions_run_url TEXT,
|
|
114
|
+
dry_run INTEGER NOT NULL,
|
|
115
|
+
planner_inputs_json TEXT NOT NULL,
|
|
116
|
+
direct_target_tag_count INTEGER NOT NULL,
|
|
117
|
+
direct_target_root_count INTEGER NOT NULL,
|
|
118
|
+
delete_root_candidate_count INTEGER NOT NULL,
|
|
119
|
+
untag_only_root_count INTEGER NOT NULL,
|
|
120
|
+
fully_deletable_root_count INTEGER NOT NULL,
|
|
121
|
+
blocked_delete_root_count INTEGER NOT NULL,
|
|
122
|
+
protected_root_count INTEGER NOT NULL,
|
|
123
|
+
CHECK(dry_run IN (0, 1)),
|
|
124
|
+
UNIQUE(cleanup_run_id, scan_id),
|
|
125
|
+
FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
|
|
126
|
+
);
|
|
127
|
+
|
|
128
|
+
CREATE TABLE IF NOT EXISTS cleanup_root_decisions (
|
|
129
|
+
cleanup_run_id INTEGER NOT NULL,
|
|
130
|
+
scan_id INTEGER NOT NULL,
|
|
131
|
+
digest TEXT NOT NULL,
|
|
132
|
+
selection_mode TEXT NOT NULL,
|
|
133
|
+
selection_reason TEXT NOT NULL,
|
|
134
|
+
validation_status TEXT NOT NULL,
|
|
135
|
+
validation_reason_code TEXT NOT NULL,
|
|
136
|
+
validation_reason TEXT NOT NULL,
|
|
137
|
+
blocking_digest TEXT,
|
|
138
|
+
overlap_digest TEXT,
|
|
139
|
+
PRIMARY KEY(cleanup_run_id, digest),
|
|
140
|
+
CHECK(validation_status IN ('fully-deletable', 'blocked', 'untag-only')),
|
|
141
|
+
CHECK(validation_reason_code IN (
|
|
142
|
+
'untag-only-partial-tag-match',
|
|
143
|
+
'fully-deletable-no-retained-overlap',
|
|
144
|
+
'blocked-overlap-with-retained-root'
|
|
145
|
+
)),
|
|
146
|
+
FOREIGN KEY(cleanup_run_id, scan_id) REFERENCES cleanup_runs(cleanup_run_id, scan_id),
|
|
147
|
+
FOREIGN KEY(scan_id, digest) REFERENCES manifests(scan_id, digest),
|
|
148
|
+
FOREIGN KEY(scan_id, blocking_digest) REFERENCES manifests(scan_id, digest),
|
|
149
|
+
FOREIGN KEY(scan_id, overlap_digest) REFERENCES manifests(scan_id, digest)
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
CREATE TABLE IF NOT EXISTS cleanup_protected_root_blocks (
|
|
153
|
+
cleanup_run_id INTEGER NOT NULL,
|
|
154
|
+
scan_id INTEGER NOT NULL,
|
|
155
|
+
protected_digest TEXT NOT NULL,
|
|
156
|
+
blocked_digest TEXT NOT NULL,
|
|
157
|
+
block_reason_code TEXT NOT NULL,
|
|
158
|
+
overlap_digest TEXT NOT NULL,
|
|
159
|
+
PRIMARY KEY(cleanup_run_id, protected_digest, blocked_digest, overlap_digest),
|
|
160
|
+
CHECK(block_reason_code IN ('overlap-with-retained-root')),
|
|
161
|
+
FOREIGN KEY(cleanup_run_id, scan_id) REFERENCES cleanup_runs(cleanup_run_id, scan_id),
|
|
162
|
+
FOREIGN KEY(scan_id, protected_digest, overlap_digest)
|
|
163
|
+
REFERENCES manifest_reachability(scan_id, ancestor_digest, descendant_digest),
|
|
164
|
+
FOREIGN KEY(scan_id, blocked_digest, overlap_digest)
|
|
165
|
+
REFERENCES manifest_reachability(scan_id, ancestor_digest, descendant_digest)
|
|
166
|
+
);
|
|
167
|
+
|
|
99
168
|
CREATE INDEX IF NOT EXISTS idx_package_versions_scan_created_at ON package_versions(scan_id, created_at);
|
|
100
|
-
CREATE INDEX IF NOT EXISTS idx_package_versions_scan_digest ON package_versions(scan_id, digest);
|
|
101
169
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_package_scans_scan_uuid ON package_scans(scan_uuid);
|
|
170
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_cleanup_runs_cleanup_uuid ON cleanup_runs(cleanup_uuid);
|
|
102
171
|
CREATE INDEX IF NOT EXISTS idx_package_scans_owner_name_started_at
|
|
103
172
|
ON package_scans(owner, package_name, scan_started_at DESC);
|
|
104
|
-
CREATE INDEX IF NOT EXISTS
|
|
173
|
+
CREATE INDEX IF NOT EXISTS idx_cleanup_runs_scan_id ON cleanup_runs(scan_id);
|
|
174
|
+
CREATE INDEX IF NOT EXISTS idx_cleanup_protected_root_blocks_run_blocked
|
|
175
|
+
ON cleanup_protected_root_blocks(cleanup_run_id, blocked_digest);
|
|
176
|
+
CREATE INDEX IF NOT EXISTS idx_tags_scan_version ON tags(scan_id, version_id);
|
|
105
177
|
CREATE INDEX IF NOT EXISTS idx_manifest_descriptors_scan_child ON manifest_descriptors(scan_id, child_digest);
|
|
106
178
|
CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_parent ON manifest_edges(scan_id, parent_digest);
|
|
107
179
|
CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_child ON manifest_edges(scan_id, child_digest);
|
|
108
180
|
CREATE INDEX IF NOT EXISTS idx_manifest_reachability_scan_descendant
|
|
109
181
|
ON manifest_reachability(scan_id, descendant_digest);
|
|
182
|
+
CREATE INDEX IF NOT EXISTS idx_manifest_reachability_scan_descendant_distance
|
|
183
|
+
ON manifest_reachability(scan_id, descendant_digest, min_distance);
|
|
@@ -17,11 +17,11 @@ FROM (
|
|
|
17
17
|
ps.scan_completed_at,
|
|
18
18
|
ROW_NUMBER() OVER (
|
|
19
19
|
PARTITION BY ps.owner, ps.package_name
|
|
20
|
-
ORDER BY ps.scan_completed_at DESC
|
|
20
|
+
ORDER BY ps.scan_completed_at DESC, ps.scan_id DESC
|
|
21
21
|
) AS rn
|
|
22
22
|
FROM package_scans ps
|
|
23
23
|
WHERE ps.scan_completed_at IS NOT NULL
|
|
24
24
|
AND ps.status = 'completed'
|
|
25
25
|
)
|
|
26
26
|
WHERE rn = 1
|
|
27
|
-
;
|
|
27
|
+
;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
DROP VIEW IF EXISTS v_scan_root_manifests;
|
|
2
|
+
|
|
3
|
+
CREATE VIEW v_scan_root_manifests AS
|
|
4
|
+
SELECT
|
|
5
|
+
ps.scan_id,
|
|
6
|
+
ps.owner,
|
|
7
|
+
ps.package_name,
|
|
8
|
+
m.version_id AS root_version_id,
|
|
9
|
+
m.digest AS root_digest,
|
|
10
|
+
m.manifest_kind AS root_manifest_kind,
|
|
11
|
+
pv.created_at,
|
|
12
|
+
pv.updated_at,
|
|
13
|
+
COUNT(t.tag) AS tag_count,
|
|
14
|
+
CASE WHEN COUNT(t.tag) > 0 THEN 1 ELSE 0 END AS is_tagged,
|
|
15
|
+
CASE
|
|
16
|
+
WHEN EXISTS (
|
|
17
|
+
SELECT 1
|
|
18
|
+
FROM manifest_reachability mr
|
|
19
|
+
WHERE mr.scan_id = m.scan_id
|
|
20
|
+
AND mr.descendant_digest = m.digest
|
|
21
|
+
AND mr.min_distance > 0
|
|
22
|
+
) THEN 1
|
|
23
|
+
ELSE 0
|
|
24
|
+
END AS has_ancestor
|
|
25
|
+
FROM manifests m
|
|
26
|
+
JOIN package_versions pv
|
|
27
|
+
ON pv.scan_id = m.scan_id
|
|
28
|
+
AND pv.version_id = m.version_id
|
|
29
|
+
JOIN package_scans ps
|
|
30
|
+
ON ps.scan_id = m.scan_id
|
|
31
|
+
LEFT JOIN tags t
|
|
32
|
+
ON t.scan_id = m.scan_id
|
|
33
|
+
AND t.version_id = m.version_id
|
|
34
|
+
GROUP BY
|
|
35
|
+
ps.scan_id,
|
|
36
|
+
ps.owner,
|
|
37
|
+
ps.package_name,
|
|
38
|
+
m.version_id,
|
|
39
|
+
m.digest,
|
|
40
|
+
m.manifest_kind,
|
|
41
|
+
pv.created_at,
|
|
42
|
+
pv.updated_at
|
|
43
|
+
;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
DROP VIEW IF EXISTS v_digest_derived_tag_relations;
|
|
2
|
+
|
|
3
|
+
CREATE VIEW v_digest_derived_tag_relations AS
|
|
4
|
+
WITH digest_like_tags AS (
|
|
5
|
+
SELECT
|
|
6
|
+
lsp.scan_id,
|
|
7
|
+
lsp.owner,
|
|
8
|
+
lsp.package_name,
|
|
9
|
+
t.tag,
|
|
10
|
+
t.version_id AS artifact_version_id,
|
|
11
|
+
m.digest AS artifact_digest,
|
|
12
|
+
m.manifest_kind AS artifact_manifest_kind,
|
|
13
|
+
m.subject_digest AS artifact_subject_digest,
|
|
14
|
+
'sha256:' || SUBSTR(t.tag, 8, 64) AS inferred_parent_digest,
|
|
15
|
+
SUBSTR(t.tag, 72) AS tag_suffix
|
|
16
|
+
FROM tags t
|
|
17
|
+
JOIN v_latest_scan_per_package lsp
|
|
18
|
+
ON lsp.scan_id = t.scan_id
|
|
19
|
+
JOIN manifests m
|
|
20
|
+
ON m.scan_id = t.scan_id
|
|
21
|
+
AND m.version_id = t.version_id
|
|
22
|
+
WHERE t.tag LIKE 'sha256-%'
|
|
23
|
+
AND LENGTH(t.tag) >= 71
|
|
24
|
+
AND SUBSTR(t.tag, 8, 64) NOT GLOB '*[^0-9A-Fa-f]*'
|
|
25
|
+
)
|
|
26
|
+
SELECT
|
|
27
|
+
dlt.scan_id,
|
|
28
|
+
dlt.owner,
|
|
29
|
+
dlt.package_name,
|
|
30
|
+
dlt.tag,
|
|
31
|
+
dlt.artifact_version_id,
|
|
32
|
+
dlt.artifact_digest,
|
|
33
|
+
dlt.artifact_manifest_kind,
|
|
34
|
+
dlt.artifact_subject_digest,
|
|
35
|
+
dlt.inferred_parent_digest,
|
|
36
|
+
dlt.tag_suffix,
|
|
37
|
+
pm.version_id AS parent_version_id,
|
|
38
|
+
pm.digest AS parent_digest,
|
|
39
|
+
pm.manifest_kind AS parent_manifest_kind,
|
|
40
|
+
CASE
|
|
41
|
+
WHEN dlt.artifact_subject_digest = dlt.inferred_parent_digest THEN 1
|
|
42
|
+
ELSE 0
|
|
43
|
+
END AS subject_matches_inferred_parent,
|
|
44
|
+
CASE
|
|
45
|
+
WHEN pm.digest IS NULL THEN 0
|
|
46
|
+
ELSE 1
|
|
47
|
+
END AS parent_exists
|
|
48
|
+
FROM digest_like_tags dlt
|
|
49
|
+
LEFT JOIN manifests pm
|
|
50
|
+
ON pm.scan_id = dlt.scan_id
|
|
51
|
+
AND pm.digest = dlt.inferred_parent_digest;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
DROP VIEW IF EXISTS v_cleanup_root_closure_members;
|
|
2
|
+
|
|
3
|
+
CREATE VIEW v_cleanup_root_closure_members AS
|
|
4
|
+
WITH selected_roots AS (
|
|
5
|
+
SELECT
|
|
6
|
+
run.cleanup_run_id,
|
|
7
|
+
run.scan_id,
|
|
8
|
+
ps.owner,
|
|
9
|
+
ps.package_name,
|
|
10
|
+
decision.digest AS root_digest,
|
|
11
|
+
root_manifest.version_id AS root_version_id,
|
|
12
|
+
root_manifest.manifest_kind AS root_manifest_kind,
|
|
13
|
+
decision.selection_mode,
|
|
14
|
+
decision.selection_reason,
|
|
15
|
+
decision.validation_status,
|
|
16
|
+
decision.validation_reason_code,
|
|
17
|
+
decision.validation_reason
|
|
18
|
+
FROM cleanup_root_decisions decision
|
|
19
|
+
JOIN cleanup_runs run
|
|
20
|
+
ON run.cleanup_run_id = decision.cleanup_run_id
|
|
21
|
+
AND run.scan_id = decision.scan_id
|
|
22
|
+
JOIN package_scans ps
|
|
23
|
+
ON ps.scan_id = run.scan_id
|
|
24
|
+
JOIN manifests root_manifest
|
|
25
|
+
ON root_manifest.scan_id = decision.scan_id
|
|
26
|
+
AND root_manifest.digest = decision.digest
|
|
27
|
+
),
|
|
28
|
+
closure_members AS (
|
|
29
|
+
SELECT
|
|
30
|
+
sr.cleanup_run_id,
|
|
31
|
+
sr.scan_id,
|
|
32
|
+
sr.owner,
|
|
33
|
+
sr.package_name,
|
|
34
|
+
sr.root_digest,
|
|
35
|
+
sr.root_version_id,
|
|
36
|
+
sr.root_manifest_kind,
|
|
37
|
+
sr.selection_mode,
|
|
38
|
+
sr.selection_reason,
|
|
39
|
+
sr.validation_status,
|
|
40
|
+
sr.validation_reason_code,
|
|
41
|
+
sr.validation_reason,
|
|
42
|
+
sr.root_digest AS member_digest,
|
|
43
|
+
sr.root_version_id AS member_version_id,
|
|
44
|
+
sr.root_manifest_kind AS member_manifest_kind,
|
|
45
|
+
0 AS hops_from_root,
|
|
46
|
+
'root' AS member_role
|
|
47
|
+
FROM selected_roots sr
|
|
48
|
+
|
|
49
|
+
UNION ALL
|
|
50
|
+
|
|
51
|
+
SELECT
|
|
52
|
+
sr.cleanup_run_id,
|
|
53
|
+
sr.scan_id,
|
|
54
|
+
sr.owner,
|
|
55
|
+
sr.package_name,
|
|
56
|
+
sr.root_digest,
|
|
57
|
+
sr.root_version_id,
|
|
58
|
+
sr.root_manifest_kind,
|
|
59
|
+
sr.selection_mode,
|
|
60
|
+
sr.selection_reason,
|
|
61
|
+
sr.validation_status,
|
|
62
|
+
sr.validation_reason_code,
|
|
63
|
+
sr.validation_reason,
|
|
64
|
+
member_manifest.digest AS member_digest,
|
|
65
|
+
member_manifest.version_id AS member_version_id,
|
|
66
|
+
member_manifest.manifest_kind AS member_manifest_kind,
|
|
67
|
+
reachability.min_distance AS hops_from_root,
|
|
68
|
+
'descendant' AS member_role
|
|
69
|
+
FROM selected_roots sr
|
|
70
|
+
JOIN manifest_reachability reachability
|
|
71
|
+
ON reachability.scan_id = sr.scan_id
|
|
72
|
+
AND reachability.ancestor_digest = sr.root_digest
|
|
73
|
+
AND reachability.min_distance > 0
|
|
74
|
+
JOIN manifests member_manifest
|
|
75
|
+
ON member_manifest.scan_id = sr.scan_id
|
|
76
|
+
AND member_manifest.digest = reachability.descendant_digest
|
|
77
|
+
)
|
|
78
|
+
SELECT
|
|
79
|
+
cm.cleanup_run_id,
|
|
80
|
+
cm.scan_id,
|
|
81
|
+
cm.owner,
|
|
82
|
+
cm.package_name,
|
|
83
|
+
cm.root_digest,
|
|
84
|
+
cm.root_version_id,
|
|
85
|
+
cm.root_manifest_kind,
|
|
86
|
+
cm.selection_mode,
|
|
87
|
+
cm.selection_reason,
|
|
88
|
+
cm.validation_status,
|
|
89
|
+
cm.validation_reason_code,
|
|
90
|
+
cm.validation_reason,
|
|
91
|
+
cm.member_digest,
|
|
92
|
+
cm.member_version_id,
|
|
93
|
+
cm.member_manifest_kind,
|
|
94
|
+
cm.hops_from_root,
|
|
95
|
+
cm.member_role,
|
|
96
|
+
tag.tag AS member_tag
|
|
97
|
+
FROM closure_members cm
|
|
98
|
+
LEFT JOIN tags tag
|
|
99
|
+
ON tag.scan_id = cm.scan_id
|
|
100
|
+
AND tag.version_id = cm.member_version_id;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
DROP VIEW IF EXISTS v_cleanup_blocking_overlaps;
|
|
2
|
+
|
|
3
|
+
CREATE VIEW v_cleanup_blocking_overlaps AS
|
|
4
|
+
SELECT
|
|
5
|
+
run.cleanup_run_id,
|
|
6
|
+
run.scan_id,
|
|
7
|
+
ps.owner,
|
|
8
|
+
ps.package_name,
|
|
9
|
+
block.protected_digest,
|
|
10
|
+
protected_manifest.version_id AS protected_version_id,
|
|
11
|
+
protected_manifest.manifest_kind AS protected_manifest_kind,
|
|
12
|
+
block.blocked_digest,
|
|
13
|
+
blocked_manifest.version_id AS blocked_version_id,
|
|
14
|
+
blocked_manifest.manifest_kind AS blocked_manifest_kind,
|
|
15
|
+
decision.selection_mode AS blocked_selection_mode,
|
|
16
|
+
decision.selection_reason AS blocked_selection_reason,
|
|
17
|
+
decision.validation_status AS blocked_validation_status,
|
|
18
|
+
decision.validation_reason_code AS blocked_validation_reason_code,
|
|
19
|
+
decision.validation_reason AS blocked_validation_reason,
|
|
20
|
+
block.block_reason_code,
|
|
21
|
+
block.overlap_digest,
|
|
22
|
+
overlap_manifest.version_id AS overlap_version_id,
|
|
23
|
+
overlap_manifest.manifest_kind AS overlap_manifest_kind
|
|
24
|
+
FROM cleanup_protected_root_blocks block
|
|
25
|
+
JOIN cleanup_runs run
|
|
26
|
+
ON run.cleanup_run_id = block.cleanup_run_id
|
|
27
|
+
AND run.scan_id = block.scan_id
|
|
28
|
+
JOIN package_scans ps
|
|
29
|
+
ON ps.scan_id = run.scan_id
|
|
30
|
+
JOIN manifests protected_manifest
|
|
31
|
+
ON protected_manifest.scan_id = block.scan_id
|
|
32
|
+
AND protected_manifest.digest = block.protected_digest
|
|
33
|
+
JOIN manifests blocked_manifest
|
|
34
|
+
ON blocked_manifest.scan_id = block.scan_id
|
|
35
|
+
AND blocked_manifest.digest = block.blocked_digest
|
|
36
|
+
JOIN cleanup_root_decisions decision
|
|
37
|
+
ON decision.cleanup_run_id = block.cleanup_run_id
|
|
38
|
+
AND decision.scan_id = block.scan_id
|
|
39
|
+
AND decision.digest = block.blocked_digest
|
|
40
|
+
LEFT JOIN manifests overlap_manifest
|
|
41
|
+
ON overlap_manifest.scan_id = block.scan_id
|
|
42
|
+
AND overlap_manifest.digest = block.overlap_digest;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { GitHubScanLogger } from "./_shared.js";
|
|
2
|
-
export interface PaginatedIngestOptions<T> {
|
|
3
|
-
loadPage(page: number): Promise<T[]>;
|
|
4
|
-
writePage(pageItems: T[], page: number): Promise<void> | void;
|
|
5
|
-
logger: GitHubScanLogger;
|
|
6
|
-
}
|
|
7
|
-
export interface PaginatedIngestResult {
|
|
8
|
-
pages: number;
|
|
9
|
-
items: number;
|
|
10
|
-
}
|
|
11
|
-
export declare function ingestPaginated<T>(options: PaginatedIngestOptions<T>): Promise<PaginatedIngestResult>;
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { paginatedIngestProgressIntervalPages } from "../../tuning/index.js";
|
|
2
|
-
const _DEFAULT_PAGE_SIZE = 100;
|
|
3
|
-
const _PROGRESS_LABEL = "GitHub package-version pages";
|
|
4
|
-
export async function ingestPaginated(options) {
|
|
5
|
-
let pages = 0;
|
|
6
|
-
let items = 0;
|
|
7
|
-
let lastLoggedPage = 0;
|
|
8
|
-
for (let page = 1;; page += 1) {
|
|
9
|
-
const pageItems = await options.loadPage(page);
|
|
10
|
-
if (pageItems.length === 0) {
|
|
11
|
-
break;
|
|
12
|
-
}
|
|
13
|
-
await options.writePage(pageItems, page);
|
|
14
|
-
pages = page;
|
|
15
|
-
items += pageItems.length;
|
|
16
|
-
if (page === 1 || page % paginatedIngestProgressIntervalPages === 0 || pageItems.length < _DEFAULT_PAGE_SIZE) {
|
|
17
|
-
options.logger.info(`Loaded ${_PROGRESS_LABEL} ${page} (${items} items total)`);
|
|
18
|
-
lastLoggedPage = page;
|
|
19
|
-
}
|
|
20
|
-
if (pageItems.length < _DEFAULT_PAGE_SIZE) {
|
|
21
|
-
break;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
if (pages > 0 && lastLoggedPage !== pages) {
|
|
25
|
-
options.logger.info(`Loaded ${_PROGRESS_LABEL} ${pages} (${items} items total)`);
|
|
26
|
-
}
|
|
27
|
-
return { pages, items };
|
|
28
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
DROP VIEW IF EXISTS v_missing_digests_related_manifests;
|
|
2
|
-
|
|
3
|
-
CREATE VIEW v_missing_digests_related_manifests AS
|
|
4
|
-
WITH
|
|
5
|
-
related_manifests AS (
|
|
6
|
-
SELECT DISTINCT
|
|
7
|
-
m.scan_id,
|
|
8
|
-
md.missing_digest,
|
|
9
|
-
m.digest AS related_manifest_digest,
|
|
10
|
-
m.media_type,
|
|
11
|
-
1 AS hops_missing_to_related_manifest
|
|
12
|
-
FROM v_missing_digests md
|
|
13
|
-
JOIN manifests m
|
|
14
|
-
ON m.scan_id = md.scan_id
|
|
15
|
-
AND m.digest = md.anchor_digest
|
|
16
|
-
|
|
17
|
-
UNION
|
|
18
|
-
|
|
19
|
-
SELECT DISTINCT
|
|
20
|
-
m.scan_id,
|
|
21
|
-
md.missing_digest,
|
|
22
|
-
m.digest AS related_manifest_digest,
|
|
23
|
-
m.media_type,
|
|
24
|
-
r.min_distance + 1 AS hops_missing_to_related_manifest
|
|
25
|
-
FROM v_missing_digests md
|
|
26
|
-
JOIN manifests m
|
|
27
|
-
ON m.scan_id = md.scan_id
|
|
28
|
-
JOIN manifest_reachability r
|
|
29
|
-
ON r.scan_id = m.scan_id
|
|
30
|
-
AND r.ancestor_digest = m.digest
|
|
31
|
-
AND r.descendant_digest = md.anchor_digest
|
|
32
|
-
|
|
33
|
-
UNION
|
|
34
|
-
|
|
35
|
-
SELECT DISTINCT
|
|
36
|
-
m.scan_id,
|
|
37
|
-
md.missing_digest,
|
|
38
|
-
m.digest AS related_manifest_digest,
|
|
39
|
-
m.media_type,
|
|
40
|
-
r.min_distance + 1 AS hops_missing_to_related_manifest
|
|
41
|
-
FROM v_missing_digests md
|
|
42
|
-
JOIN manifests m
|
|
43
|
-
ON m.scan_id = md.scan_id
|
|
44
|
-
JOIN manifest_reachability r
|
|
45
|
-
ON r.scan_id = m.scan_id
|
|
46
|
-
AND r.ancestor_digest = md.anchor_digest
|
|
47
|
-
AND r.descendant_digest = m.digest
|
|
48
|
-
),
|
|
49
|
-
closest_related_manifests AS (
|
|
50
|
-
SELECT
|
|
51
|
-
scan_id,
|
|
52
|
-
missing_digest,
|
|
53
|
-
related_manifest_digest,
|
|
54
|
-
media_type,
|
|
55
|
-
MIN(hops_missing_to_related_manifest) AS hops_missing_to_related_manifest
|
|
56
|
-
FROM related_manifests
|
|
57
|
-
GROUP BY
|
|
58
|
-
missing_digest,
|
|
59
|
-
scan_id,
|
|
60
|
-
related_manifest_digest,
|
|
61
|
-
media_type
|
|
62
|
-
)
|
|
63
|
-
SELECT
|
|
64
|
-
ps.scan_id,
|
|
65
|
-
ps.owner,
|
|
66
|
-
ps.package_name,
|
|
67
|
-
crm.missing_digest,
|
|
68
|
-
crm.related_manifest_digest,
|
|
69
|
-
crm.media_type,
|
|
70
|
-
crm.hops_missing_to_related_manifest,
|
|
71
|
-
t.tag,
|
|
72
|
-
t.version_id
|
|
73
|
-
FROM closest_related_manifests crm
|
|
74
|
-
JOIN package_scans ps
|
|
75
|
-
ON ps.scan_id = crm.scan_id
|
|
76
|
-
LEFT JOIN tags t
|
|
77
|
-
ON t.scan_id = crm.scan_id
|
|
78
|
-
AND t.digest = crm.related_manifest_digest;
|