ghcr-manager 0.0.4 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +50 -1
  2. package/README.md +166 -57
  3. package/dist/cleanup-summary/_cleanup-summary-markdown.d.ts +6 -0
  4. package/dist/cleanup-summary/_cleanup-summary-markdown.js +113 -0
  5. package/dist/cleanup-summary/_cleanup-summary.d.ts +40 -0
  6. package/dist/cleanup-summary/_cleanup-summary.js +40 -0
  7. package/dist/cleanup-summary/index.d.ts +2 -0
  8. package/dist/cleanup-summary/index.js +2 -0
  9. package/dist/cli/_args.d.ts +1 -0
  10. package/dist/cli/_args.js +3 -0
  11. package/dist/cli/_cleanup-command.d.ts +1 -0
  12. package/dist/cli/_cleanup-command.js +63 -0
  13. package/dist/cli/_db-merge-command.d.ts +1 -0
  14. package/dist/cli/_db-merge-command.js +41 -0
  15. package/dist/cli/_github-output.d.ts +10 -0
  16. package/dist/cli/_github-output.js +13 -0
  17. package/dist/cli/_logger.d.ts +2 -1
  18. package/dist/cli/_logger.js +2 -0
  19. package/dist/cli/_older-than.d.ts +5 -0
  20. package/dist/cli/_older-than.js +42 -0
  21. package/dist/cli/_planner-options.d.ts +20 -0
  22. package/dist/cli/_planner-options.js +101 -0
  23. package/dist/cli/_scan-command.js +11 -4
  24. package/dist/cli/_tag-selector-resolver.d.ts +3 -0
  25. package/dist/cli/_tag-selector-resolver.js +109 -0
  26. package/dist/cli/_untag-command.d.ts +1 -0
  27. package/dist/cli/_untag-command.js +57 -0
  28. package/dist/cli/index.js +19 -1
  29. package/dist/config/_service-constants.d.ts +3 -0
  30. package/dist/config/_service-constants.js +3 -0
  31. package/dist/{tuning → config}/index.d.ts +3 -0
  32. package/dist/{tuning → config}/index.js +3 -0
  33. package/dist/core/_github-package-owner.d.ts +11 -0
  34. package/dist/core/_github-package-owner.js +45 -0
  35. package/dist/core/_http-error.d.ts +6 -0
  36. package/dist/core/_http-error.js +33 -0
  37. package/dist/core/_types.d.ts +3 -2
  38. package/dist/core/index.d.ts +4 -1
  39. package/dist/core/index.js +2 -1
  40. package/dist/db/_cleanup-run-writer.d.ts +10 -0
  41. package/dist/db/_cleanup-run-writer.js +73 -0
  42. package/dist/db/_db-merge-cleanup-copy.d.ts +7 -0
  43. package/dist/db/_db-merge-cleanup-copy.js +122 -0
  44. package/dist/db/_db-merge-history.d.ts +2 -0
  45. package/dist/db/_db-merge-history.js +15 -0
  46. package/dist/db/_db-merge-repository.d.ts +8 -0
  47. package/dist/db/_db-merge-repository.js +95 -0
  48. package/dist/db/_db-merge-scan-copy.d.ts +10 -0
  49. package/dist/db/_db-merge-scan-copy.js +69 -0
  50. package/dist/db/_db-merge-types.d.ts +44 -0
  51. package/dist/db/_db-merge-types.js +1 -0
  52. package/dist/db/_github-actions-run-url.d.ts +1 -0
  53. package/dist/db/_github-actions-run-url.js +9 -0
  54. package/dist/db/_scan-writer.d.ts +3 -1
  55. package/dist/db/_scan-writer.js +28 -13
  56. package/dist/db/_snapshot-repository.d.ts +9 -9
  57. package/dist/db/_snapshot-repository.js +37 -49
  58. package/dist/db/_sql-placeholders.d.ts +2 -0
  59. package/dist/db/_sql-placeholders.js +16 -0
  60. package/dist/db/index.d.ts +5 -0
  61. package/dist/db/index.js +3 -0
  62. package/dist/db/planner/_planner-delete-tag-root-targets.d.ts +7 -0
  63. package/dist/db/planner/_planner-delete-tag-root-targets.js +130 -0
  64. package/dist/db/planner/_planner-direct-target-tags.d.ts +6 -0
  65. package/dist/db/planner/_planner-direct-target-tags.js +47 -0
  66. package/dist/db/planner/_planner-keep-tagged-root-targets.d.ts +7 -0
  67. package/dist/db/planner/_planner-keep-tagged-root-targets.js +74 -0
  68. package/dist/db/planner/_planner-output.d.ts +5 -0
  69. package/dist/db/planner/_planner-output.js +101 -0
  70. package/dist/db/planner/_planner-plan-artifacts.d.ts +7 -0
  71. package/dist/db/planner/_planner-plan-artifacts.js +211 -0
  72. package/dist/db/planner/_planner-repository.d.ts +34 -0
  73. package/dist/db/planner/_planner-repository.js +126 -0
  74. package/dist/db/planner/_planner-sql.d.ts +12 -0
  75. package/dist/db/planner/_planner-sql.js +35 -0
  76. package/dist/db/planner/_planner-tag-selectors.d.ts +8 -0
  77. package/dist/db/planner/_planner-tag-selectors.js +57 -0
  78. package/dist/db/planner/_planner-tagged-root-targets.d.ts +15 -0
  79. package/dist/db/planner/_planner-tagged-root-targets.js +19 -0
  80. package/dist/db/planner/_planner-tagged-targets.d.ts +8 -0
  81. package/dist/db/planner/_planner-tagged-targets.js +16 -0
  82. package/dist/db/planner/_planner-types.d.ts +135 -0
  83. package/dist/db/planner/_planner-types.js +38 -0
  84. package/dist/db/planner/_planner-untagged-targets.d.ts +9 -0
  85. package/dist/db/planner/_planner-untagged-targets.js +91 -0
  86. package/dist/db/planner/index.d.ts +2 -0
  87. package/dist/db/planner/index.js +1 -0
  88. package/dist/execute/_http.d.ts +7 -0
  89. package/dist/execute/_http.js +48 -0
  90. package/dist/execute/_manifest-detach.d.ts +4 -0
  91. package/dist/execute/_manifest-detach.js +31 -0
  92. package/dist/execute/_package-version-delete-client.d.ts +4 -0
  93. package/dist/execute/_package-version-delete-client.js +34 -0
  94. package/dist/execute/_package-version-page-client.d.ts +14 -0
  95. package/dist/execute/_package-version-page-client.js +64 -0
  96. package/dist/execute/_package-version-tag-source-client.d.ts +12 -0
  97. package/dist/execute/_package-version-tag-source-client.js +65 -0
  98. package/dist/execute/_plan-executor.d.ts +3 -0
  99. package/dist/execute/_plan-executor.js +47 -0
  100. package/dist/execute/_registry-manifest-client.d.ts +12 -0
  101. package/dist/execute/_registry-manifest-client.js +79 -0
  102. package/dist/execute/_registry-token-client.d.ts +4 -0
  103. package/dist/execute/_registry-token-client.js +37 -0
  104. package/dist/execute/_types.d.ts +51 -0
  105. package/dist/execute/_types.js +1 -0
  106. package/dist/execute/_untag-client.d.ts +2 -0
  107. package/dist/execute/_untag-client.js +71 -0
  108. package/dist/execute/index.d.ts +5 -0
  109. package/dist/execute/index.js +3 -0
  110. package/dist/ingest/github/_manifest-client.d.ts +7 -1
  111. package/dist/ingest/github/_manifest-client.js +8 -0
  112. package/dist/ingest/github/_manifest-ingest.js +39 -53
  113. package/dist/ingest/github/_manifest-kind.d.ts +20 -0
  114. package/dist/ingest/github/_manifest-kind.js +50 -0
  115. package/dist/ingest/github/_package-metadata-load.d.ts +5 -0
  116. package/dist/ingest/github/_package-metadata-load.js +45 -0
  117. package/dist/ingest/github/_package-version-page-load.d.ts +1 -1
  118. package/dist/ingest/github/_package-version-page-load.js +8 -5
  119. package/dist/ingest/github/_packages-client.d.ts +1 -1
  120. package/dist/ingest/github/_packages-client.js +21 -4
  121. package/dist/ingest/github/_parallel-paginated-ingest.d.ts +1 -0
  122. package/dist/ingest/github/_parallel-paginated-ingest.js +2 -2
  123. package/dist/ingest/github/_shared.d.ts +1 -1
  124. package/dist/ingest/github/_shared.js +2 -34
  125. package/dist/ingest/github/index.d.ts +4 -0
  126. package/dist/ingest/github/index.js +8 -5
  127. package/package.json +7 -5
  128. package/resources/sql/schema/001_schema.sql +82 -8
  129. package/resources/sql/views/001_v_latest_scan_per_package.sql +2 -2
  130. package/resources/sql/views/003_v_scan_root_manifests.sql +43 -0
  131. package/resources/sql/views/004_v_digest_derived_tag_relations.sql +51 -0
  132. package/resources/sql/views/005_v_cleanup_root_closure_members.sql +100 -0
  133. package/resources/sql/views/006_v_cleanup_blocking_overlaps.sql +42 -0
  134. package/dist/ingest/github/_paginated-ingest.d.ts +0 -11
  135. package/dist/ingest/github/_paginated-ingest.js +0 -28
  136. package/resources/sql/views/003_v_missing_digests_related_manifests.sql +0 -78
  137. package/resources/sql/views/004_v_manifests_related_manifests.sql +0 -142
@@ -1,20 +1,23 @@
1
+ import { ghcrRegistryBaseUrl } from "../../config/index.js";
1
2
  import { ingestManifests } from "./_manifest-ingest.js";
3
+ import { loadPackageMetadata } from "./_package-metadata-load.js";
2
4
  import { ingestPackageVersions } from "./_packages-client.js";
3
5
  import { defaultFetch } from "./_shared.js";
4
- const _GITHUB_API_BASE_URL = "https://api.github.com";
5
- const _REGISTRY_BASE_URL = "https://ghcr.io";
6
+ export { loadPackageMetadata } from "./_package-metadata-load.js";
7
+ export { defaultFetch } from "./_shared.js";
6
8
  export async function importGitHubScan(options, writer, repository, runtime) {
7
9
  const fetchImpl = runtime?.fetchImpl ?? defaultFetch;
8
10
  const scanStartedAt = new Date().toISOString();
9
11
  const fullPackageName = `${options.owner}/${options.packageName}`;
10
- writer.resetScan(options.owner, options.packageName, scanStartedAt);
12
+ const packageMetadata = runtime?.packageMetadata ?? (await loadPackageMetadata(fetchImpl, options));
13
+ writer.startScan(options.owner, options.packageName, scanStartedAt, packageMetadata);
11
14
  const scanId = writer.getActiveScanId();
12
15
  options.logger.info(`Starting GitHub package scan for ${fullPackageName}`);
13
16
  try {
14
17
  options.logger.info(`Starting remote data pull for ${fullPackageName}`);
15
- const counts = await ingestPackageVersions(fetchImpl, _GITHUB_API_BASE_URL, options, writer);
18
+ const counts = await ingestPackageVersions(fetchImpl, options, writer);
16
19
  options.logger.info(`Loaded ${counts.packageVersions} package versions and ${counts.tags} tags`);
17
- await ingestManifests(fetchImpl, _REGISTRY_BASE_URL, options, writer, repository, scanId);
20
+ await ingestManifests(fetchImpl, ghcrRegistryBaseUrl, options, writer, repository, scanId);
18
21
  options.logger.info(`Completed remote data pull for ${fullPackageName}`);
19
22
  writer.markScanCompleted(new Date().toISOString());
20
23
  options.logger.info(`Completed GitHub package scan for ${fullPackageName}`);
package/package.json CHANGED
@@ -9,7 +9,7 @@
9
9
  "type": "git",
10
10
  "url": "https://github.com/gh-workflow/ghcr-manager"
11
11
  },
12
- "version": "0.0.4",
12
+ "version": "0.9.0",
13
13
  "type": "module",
14
14
  "engines": {
15
15
  "node": ">=20.0.0"
@@ -28,11 +28,12 @@
28
28
  "build": "tsc --project tsconfig.json",
29
29
  "coverage": "c8 --src src --reporter=text --reporter=text-summary node --import tsx --test",
30
30
  "ghcr-manager": "tsx src/cli/index.ts",
31
+ "ghcr-manager:dist": "node dist/cli/index.js",
31
32
  "format": "prettier --write .",
32
33
  "format:check": "prettier --check .",
33
34
  "lint": "npm run check:file-names && npm run check:test-mapping && npm run typecheck && npm run lint:ts && npm run lint:yaml && npm run lint:markdown && npm run format:check",
34
- "check:file-names": "node tools/check-src-file-names.mjs",
35
- "check:test-mapping": "node tools/check-test-mapping.mjs",
35
+ "check:file-names": "node tools/tests/check-src-file-names.mjs",
36
+ "check:test-mapping": "node tools/tests/check-test-mapping.mjs",
36
37
  "lint:ts": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
37
38
  "lint:yaml": "eslint \".github/**/*.yml\" \"*.yml\"",
38
39
  "lint:markdown": "markdownlint-cli2",
@@ -51,9 +52,10 @@
51
52
  "eslint-plugin-yml": "^3.3.2",
52
53
  "globals": "^17.5.0",
53
54
  "markdownlint-cli2": "^0.22.1",
54
- "prettier": "^3.6.2",
55
+ "prettier": "^3.8.3",
55
56
  "tsx": "^4.20.6",
57
+ "typescript": "^6.0.3",
56
58
  "typescript-eslint": "^8.46.2",
57
- "typescript": "^6.0.3"
59
+ "yaml": "^2.9.0"
58
60
  }
59
61
  }
@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS package_scans (
5
5
  scan_uuid TEXT NOT NULL UNIQUE,
6
6
  owner TEXT NOT NULL,
7
7
  package_name TEXT NOT NULL,
8
+ package_metadata_json TEXT NOT NULL,
9
+ github_actions_run_url TEXT,
8
10
  scan_started_at TEXT NOT NULL,
9
11
  scan_completed_at TEXT,
10
12
  status TEXT NOT NULL,
@@ -14,11 +16,9 @@ CREATE TABLE IF NOT EXISTS package_scans (
14
16
  CREATE TABLE IF NOT EXISTS package_versions (
15
17
  scan_id INTEGER NOT NULL,
16
18
  version_id INTEGER NOT NULL,
17
- digest TEXT NOT NULL,
18
19
  created_at TEXT NOT NULL,
19
20
  updated_at TEXT NOT NULL,
20
21
  PRIMARY KEY(scan_id, version_id),
21
- UNIQUE(scan_id, version_id, digest),
22
22
  FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
23
23
  );
24
24
 
@@ -33,14 +33,14 @@ CREATE TABLE IF NOT EXISTS package_version_payloads (
33
33
  CREATE TABLE IF NOT EXISTS tags (
34
34
  scan_id INTEGER NOT NULL,
35
35
  tag TEXT NOT NULL,
36
- digest TEXT NOT NULL,
37
36
  version_id INTEGER NOT NULL,
38
37
  PRIMARY KEY(scan_id, tag),
39
- FOREIGN KEY(scan_id, version_id, digest) REFERENCES package_versions(scan_id, version_id, digest)
38
+ FOREIGN KEY(scan_id, version_id) REFERENCES package_versions(scan_id, version_id)
40
39
  );
41
40
 
42
41
  CREATE TABLE IF NOT EXISTS manifests (
43
42
  scan_id INTEGER NOT NULL,
43
+ version_id INTEGER NOT NULL,
44
44
  digest TEXT NOT NULL,
45
45
  media_type TEXT NOT NULL,
46
46
  artifact_type TEXT,
@@ -50,8 +50,17 @@ CREATE TABLE IF NOT EXISTS manifests (
50
50
  platform_os TEXT,
51
51
  platform_architecture TEXT,
52
52
  platform_variant TEXT,
53
- PRIMARY KEY(scan_id, digest),
54
- FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
53
+ manifest_kind TEXT,
54
+ CHECK(manifest_kind IN (
55
+ 'image_index',
56
+ 'image_manifest',
57
+ 'artifact_manifest',
58
+ 'attestation_manifest',
59
+ 'signature_manifest'
60
+ )),
61
+ PRIMARY KEY(scan_id, version_id),
62
+ UNIQUE(scan_id, digest),
63
+ FOREIGN KEY(scan_id, version_id) REFERENCES package_versions(scan_id, version_id)
55
64
  );
56
65
 
57
66
  CREATE TABLE IF NOT EXISTS manifest_descriptors (
@@ -96,14 +105,79 @@ CREATE TABLE IF NOT EXISTS manifest_reachability (
96
105
  CHECK(min_distance >= 0)
97
106
  );
98
107
 
108
+ CREATE TABLE IF NOT EXISTS cleanup_runs (
109
+ cleanup_run_id INTEGER PRIMARY KEY,
110
+ scan_id INTEGER NOT NULL,
111
+ cleanup_uuid TEXT NOT NULL,
112
+ cleanup_started_at TEXT NOT NULL,
113
+ github_actions_run_url TEXT,
114
+ dry_run INTEGER NOT NULL,
115
+ planner_inputs_json TEXT NOT NULL,
116
+ direct_target_tag_count INTEGER NOT NULL,
117
+ direct_target_root_count INTEGER NOT NULL,
118
+ delete_root_candidate_count INTEGER NOT NULL,
119
+ untag_only_root_count INTEGER NOT NULL,
120
+ fully_deletable_root_count INTEGER NOT NULL,
121
+ blocked_delete_root_count INTEGER NOT NULL,
122
+ protected_root_count INTEGER NOT NULL,
123
+ CHECK(dry_run IN (0, 1)),
124
+ UNIQUE(cleanup_run_id, scan_id),
125
+ FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
126
+ );
127
+
128
+ CREATE TABLE IF NOT EXISTS cleanup_root_decisions (
129
+ cleanup_run_id INTEGER NOT NULL,
130
+ scan_id INTEGER NOT NULL,
131
+ digest TEXT NOT NULL,
132
+ selection_mode TEXT NOT NULL,
133
+ selection_reason TEXT NOT NULL,
134
+ validation_status TEXT NOT NULL,
135
+ validation_reason_code TEXT NOT NULL,
136
+ validation_reason TEXT NOT NULL,
137
+ blocking_digest TEXT,
138
+ overlap_digest TEXT,
139
+ PRIMARY KEY(cleanup_run_id, digest),
140
+ CHECK(validation_status IN ('fully-deletable', 'blocked', 'untag-only')),
141
+ CHECK(validation_reason_code IN (
142
+ 'untag-only-partial-tag-match',
143
+ 'fully-deletable-no-retained-overlap',
144
+ 'blocked-overlap-with-retained-root'
145
+ )),
146
+ FOREIGN KEY(cleanup_run_id, scan_id) REFERENCES cleanup_runs(cleanup_run_id, scan_id),
147
+ FOREIGN KEY(scan_id, digest) REFERENCES manifests(scan_id, digest),
148
+ FOREIGN KEY(scan_id, blocking_digest) REFERENCES manifests(scan_id, digest),
149
+ FOREIGN KEY(scan_id, overlap_digest) REFERENCES manifests(scan_id, digest)
150
+ );
151
+
152
+ CREATE TABLE IF NOT EXISTS cleanup_protected_root_blocks (
153
+ cleanup_run_id INTEGER NOT NULL,
154
+ scan_id INTEGER NOT NULL,
155
+ protected_digest TEXT NOT NULL,
156
+ blocked_digest TEXT NOT NULL,
157
+ block_reason_code TEXT NOT NULL,
158
+ overlap_digest TEXT NOT NULL,
159
+ PRIMARY KEY(cleanup_run_id, protected_digest, blocked_digest, overlap_digest),
160
+ CHECK(block_reason_code IN ('overlap-with-retained-root')),
161
+ FOREIGN KEY(cleanup_run_id, scan_id) REFERENCES cleanup_runs(cleanup_run_id, scan_id),
162
+ FOREIGN KEY(scan_id, protected_digest, overlap_digest)
163
+ REFERENCES manifest_reachability(scan_id, ancestor_digest, descendant_digest),
164
+ FOREIGN KEY(scan_id, blocked_digest, overlap_digest)
165
+ REFERENCES manifest_reachability(scan_id, ancestor_digest, descendant_digest)
166
+ );
167
+
99
168
  CREATE INDEX IF NOT EXISTS idx_package_versions_scan_created_at ON package_versions(scan_id, created_at);
100
- CREATE INDEX IF NOT EXISTS idx_package_versions_scan_digest ON package_versions(scan_id, digest);
101
169
  CREATE UNIQUE INDEX IF NOT EXISTS idx_package_scans_scan_uuid ON package_scans(scan_uuid);
170
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_cleanup_runs_cleanup_uuid ON cleanup_runs(cleanup_uuid);
102
171
  CREATE INDEX IF NOT EXISTS idx_package_scans_owner_name_started_at
103
172
  ON package_scans(owner, package_name, scan_started_at DESC);
104
- CREATE INDEX IF NOT EXISTS idx_tags_scan_digest ON tags(scan_id, digest);
173
+ CREATE INDEX IF NOT EXISTS idx_cleanup_runs_scan_id ON cleanup_runs(scan_id);
174
+ CREATE INDEX IF NOT EXISTS idx_cleanup_protected_root_blocks_run_blocked
175
+ ON cleanup_protected_root_blocks(cleanup_run_id, blocked_digest);
176
+ CREATE INDEX IF NOT EXISTS idx_tags_scan_version ON tags(scan_id, version_id);
105
177
  CREATE INDEX IF NOT EXISTS idx_manifest_descriptors_scan_child ON manifest_descriptors(scan_id, child_digest);
106
178
  CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_parent ON manifest_edges(scan_id, parent_digest);
107
179
  CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_child ON manifest_edges(scan_id, child_digest);
108
180
  CREATE INDEX IF NOT EXISTS idx_manifest_reachability_scan_descendant
109
181
  ON manifest_reachability(scan_id, descendant_digest);
182
+ CREATE INDEX IF NOT EXISTS idx_manifest_reachability_scan_descendant_distance
183
+ ON manifest_reachability(scan_id, descendant_digest, min_distance);
@@ -17,11 +17,11 @@ FROM (
17
17
  ps.scan_completed_at,
18
18
  ROW_NUMBER() OVER (
19
19
  PARTITION BY ps.owner, ps.package_name
20
- ORDER BY ps.scan_completed_at DESC
20
+ ORDER BY ps.scan_completed_at DESC, ps.scan_id DESC
21
21
  ) AS rn
22
22
  FROM package_scans ps
23
23
  WHERE ps.scan_completed_at IS NOT NULL
24
24
  AND ps.status = 'completed'
25
25
  )
26
26
  WHERE rn = 1
27
- ;
27
+ ;
@@ -0,0 +1,43 @@
1
+ DROP VIEW IF EXISTS v_scan_root_manifests;
2
+
3
+ CREATE VIEW v_scan_root_manifests AS
4
+ SELECT
5
+ ps.scan_id,
6
+ ps.owner,
7
+ ps.package_name,
8
+ m.version_id AS root_version_id,
9
+ m.digest AS root_digest,
10
+ m.manifest_kind AS root_manifest_kind,
11
+ pv.created_at,
12
+ pv.updated_at,
13
+ COUNT(t.tag) AS tag_count,
14
+ CASE WHEN COUNT(t.tag) > 0 THEN 1 ELSE 0 END AS is_tagged,
15
+ CASE
16
+ WHEN EXISTS (
17
+ SELECT 1
18
+ FROM manifest_reachability mr
19
+ WHERE mr.scan_id = m.scan_id
20
+ AND mr.descendant_digest = m.digest
21
+ AND mr.min_distance > 0
22
+ ) THEN 1
23
+ ELSE 0
24
+ END AS has_ancestor
25
+ FROM manifests m
26
+ JOIN package_versions pv
27
+ ON pv.scan_id = m.scan_id
28
+ AND pv.version_id = m.version_id
29
+ JOIN package_scans ps
30
+ ON ps.scan_id = m.scan_id
31
+ LEFT JOIN tags t
32
+ ON t.scan_id = m.scan_id
33
+ AND t.version_id = m.version_id
34
+ GROUP BY
35
+ ps.scan_id,
36
+ ps.owner,
37
+ ps.package_name,
38
+ m.version_id,
39
+ m.digest,
40
+ m.manifest_kind,
41
+ pv.created_at,
42
+ pv.updated_at
43
+ ;
@@ -0,0 +1,51 @@
1
+ DROP VIEW IF EXISTS v_digest_derived_tag_relations;
2
+
3
+ CREATE VIEW v_digest_derived_tag_relations AS
4
+ WITH digest_like_tags AS (
5
+ SELECT
6
+ lsp.scan_id,
7
+ lsp.owner,
8
+ lsp.package_name,
9
+ t.tag,
10
+ t.version_id AS artifact_version_id,
11
+ m.digest AS artifact_digest,
12
+ m.manifest_kind AS artifact_manifest_kind,
13
+ m.subject_digest AS artifact_subject_digest,
14
+ 'sha256:' || SUBSTR(t.tag, 8, 64) AS inferred_parent_digest,
15
+ SUBSTR(t.tag, 72) AS tag_suffix
16
+ FROM tags t
17
+ JOIN v_latest_scan_per_package lsp
18
+ ON lsp.scan_id = t.scan_id
19
+ JOIN manifests m
20
+ ON m.scan_id = t.scan_id
21
+ AND m.version_id = t.version_id
22
+ WHERE t.tag LIKE 'sha256-%'
23
+ AND LENGTH(t.tag) >= 71
24
+ AND SUBSTR(t.tag, 8, 64) NOT GLOB '*[^0-9A-Fa-f]*'
25
+ )
26
+ SELECT
27
+ dlt.scan_id,
28
+ dlt.owner,
29
+ dlt.package_name,
30
+ dlt.tag,
31
+ dlt.artifact_version_id,
32
+ dlt.artifact_digest,
33
+ dlt.artifact_manifest_kind,
34
+ dlt.artifact_subject_digest,
35
+ dlt.inferred_parent_digest,
36
+ dlt.tag_suffix,
37
+ pm.version_id AS parent_version_id,
38
+ pm.digest AS parent_digest,
39
+ pm.manifest_kind AS parent_manifest_kind,
40
+ CASE
41
+ WHEN dlt.artifact_subject_digest = dlt.inferred_parent_digest THEN 1
42
+ ELSE 0
43
+ END AS subject_matches_inferred_parent,
44
+ CASE
45
+ WHEN pm.digest IS NULL THEN 0
46
+ ELSE 1
47
+ END AS parent_exists
48
+ FROM digest_like_tags dlt
49
+ LEFT JOIN manifests pm
50
+ ON pm.scan_id = dlt.scan_id
51
+ AND pm.digest = dlt.inferred_parent_digest;
@@ -0,0 +1,100 @@
1
+ DROP VIEW IF EXISTS v_cleanup_root_closure_members;
2
+
3
+ CREATE VIEW v_cleanup_root_closure_members AS
4
+ WITH selected_roots AS (
5
+ SELECT
6
+ run.cleanup_run_id,
7
+ run.scan_id,
8
+ ps.owner,
9
+ ps.package_name,
10
+ decision.digest AS root_digest,
11
+ root_manifest.version_id AS root_version_id,
12
+ root_manifest.manifest_kind AS root_manifest_kind,
13
+ decision.selection_mode,
14
+ decision.selection_reason,
15
+ decision.validation_status,
16
+ decision.validation_reason_code,
17
+ decision.validation_reason
18
+ FROM cleanup_root_decisions decision
19
+ JOIN cleanup_runs run
20
+ ON run.cleanup_run_id = decision.cleanup_run_id
21
+ AND run.scan_id = decision.scan_id
22
+ JOIN package_scans ps
23
+ ON ps.scan_id = run.scan_id
24
+ JOIN manifests root_manifest
25
+ ON root_manifest.scan_id = decision.scan_id
26
+ AND root_manifest.digest = decision.digest
27
+ ),
28
+ closure_members AS (
29
+ SELECT
30
+ sr.cleanup_run_id,
31
+ sr.scan_id,
32
+ sr.owner,
33
+ sr.package_name,
34
+ sr.root_digest,
35
+ sr.root_version_id,
36
+ sr.root_manifest_kind,
37
+ sr.selection_mode,
38
+ sr.selection_reason,
39
+ sr.validation_status,
40
+ sr.validation_reason_code,
41
+ sr.validation_reason,
42
+ sr.root_digest AS member_digest,
43
+ sr.root_version_id AS member_version_id,
44
+ sr.root_manifest_kind AS member_manifest_kind,
45
+ 0 AS hops_from_root,
46
+ 'root' AS member_role
47
+ FROM selected_roots sr
48
+
49
+ UNION ALL
50
+
51
+ SELECT
52
+ sr.cleanup_run_id,
53
+ sr.scan_id,
54
+ sr.owner,
55
+ sr.package_name,
56
+ sr.root_digest,
57
+ sr.root_version_id,
58
+ sr.root_manifest_kind,
59
+ sr.selection_mode,
60
+ sr.selection_reason,
61
+ sr.validation_status,
62
+ sr.validation_reason_code,
63
+ sr.validation_reason,
64
+ member_manifest.digest AS member_digest,
65
+ member_manifest.version_id AS member_version_id,
66
+ member_manifest.manifest_kind AS member_manifest_kind,
67
+ reachability.min_distance AS hops_from_root,
68
+ 'descendant' AS member_role
69
+ FROM selected_roots sr
70
+ JOIN manifest_reachability reachability
71
+ ON reachability.scan_id = sr.scan_id
72
+ AND reachability.ancestor_digest = sr.root_digest
73
+ AND reachability.min_distance > 0
74
+ JOIN manifests member_manifest
75
+ ON member_manifest.scan_id = sr.scan_id
76
+ AND member_manifest.digest = reachability.descendant_digest
77
+ )
78
+ SELECT
79
+ cm.cleanup_run_id,
80
+ cm.scan_id,
81
+ cm.owner,
82
+ cm.package_name,
83
+ cm.root_digest,
84
+ cm.root_version_id,
85
+ cm.root_manifest_kind,
86
+ cm.selection_mode,
87
+ cm.selection_reason,
88
+ cm.validation_status,
89
+ cm.validation_reason_code,
90
+ cm.validation_reason,
91
+ cm.member_digest,
92
+ cm.member_version_id,
93
+ cm.member_manifest_kind,
94
+ cm.hops_from_root,
95
+ cm.member_role,
96
+ tag.tag AS member_tag
97
+ FROM closure_members cm
98
+ LEFT JOIN tags tag
99
+ ON tag.scan_id = cm.scan_id
100
+ AND tag.version_id = cm.member_version_id;
@@ -0,0 +1,42 @@
1
+ DROP VIEW IF EXISTS v_cleanup_blocking_overlaps;
2
+
3
+ CREATE VIEW v_cleanup_blocking_overlaps AS
4
+ SELECT
5
+ run.cleanup_run_id,
6
+ run.scan_id,
7
+ ps.owner,
8
+ ps.package_name,
9
+ block.protected_digest,
10
+ protected_manifest.version_id AS protected_version_id,
11
+ protected_manifest.manifest_kind AS protected_manifest_kind,
12
+ block.blocked_digest,
13
+ blocked_manifest.version_id AS blocked_version_id,
14
+ blocked_manifest.manifest_kind AS blocked_manifest_kind,
15
+ decision.selection_mode AS blocked_selection_mode,
16
+ decision.selection_reason AS blocked_selection_reason,
17
+ decision.validation_status AS blocked_validation_status,
18
+ decision.validation_reason_code AS blocked_validation_reason_code,
19
+ decision.validation_reason AS blocked_validation_reason,
20
+ block.block_reason_code,
21
+ block.overlap_digest,
22
+ overlap_manifest.version_id AS overlap_version_id,
23
+ overlap_manifest.manifest_kind AS overlap_manifest_kind
24
+ FROM cleanup_protected_root_blocks block
25
+ JOIN cleanup_runs run
26
+ ON run.cleanup_run_id = block.cleanup_run_id
27
+ AND run.scan_id = block.scan_id
28
+ JOIN package_scans ps
29
+ ON ps.scan_id = run.scan_id
30
+ JOIN manifests protected_manifest
31
+ ON protected_manifest.scan_id = block.scan_id
32
+ AND protected_manifest.digest = block.protected_digest
33
+ JOIN manifests blocked_manifest
34
+ ON blocked_manifest.scan_id = block.scan_id
35
+ AND blocked_manifest.digest = block.blocked_digest
36
+ JOIN cleanup_root_decisions decision
37
+ ON decision.cleanup_run_id = block.cleanup_run_id
38
+ AND decision.scan_id = block.scan_id
39
+ AND decision.digest = block.blocked_digest
40
+ LEFT JOIN manifests overlap_manifest
41
+ ON overlap_manifest.scan_id = block.scan_id
42
+ AND overlap_manifest.digest = block.overlap_digest;
@@ -1,11 +0,0 @@
1
- import type { GitHubScanLogger } from "./_shared.js";
2
- export interface PaginatedIngestOptions<T> {
3
- loadPage(page: number): Promise<T[]>;
4
- writePage(pageItems: T[], page: number): Promise<void> | void;
5
- logger: GitHubScanLogger;
6
- }
7
- export interface PaginatedIngestResult {
8
- pages: number;
9
- items: number;
10
- }
11
- export declare function ingestPaginated<T>(options: PaginatedIngestOptions<T>): Promise<PaginatedIngestResult>;
@@ -1,28 +0,0 @@
1
- import { paginatedIngestProgressIntervalPages } from "../../tuning/index.js";
2
- const _DEFAULT_PAGE_SIZE = 100;
3
- const _PROGRESS_LABEL = "GitHub package-version pages";
4
- export async function ingestPaginated(options) {
5
- let pages = 0;
6
- let items = 0;
7
- let lastLoggedPage = 0;
8
- for (let page = 1;; page += 1) {
9
- const pageItems = await options.loadPage(page);
10
- if (pageItems.length === 0) {
11
- break;
12
- }
13
- await options.writePage(pageItems, page);
14
- pages = page;
15
- items += pageItems.length;
16
- if (page === 1 || page % paginatedIngestProgressIntervalPages === 0 || pageItems.length < _DEFAULT_PAGE_SIZE) {
17
- options.logger.info(`Loaded ${_PROGRESS_LABEL} ${page} (${items} items total)`);
18
- lastLoggedPage = page;
19
- }
20
- if (pageItems.length < _DEFAULT_PAGE_SIZE) {
21
- break;
22
- }
23
- }
24
- if (pages > 0 && lastLoggedPage !== pages) {
25
- options.logger.info(`Loaded ${_PROGRESS_LABEL} ${pages} (${items} items total)`);
26
- }
27
- return { pages, items };
28
- }
@@ -1,78 +0,0 @@
1
- DROP VIEW IF EXISTS v_missing_digests_related_manifests;
2
-
3
- CREATE VIEW v_missing_digests_related_manifests AS
4
- WITH
5
- related_manifests AS (
6
- SELECT DISTINCT
7
- m.scan_id,
8
- md.missing_digest,
9
- m.digest AS related_manifest_digest,
10
- m.media_type,
11
- 1 AS hops_missing_to_related_manifest
12
- FROM v_missing_digests md
13
- JOIN manifests m
14
- ON m.scan_id = md.scan_id
15
- AND m.digest = md.anchor_digest
16
-
17
- UNION
18
-
19
- SELECT DISTINCT
20
- m.scan_id,
21
- md.missing_digest,
22
- m.digest AS related_manifest_digest,
23
- m.media_type,
24
- r.min_distance + 1 AS hops_missing_to_related_manifest
25
- FROM v_missing_digests md
26
- JOIN manifests m
27
- ON m.scan_id = md.scan_id
28
- JOIN manifest_reachability r
29
- ON r.scan_id = m.scan_id
30
- AND r.ancestor_digest = m.digest
31
- AND r.descendant_digest = md.anchor_digest
32
-
33
- UNION
34
-
35
- SELECT DISTINCT
36
- m.scan_id,
37
- md.missing_digest,
38
- m.digest AS related_manifest_digest,
39
- m.media_type,
40
- r.min_distance + 1 AS hops_missing_to_related_manifest
41
- FROM v_missing_digests md
42
- JOIN manifests m
43
- ON m.scan_id = md.scan_id
44
- JOIN manifest_reachability r
45
- ON r.scan_id = m.scan_id
46
- AND r.ancestor_digest = md.anchor_digest
47
- AND r.descendant_digest = m.digest
48
- ),
49
- closest_related_manifests AS (
50
- SELECT
51
- scan_id,
52
- missing_digest,
53
- related_manifest_digest,
54
- media_type,
55
- MIN(hops_missing_to_related_manifest) AS hops_missing_to_related_manifest
56
- FROM related_manifests
57
- GROUP BY
58
- missing_digest,
59
- scan_id,
60
- related_manifest_digest,
61
- media_type
62
- )
63
- SELECT
64
- ps.scan_id,
65
- ps.owner,
66
- ps.package_name,
67
- crm.missing_digest,
68
- crm.related_manifest_digest,
69
- crm.media_type,
70
- crm.hops_missing_to_related_manifest,
71
- t.tag,
72
- t.version_id
73
- FROM closest_related_manifests crm
74
- JOIN package_scans ps
75
- ON ps.scan_id = crm.scan_id
76
- LEFT JOIN tags t
77
- ON t.scan_id = crm.scan_id
78
- AND t.digest = crm.related_manifest_digest;