@caddis/cli 0.4.22 → 0.4.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundles/antigravity-plugin/plugin.json +1 -1
- package/bundles/antigravity-plugin/skills/_registry.md +1 -1
- package/bundles/antigravity-plugin/skills/port-to-github/SKILL.md +64 -3
- package/bundles/antigravity-plugin/skills/port-to-github/scripts/Get-PortSource.ps1 +30 -2
- package/bundles/antigravity-plugin/skills/port-to-github/scripts/Invoke-PortHistoryClean.ps1 +123 -0
- package/bundles/antigravity-plugin/skills/port-to-github/scripts/New-PortRuntimeManifest.ps1 +258 -0
- package/bundles/antigravity-plugin/skills/port-to-github/scripts/Publish-PortToGitHub.ps1 +20 -1
- package/bundles/antigravity-plugin/skills/port-to-github/scripts/Test-PortContentPolicy.ps1 +218 -0
- package/bundles/codex/.codex/skills/devops/port-to-github/SKILL.md +64 -3
- package/bundles/codex/.codex/skills/devops/port-to-github/scripts/Get-PortSource.ps1 +30 -2
- package/bundles/codex/.codex/skills/devops/port-to-github/scripts/Invoke-PortHistoryClean.ps1 +123 -0
- package/bundles/codex/.codex/skills/devops/port-to-github/scripts/New-PortRuntimeManifest.ps1 +258 -0
- package/bundles/codex/.codex/skills/devops/port-to-github/scripts/Publish-PortToGitHub.ps1 +20 -1
- package/bundles/codex/.codex/skills/devops/port-to-github/scripts/Test-PortContentPolicy.ps1 +218 -0
- package/bundles/manifest.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Check a staged repository against the source-only content policy, over the tips AND all history.
|
|
4
|
+
|
|
5
|
+
.DESCRIPTION
|
|
6
|
+
The policy: a repository holds application code, configuration and templates, migration
|
|
7
|
+
SCRIPTS, tests, and the scripts/manifests that rebuild the environment. It does NOT hold:
|
|
8
|
+
|
|
9
|
+
runtime a runtime or server distribution (MariaDB, PHP, JDK, Node, Python, Apache...)
|
|
10
|
+
binary compiled or packaged files (.exe .dll .so .jar .msi, archives)
|
|
11
|
+
db-data database data files (ibdata1, *.ibd, *.frm, *.mdf, SQLite files, dumps)
|
|
12
|
+
data-export data files above the size threshold (.csv .xlsx .json .xml .sql ...), or any
|
|
13
|
+
data file under a backups/dumps/exports folder
|
|
14
|
+
dependency installed dependencies (node_modules, vendor, .venv, site-packages)
|
|
15
|
+
backup editor and copy backups (*.bak, *.orig, *.old, *~)
|
|
16
|
+
|
|
17
|
+
Large files that match no rule are reported as 'large-review' (a decision, not a violation).
|
|
18
|
+
|
|
19
|
+
.gitignore does not remove a file that was committed before it: every excluded path is
|
|
20
|
+
checked in ALL history, not only in the current files.
|
|
21
|
+
|
|
22
|
+
Reads <staging>\<Name>\mirror.git. Changes nothing. Writes content-policy.json next to it.
|
|
23
|
+
Exit 0: compliant. Exit 1: at least one excluded path exists somewhere in history.
|
|
24
|
+
|
|
25
|
+
Project overrides (optional) in the config file, key "contentPolicy":
|
|
26
|
+
{ "keep": ["regex", ...], "exclude": ["regex", ...], "dataThresholdMB": 5, "largeMB": 10 }
|
|
27
|
+
Regexes match the repository path (forward slashes).
|
|
28
|
+
#>
|
|
29
|
+
[CmdletBinding()]
|
|
30
|
+
param(
|
|
31
|
+
[Parameter(Mandatory)][string]$Name,
|
|
32
|
+
[string]$StagingRoot,
|
|
33
|
+
[string]$ConfigPath,
|
|
34
|
+
[double]$DataThresholdMB,
|
|
35
|
+
[double]$LargeMB,
|
|
36
|
+
# Print every finding, not only the per-folder summary.
|
|
37
|
+
[switch]$Detail
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
$ErrorActionPreference = 'Stop'
|
|
41
|
+
. (Join-Path $PSScriptRoot 'PortCommon.ps1')
|
|
42
|
+
|
|
43
|
+
$config = Get-PortConfig $ConfigPath
|
|
44
|
+
$stage = Join-Path (Get-PortStagingRoot $config $StagingRoot) $Name
|
|
45
|
+
$mirror = Join-Path $stage 'mirror.git'
|
|
46
|
+
if (-not (Test-Path -LiteralPath $mirror)) { throw "No staged mirror at $mirror. Run Get-PortSource.ps1 first." }
|
|
47
|
+
$manifest = Get-Content -LiteralPath (Join-Path $stage 'manifest.json') -Raw | ConvertFrom-Json
|
|
48
|
+
$branch = $manifest.defaultBranch
|
|
49
|
+
|
|
50
|
+
$policy = Get-PortConfigValue $config 'contentPolicy' $null
|
|
51
|
+
$keepRx = @(); $extraExcludeRx = @()
|
|
52
|
+
if ($policy) {
|
|
53
|
+
if ($policy.PSObject.Properties.Name -contains 'keep') { $keepRx = @($policy.keep) }
|
|
54
|
+
if ($policy.PSObject.Properties.Name -contains 'exclude') { $extraExcludeRx = @($policy.exclude) }
|
|
55
|
+
if (-not $DataThresholdMB -and $policy.PSObject.Properties.Name -contains 'dataThresholdMB') { $DataThresholdMB = [double]$policy.dataThresholdMB }
|
|
56
|
+
if (-not $LargeMB -and $policy.PSObject.Properties.Name -contains 'largeMB') { $LargeMB = [double]$policy.largeMB }
|
|
57
|
+
}
|
|
58
|
+
if (-not $DataThresholdMB) { $DataThresholdMB = 5 }
|
|
59
|
+
if (-not $LargeMB) { $LargeMB = 10 }
|
|
60
|
+
$dataBytes = [int64]($DataThresholdMB * 1MB)
|
|
61
|
+
$largeBytes = [int64]($LargeMB * 1MB)
|
|
62
|
+
|
|
63
|
+
# A runtime folder is the runtime's NAME, optionally followed by a version or platform tag
|
|
64
|
+
# (php-8.3.33, jdk-17.0.2, node-v20.11.0-win-x64, apache24). Not any folder that merely STARTS
|
|
65
|
+
# with one: redis-migrations/ or apache-poi-wrapper/ are source.
|
|
66
|
+
$ver = '([-_ ]?v?\d[\w.]*(-(win|linux|x64|x86|amd64|arm64)[\w.-]*)?)?'
|
|
67
|
+
$runtimeDirRx = "(?i)^(mariadb|mysql|mysql-server|php|jdk|jre|openjdk|java|python|python-embed|nodejs|node|apache|httpd|nginx|tomcat|apache-tomcat|redis|postgres|postgresql|pgsql|erlang|ruby|dotnet|go)$ver$"
|
|
68
|
+
$runtimeBinRx = '(?i)^(mariadbd|mysqld|mariadb|mysql|php|php-cgi|node|python|pythonw|java|javaw|httpd|nginx|redis-server|postgres|pg_ctl|erl|ruby|dotnet)\.exe$'
|
|
69
|
+
$dependencyDirRx = '(?i)^(node_modules|vendor|\.venv|venv|site-packages|bower_components|\.yarn|jspm_packages)$'
|
|
70
|
+
$binaryExtRx = '(?i)\.(exe|dll|so|dylib|msi|msix|bin|iso|img|jar|war|ear|class|pyc|pdb|lib|a|o|obj|zip|7z|tar|gz|tgz|rar|bz2|xz|cab|nupkg|whl|deb|rpm)$'
|
|
71
|
+
$dbDataRx = '(?i)(^|/)(ibdata\d*|ib_logfile\d*|ib_buffer_pool|aria_log[\w.]*|undo\d+)$|\.(ibd|frm|myd|myi|mdf|ldf|ndf|sqlite3?|db3|dmp)$'
|
|
72
|
+
$backupRx = '(?i)(\.(bak|orig|old|swp)|~)$'
|
|
73
|
+
$dataExtRx = '(?i)\.(csv|tsv|xlsx|xlsm|xlsb|xls|ods|parquet|avro|orc|jsonl|ndjson|json|xml|sql|dat|txt|log)$'
|
|
74
|
+
$dataDirRx = '(?i)^(backups?|dumps?|exports?|snapshots?|data-?dumps?)$'
|
|
75
|
+
|
|
76
|
+
function Get-Category([string]$Path, [int64]$Bytes) {
|
|
77
|
+
foreach ($rx in $keepRx) { if ($Path -match $rx) { return $null } }
|
|
78
|
+
foreach ($rx in $extraExcludeRx) { if ($Path -match $rx) { return 'project-exclude' } }
|
|
79
|
+
$parts = $Path -split '/'
|
|
80
|
+
$dirs = if ($parts.Count -gt 1) { $parts[0..($parts.Count - 2)] } else { @() }
|
|
81
|
+
$leaf = $parts[-1]
|
|
82
|
+
foreach ($d in $dirs) { if ($d -match $runtimeDirRx) { return 'runtime' } }
|
|
83
|
+
if ($leaf -match $runtimeBinRx) { return 'runtime' }
|
|
84
|
+
foreach ($d in $dirs) { if ($d -match $dependencyDirRx) { return 'dependency' } }
|
|
85
|
+
if ($Path -match $dbDataRx) { return 'db-data' }
|
|
86
|
+
if ($leaf -match $backupRx) { return 'backup' }
|
|
87
|
+
if ($leaf -match $binaryExtRx) { return 'binary' }
|
|
88
|
+
if ($leaf -match $dataExtRx) {
|
|
89
|
+
# Under a backups/dumps folder any data file is excluded, but a plain note (.txt/.md) is not data.
|
|
90
|
+
if ($leaf -notmatch '(?i)\.(txt|md)$') { foreach ($d in $dirs) { if ($d -match $dataDirRx) { return 'data-export' } } }
|
|
91
|
+
# Migration SCRIPTS stay by policy. A large .sql under migration(s)/ may be a script or a
|
|
92
|
+
# data load, so it is a decision, not an automatic exclusion. Data FILES there (csv, json,
|
|
93
|
+
# xlsx) are still data.
|
|
94
|
+
if ($leaf -match '(?i)\.sql$' -and ($dirs | Where-Object { $_ -match '(?i)^migrations?$' })) {
|
|
95
|
+
if ($Bytes -ge $largeBytes -or $Bytes -ge $dataBytes) { return 'large-review' }
|
|
96
|
+
return $null
|
|
97
|
+
}
|
|
98
|
+
if ($Bytes -ge $dataBytes) { return 'data-export' }
|
|
99
|
+
}
|
|
100
|
+
if ($Bytes -ge $largeBytes) { return 'large-review' }
|
|
101
|
+
return $null
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function MGit([string[]]$a) { Invoke-PortGit (@('-C', $mirror) + $a) }
|
|
105
|
+
|
|
106
|
+
# ---- every blob in history, with every path it ever had ----------------------------------
|
|
107
|
+
Write-PortStep 'Reading every path and blob in history'
|
|
108
|
+
$pathsBySha = @{}
|
|
109
|
+
$submodules = New-Object System.Collections.Generic.HashSet[string]
|
|
110
|
+
function Add-PathForSha([string]$sha, [string]$p) {
|
|
111
|
+
if (-not $pathsBySha.ContainsKey($sha)) { $pathsBySha[$sha] = New-Object System.Collections.Generic.HashSet[string] }
|
|
112
|
+
[void]$pathsBySha[$sha].Add($p)
|
|
113
|
+
}
|
|
114
|
+
# EVERY path a blob was ever written to. rev-list --objects names each blob only ONCE, by the
|
|
115
|
+
# first path it meets, so identical content at a second path was invisible (found by the
|
|
116
|
+
# dogfood: a 3-byte test image at two paths survived a clean). log --raw lists every add or
|
|
117
|
+
# change, per commit, including merges (-m) and root commits.
|
|
118
|
+
foreach ($l in (MGit @('-c', 'core.quotePath=false', 'log', '--all', '--raw', '--no-abbrev', '--no-renames', '-m', '--root', '--format=', '--diff-filter=AMT'))) {
|
|
119
|
+
if (-not $l -or $l[0] -ne ':') { continue }
|
|
120
|
+
$meta, $p = $l -split "`t", 2
|
|
121
|
+
$f = $meta -split ' '
|
|
122
|
+
if ($f.Count -lt 4 -or -not $p -or $f[3] -match '^0+$') { continue }
|
|
123
|
+
# Mode 160000 is a submodule: its "blob" is a commit in ANOTHER repository, invisible to a
|
|
124
|
+
# size scan. Record it, so a runtime vendored as a submodule cannot pass unnoticed.
|
|
125
|
+
if ($f[1] -eq '160000') { [void]$submodules.Add($p); continue }
|
|
126
|
+
Add-PathForSha $f[3] $p
|
|
127
|
+
}
|
|
128
|
+
# Safety net: anything reachable that the log did not show.
|
|
129
|
+
foreach ($l in (MGit @('rev-list', '--objects', '--all'))) {
|
|
130
|
+
$sha, $p = $l -split ' ', 2
|
|
131
|
+
if ($p -and -not $pathsBySha.ContainsKey($sha)) { Add-PathForSha $sha $p }
|
|
132
|
+
}
|
|
133
|
+
$sizeBySha = @{}
|
|
134
|
+
foreach ($l in (MGit @('cat-file', '--batch-all-objects', '--batch-check=%(objecttype) %(objectname) %(objectsize)'))) {
|
|
135
|
+
$t, $o, $s = $l -split ' '
|
|
136
|
+
if ($t -eq 'blob' -and $pathsBySha.ContainsKey($o)) { $sizeBySha[$o] = [int64]$s }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
# Aggregate per path: largest version, number of versions.
|
|
140
|
+
$byPath = @{}
|
|
141
|
+
foreach ($sha in $sizeBySha.Keys) {
|
|
142
|
+
foreach ($p in $pathsBySha[$sha]) {
|
|
143
|
+
if (-not $byPath.ContainsKey($p)) { $byPath[$p] = [pscustomobject]@{ path = $p; maxBytes = [int64]0; versions = 0 } }
|
|
144
|
+
$e = $byPath[$p]; $e.versions++
|
|
145
|
+
if ($sizeBySha[$sha] -gt $e.maxBytes) { $e.maxBytes = $sizeBySha[$sha] }
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
$tipFiles = @{}
|
|
150
|
+
foreach ($l in (MGit @('-c', 'core.quotePath=false', 'ls-tree', '-r', '--name-only', $branch))) { $tipFiles[$l] = $true }
|
|
151
|
+
|
|
152
|
+
# ---- classify ----------------------------------------------------------------------------------
|
|
153
|
+
$findings = New-Object System.Collections.Generic.List[object]
|
|
154
|
+
foreach ($e in $byPath.Values) {
|
|
155
|
+
$cat = Get-Category $e.path $e.maxBytes
|
|
156
|
+
if (-not $cat) { continue }
|
|
157
|
+
$findings.Add([pscustomobject]@{
|
|
158
|
+
path = $e.path; category = $cat; maxBytes = $e.maxBytes; versions = $e.versions
|
|
159
|
+
inDefaultBranch = [bool]$tipFiles[$e.path]
|
|
160
|
+
action = if ($cat -eq 'large-review') { 'review' } else { 'exclude' }
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
foreach ($sm in $submodules) {
|
|
164
|
+
$findings.Add([pscustomobject]@{ path = $sm; category = 'submodule'; maxBytes = [int64]0; versions = 1; inDefaultBranch = [bool]$tipFiles[$sm]; action = 'review' })
|
|
165
|
+
}
|
|
166
|
+
$exclude = @($findings | Where-Object { $_.action -eq 'exclude' })
|
|
167
|
+
$review = @($findings | Where-Object { $_.action -eq 'review' })
|
|
168
|
+
|
|
169
|
+
# Group by top two path segments so a 3,000-file vendored runtime reads as one line.
|
|
170
|
+
# Group by folder, at most 4 levels deep, so one vendored runtime reads as one line and a
|
|
171
|
+
# data folder deep in an app is still named precisely.
|
|
172
|
+
function Get-Group([string]$p) { $s = $p -split '/'; if ($s.Count -le 1) { return $p }; $d = $s[0..([math]::Min(3, $s.Count - 2))]; ($d -join '/') + '/' }
|
|
173
|
+
$groups = @($findings | Group-Object { "$($_.category)|$(Get-Group $_.path)" } | ForEach-Object {
|
|
174
|
+
$cat, $grp = $_.Name -split '\|', 2
|
|
175
|
+
[pscustomobject]@{
|
|
176
|
+
category = $cat; group = $grp; files = $_.Count
|
|
177
|
+
historyMB = [math]::Round((($_.Group | Measure-Object maxBytes -Sum).Sum) / 1MB, 1)
|
|
178
|
+
inDefaultBranch = @($_.Group | Where-Object { $_.inDefaultBranch }).Count
|
|
179
|
+
}
|
|
180
|
+
} | Sort-Object historyMB -Descending)
|
|
181
|
+
|
|
182
|
+
$totalBytes = [int64]0; foreach ($v in $sizeBySha.Values) { $totalBytes += $v }
|
|
183
|
+
# Count each blob once, even when it sits at several excluded paths (a per-path sum can exceed
|
|
184
|
+
# the whole history). Plain loops: Measure-Object on NO input has no .Sum under strict mode.
|
|
185
|
+
$exPaths = New-Object System.Collections.Generic.HashSet[string]
|
|
186
|
+
foreach ($x in $exclude) { [void]$exPaths.Add($x.path) }
|
|
187
|
+
$exBytes = [int64]0
|
|
188
|
+
foreach ($sha in $sizeBySha.Keys) { foreach ($p in $pathsBySha[$sha]) { if ($exPaths.Contains($p)) { $exBytes += $sizeBySha[$sha]; break } } }
|
|
189
|
+
$report = [ordered]@{
|
|
190
|
+
name = $Name; defaultBranch = $branch; checkedAt = (Get-Date).ToString('o')
|
|
191
|
+
thresholds = [ordered]@{ dataThresholdMB = $DataThresholdMB; largeMB = $LargeMB }
|
|
192
|
+
historyBlobMB = [math]::Round($totalBytes / 1MB, 1)
|
|
193
|
+
excludedPaths = $exclude.Count; excludedMB = [math]::Round($exBytes / 1MB, 1)
|
|
194
|
+
reviewPaths = $review.Count
|
|
195
|
+
compliant = ($exclude.Count -eq 0)
|
|
196
|
+
groups = $groups
|
|
197
|
+
findings = @($findings | Sort-Object maxBytes -Descending)
|
|
198
|
+
}
|
|
199
|
+
ConvertTo-PortJsonFile $report (Join-Path $stage 'content-policy.json')
|
|
200
|
+
# Plain path list, one per line, for Invoke-PortHistoryClean.ps1 and for .gitignore review.
|
|
201
|
+
Set-Content -LiteralPath (Join-Path $stage 'content-policy-exclude.txt') -Value @($exclude | Sort-Object path | ForEach-Object { $_.path }) -Encoding UTF8
|
|
202
|
+
|
|
203
|
+
Write-Host ''
|
|
204
|
+
Write-Host ("Content policy for {0}: {1} excluded path(s), {2} MB of history; {3} path(s) to review. History holds {4} MB of blobs in total." -f $Name, $exclude.Count, $report.excludedMB, $review.Count, $report.historyBlobMB)
|
|
205
|
+
if ($groups.Count -gt 0) {
|
|
206
|
+
Write-Host ''
|
|
207
|
+
Write-Host ('{0,-13} {1,-55} {2,7} {3,10} {4}' -f 'category', 'where', 'files', 'MB', 'in default branch')
|
|
208
|
+
foreach ($g in $groups) { Write-Host ('{0,-13} {1,-55} {2,7} {3,10} {4}' -f $g.category, $g.group, $g.files, $g.historyMB, $g.inDefaultBranch) }
|
|
209
|
+
}
|
|
210
|
+
if ($Detail) { $findings | Sort-Object maxBytes -Descending | ForEach-Object { Write-Host (' {0,-13} {1,9:N0} KB {2}' -f $_.category, ($_.maxBytes / 1KB), $_.path) } }
|
|
211
|
+
Write-Host " report: $(Join-Path $stage 'content-policy.json')"
|
|
212
|
+
Write-Host " exclude list: $(Join-Path $stage 'content-policy-exclude.txt')"
|
|
213
|
+
if ($exclude.Count -gt 0) {
|
|
214
|
+
Write-PortWarn 'Not compliant. Decide per group: keep (add a "keep" regex to contentPolicy), or remove from history with Invoke-PortHistoryClean.ps1. Adding a path to .gitignore does NOT remove it from history.'
|
|
215
|
+
exit 1
|
|
216
|
+
}
|
|
217
|
+
Write-PortOk 'Compliant: no excluded content anywhere in history.'
|
|
218
|
+
exit 0
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: port-to-github
|
|
3
|
-
description: Move (port) an app or repository to GitHub from ANY source - a Gitea/GitLab/any git server URL, a local folder, a network (UNC) share, or a folder on a remote Windows box reached over WinRM - with or without existing git history. Keeps every branch and tag, verifies by SHA, translates Gitea/Forgejo Actions CI, switches deploy checkouts to GitHub with automatic rollback, and retires the old host safely. Use when the user says "move this app to GitHub", "port X to GitHub", "migrate from Gitea", "put this folder on GitHub", "this app only exists on the server", "import the repo from the prod box", or "cut the deploy over to GitHub".
|
|
3
|
+
description: Move (port) an app or repository to GitHub from ANY source - a Gitea/GitLab/any git server URL, a local folder, a network (UNC) share, or a folder on a remote Windows box reached over WinRM - with or without existing git history. Keeps every branch and tag, verifies by SHA, translates Gitea/Forgejo Actions CI, enforces a source-only content policy (runtimes, binaries, data exports and dependencies stay out of the repo, over ALL history, with a runtime manifest and provisioning script so the environment is still reproducible), switches deploy checkouts to GitHub with automatic rollback, and retires the old host safely. Use when the user says "move this app to GitHub", "port X to GitHub", "migrate from Gitea", "put this folder on GitHub", "this app only exists on the server", "import the repo from the prod box", "keep binaries / MariaDB / PHP out of the repo", "check the repo only holds source", or "cut the deploy over to GitHub".
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# port-to-github
|
|
7
7
|
|
|
8
8
|
Move one app to GitHub without losing history, leaking a secret, or breaking its deploy.
|
|
9
9
|
|
|
10
|
-
The work is in **six phases
|
|
10
|
+
The work is in **six phases**, plus the content policy (2b). Each ends in a check that can FAIL. Do not start a phase until
|
|
11
11
|
the one before it passed. Scripts are in `scripts/` next to this file. They are PowerShell 7
|
|
12
12
|
(`pwsh`); the parts that run on a remote box are Windows PowerShell 5.1 compatible.
|
|
13
13
|
|
|
@@ -15,6 +15,7 @@ the one before it passed. Scripts are in `scripts/` next to this file. They are
|
|
|
15
15
|
|---|---|---|
|
|
16
16
|
| 1. Stage the source | `Get-PortSource.ps1` | No. The source is only read. |
|
|
17
17
|
| 2. Readiness | `Test-PortReadiness.ps1` | No. |
|
|
18
|
+
| 2b. Content policy | `Test-PortContentPolicy.ps1`, `New-PortRuntimeManifest.ps1`, `Invoke-PortHistoryClean.ps1` | No. Rewrites only the local stage. |
|
|
18
19
|
| 3. Publish | `Publish-PortToGitHub.ps1` | **Yes: creates a GitHub repo and pushes.** |
|
|
19
20
|
| 4. CI | `Convert-PortWorkflow.ps1`, `Test-PortWorkflowParses.ps1` | Yes: a feature branch and a PR. |
|
|
20
21
|
| 5. Cutover | `Switch-CheckoutOrigin.ps1` | **Yes: changes where a deploy host pulls from.** |
|
|
@@ -64,6 +65,9 @@ Read the output. These need a decision, not a shrug:
|
|
|
64
65
|
- **Uncommitted changes** in a source checkout are left out unless you pass
|
|
65
66
|
`-IncludeUncommitted`, which puts them on a separate branch `port/uncommitted-snapshot-*`.
|
|
66
67
|
The real branches stay exactly as committed. Ask the user which they want.
|
|
68
|
+
- **Default branch.** From a server or a bare repository, its HEAD. From a WORKING checkout,
|
|
69
|
+
`main` (or `master`) when it exists, because the checked-out branch is only what someone had
|
|
70
|
+
open; the script says when it made that choice. Override with `-DefaultBranch <name>`.
|
|
67
71
|
- **Remote-only branches**: branches the checkout only knows as `origin/x`. If the checkout is
|
|
68
72
|
the only copy, re-run with `-PromoteRemoteBranches origin`. If a server has them, port from it.
|
|
69
73
|
- **Likely secrets** in a plain folder or a snapshot stop the script (exit 2). Nothing has left
|
|
@@ -93,6 +97,63 @@ deploy jobs included. Readiness lists those branches; publish refuses them unles
|
|
|
93
97
|
(the new repo will expose them), CI credentials, dependency hosts, runner labels, files that
|
|
94
98
|
still name the old host. The report is in `readiness.json`.
|
|
95
99
|
|
|
100
|
+
## Phase 2b - content policy: source only
|
|
101
|
+
|
|
102
|
+
**The rule.** The repository holds application code, configuration and templates, migration
|
|
103
|
+
SCRIPTS, tests, and the scripts and manifests that rebuild the environment. Runtime and server
|
|
104
|
+
distributions (MariaDB, PHP, JDK, Node, Python, Apache...), binaries and archives, database data
|
|
105
|
+
files, data exports and installed dependencies stay OUT, and are installed or provisioned
|
|
106
|
+
separately. This is the default for every port; a project narrows or widens it with a
|
|
107
|
+
`contentPolicy` block in its config (`keep` / `exclude` regexes, `dataThresholdMB`, `largeMB`).
|
|
108
|
+
|
|
109
|
+
```powershell
|
|
110
|
+
& $s/Test-PortContentPolicy.ps1 -Name <repo-name>
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
It checks the default branch AND every blob in history, because **`.gitignore` does not remove a
|
|
114
|
+
file that was committed before it**. Exit 1 means excluded content exists somewhere in history.
|
|
115
|
+
The output groups findings by folder; `content-policy.json` has every path.
|
|
116
|
+
|
|
117
|
+
For each group, decide with the user:
|
|
118
|
+
|
|
119
|
+
1. **It belongs** (a small fixture, a vendored library the team maintains) -> add a `keep` regex
|
|
120
|
+
to the project config and re-run.
|
|
121
|
+
2. **It does not** -> remove it from the STAGE's history. Two ways, and it is the owner's call:
|
|
122
|
+
```powershell
|
|
123
|
+
& $s/Invoke-PortHistoryClean.ps1 -Name <repo-name> -Mode Rewrite # keep history, drop the paths
|
|
124
|
+
& $s/Invoke-PortHistoryClean.ps1 -Name <repo-name> -Mode FreshStart # one commit of today's files
|
|
125
|
+
```
|
|
126
|
+
`Rewrite` keeps every branch, tag and commit but CHANGES commit SHAs from the first affected
|
|
127
|
+
commit on; anything quoting an old SHA stops matching. `FreshStart` is smallest and drops all
|
|
128
|
+
history, other branches and tags. Both re-run the policy check and exit 1 if anything
|
|
129
|
+
excluded survived. The source keeps its original history either way.
|
|
130
|
+
|
|
131
|
+
**Keep the environment reproducible.** Excluding a runtime is only safe if the repo says how to
|
|
132
|
+
get it back. Scan the SOURCE FOLDER (runtimes usually live in an ignored `tools/`):
|
|
133
|
+
|
|
134
|
+
```powershell
|
|
135
|
+
& $s/New-PortRuntimeManifest.ps1 -Path <source-folder> [-ComputerName <box>] -OutDir <clone>\environment
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
It writes `runtimes.json` (kind, exact version, binary SHA-256, data folders), `provision.ps1`
|
|
139
|
+
(downloads and unpacks each runtime with a known public source and FAILS for any it cannot
|
|
140
|
+
provision), `README.md`, and `config-templates/`: configuration found inside runtime folders
|
|
141
|
+
(`my.ini`, `php.ini`, `config/`, `metadata/`). Configuration is app content even when the runtime
|
|
142
|
+
around it is not. Files that look like they hold a credential are withheld and listed; make
|
|
143
|
+
those templates by hand with placeholders. **The credential check is a heuristic: read every
|
|
144
|
+
copied template yourself before committing it.** The dogfood found one it missed (a SAML test
|
|
145
|
+
user written as `'name:password' => array(...)`); the rule now covers that form, but the next
|
|
146
|
+
form may be new. Nothing it finds is executed; versions come from file
|
|
147
|
+
metadata. Commit `environment/` on the phase 4 branch, by file name.
|
|
148
|
+
|
|
149
|
+
Two things it cannot decide, so you must:
|
|
150
|
+
- **UNCLASSIFIED folders** beside a runtime (a deployed web tree, a library with no binary).
|
|
151
|
+
Decide each: provision it, rebuild it from repo code, or keep it in the repo.
|
|
152
|
+
- **Data folders** (e.g. a database `data/` directory). This is state, not code. The repo carries
|
|
153
|
+
the schema and migrations; the data comes from a backup. Say where that backup lives.
|
|
154
|
+
|
|
155
|
+
Then add every excluded folder to `.gitignore` on the phase 4 branch, so it cannot come back.
|
|
156
|
+
|
|
96
157
|
## Phase 3 - publish (ask first)
|
|
97
158
|
|
|
98
159
|
Tell the user: the repo name, owner, visibility, branch and tag counts. Then:
|
|
@@ -101,7 +162,7 @@ Tell the user: the repo name, owner, visibility, branch and tag counts. Then:
|
|
|
101
162
|
& $s/Publish-PortToGitHub.ps1 -Name <repo-name> [-Owner <org>] [-Visibility private]
|
|
102
163
|
```
|
|
103
164
|
|
|
104
|
-
It
|
|
165
|
+
It refuses a stage whose content policy is not compliant (run phase 2b; `-AcceptContentPolicy` records an explicit owner exception instead). It then checks the stage still holds exactly the refs phase 1 recorded (anything else would be
|
|
105
166
|
pushed and then "verified" against itself), creates the repo EMPTY, pushes `refs/heads/*` and
|
|
106
167
|
`refs/tags/*` with an explicit refspec, sets the default branch, and compares every ref by SHA
|
|
107
168
|
(exit 1 on any difference). Do not run other git commands inside the stage; if you did,
|
|
@@ -37,6 +37,9 @@ param(
|
|
|
37
37
|
# What to do when a plain folder or an uncommitted snapshot contains a likely secret.
|
|
38
38
|
[ValidateSet('Stop', 'Exclude')][string]$OnSecret = 'Stop',
|
|
39
39
|
[string]$InitialBranch = 'main',
|
|
40
|
+
# The GitHub default branch. Default: the server's HEAD; for a working checkout, main/master
|
|
41
|
+
# when present (the checked-out branch is only what someone had open).
|
|
42
|
+
[string]$DefaultBranch,
|
|
40
43
|
# Replace an existing staging folder for this Name.
|
|
41
44
|
[switch]$Force
|
|
42
45
|
)
|
|
@@ -315,6 +318,7 @@ switch ($kind) {
|
|
|
315
318
|
}
|
|
316
319
|
}
|
|
317
320
|
$r.isRepo = $isRepo
|
|
321
|
+
$r.isBare = $isBare
|
|
318
322
|
if ($isRepo) {
|
|
319
323
|
$bundle = Join-Path $tmp 'repo.bundle'
|
|
320
324
|
& $git -c safe.directory=* -C $Path bundle create $bundle --all 2>&1 | Out-Null
|
|
@@ -362,7 +366,13 @@ switch ($kind) {
|
|
|
362
366
|
Invoke-PortGit @('bundle', 'verify', $localBundle) | Out-Null
|
|
363
367
|
Invoke-PortGit @('clone', '-q', '--mirror', $localBundle, $mirror) | Out-Null
|
|
364
368
|
$manifest.hadGitHistory = $true
|
|
365
|
-
|
|
369
|
+
# A bare repository's HEAD is authoritative, like a server's; only a WORKING checkout's
|
|
370
|
+
# HEAD is 'whatever was open'. Leaving sourceHead empty keeps the main-line preference off.
|
|
371
|
+
$manifest.sourceHead = if ($remote.isBare) { $null } elseif ($remote.head) { $remote.head } else { $remote.headSha }
|
|
372
|
+
# A bundle records HEAD as a commit, not a branch name, so a clone of it GUESSES the
|
|
373
|
+
# default when two branches share that commit. For a bare repo the name read on the
|
|
374
|
+
# far side is authoritative; carry it across.
|
|
375
|
+
if ($remote.isBare -and $remote.head) { $authHead = $remote.head }
|
|
366
376
|
$manifest.dirtyFiles = @($remote.dirty)
|
|
367
377
|
if ($manifest.dirtyFiles.Count -gt 0) {
|
|
368
378
|
if ($IncludeUncommitted -and $remote.dirtyZip) {
|
|
@@ -430,11 +440,29 @@ if ($heads.Count -eq 0) { throw "The source produced no branches. Nothing to por
|
|
|
430
440
|
|
|
431
441
|
$symHead = Invoke-PortGit -AllowFailure @('-C', $mirror, 'symbolic-ref', '--short', 'HEAD')
|
|
432
442
|
$def = if ($symHead.ExitCode -eq 0) { $symHead.Output -join '' } else { $null }
|
|
443
|
+
$mainLine = @('main', 'master') | Where-Object { $headNames -contains $_ } | Select-Object -First 1
|
|
444
|
+
if (-not (Get-Variable -Name authHead -ErrorAction SilentlyContinue)) { $authHead = $null }
|
|
445
|
+
# The branch the source checkout was really on (a bundle's HEAD is only a guess), so the
|
|
446
|
+
# main-line choice below can say what it replaced.
|
|
447
|
+
if ($manifest.sourceHead -and ($headNames -contains $manifest.sourceHead)) { $def = $manifest.sourceHead }
|
|
448
|
+
if ($DefaultBranch) {
|
|
449
|
+
if (-not ($headNames -contains $DefaultBranch)) { throw "-DefaultBranch '$DefaultBranch' is not a branch in the source." }
|
|
450
|
+
$def = $DefaultBranch
|
|
451
|
+
}
|
|
452
|
+
elseif ($authHead -and ($headNames -contains $authHead)) {
|
|
453
|
+
$def = $authHead
|
|
454
|
+
}
|
|
455
|
+
elseif ($kind -ne 'url' -and $manifest.sourceHead -and $mainLine -and $def -ne $mainLine) {
|
|
456
|
+
# A server's HEAD IS its default branch. A working checkout's HEAD is only whatever branch
|
|
457
|
+
# someone had open, so prefer the main line when there is one, and say so.
|
|
458
|
+
$manifest.warnings += "The checkout was on '$def', but '$mainLine' exists, so '$mainLine' is the default branch on GitHub. Pass -DefaultBranch to choose another."
|
|
459
|
+
$def = $mainLine
|
|
460
|
+
}
|
|
433
461
|
if (-not $def -or -not ($headNames -contains $def)) {
|
|
434
462
|
$def = @('main', 'master', 'develop') | Where-Object { $headNames -contains $_ } | Select-Object -First 1
|
|
435
463
|
if (-not $def) { $def = $headNames[0] }
|
|
436
|
-
Invoke-PortGit @('-C', $mirror, 'symbolic-ref', 'HEAD', "refs/heads/$def") | Out-Null
|
|
437
464
|
}
|
|
465
|
+
Invoke-PortGit @('-C', $mirror, 'symbolic-ref', 'HEAD', "refs/heads/$def") | Out-Null
|
|
438
466
|
$manifest.defaultBranch = $def
|
|
439
467
|
|
|
440
468
|
if ($manifest.remoteOnlyBranches.Count -gt 0) {
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Remove excluded paths from the STAGED repository's history, then prove they are gone.
|
|
4
|
+
|
|
5
|
+
.DESCRIPTION
|
|
6
|
+
Works only on <staging>\<Name>\mirror.git. The source is never touched.
|
|
7
|
+
|
|
8
|
+
-Mode Rewrite keep every branch, tag and commit, minus the excluded paths. Commit SHAs
|
|
9
|
+
change from the first commit that touched an excluded path onward, so
|
|
10
|
+
anything that quotes an old SHA (a deploy record, a tag note) stops matching.
|
|
11
|
+
Uses git filter-repo if installed, otherwise git filter-branch.
|
|
12
|
+
-Mode FreshStart one new commit holding the default branch's current files minus the excluded
|
|
13
|
+
paths. All other branches, tags and history are dropped. Smallest, simplest,
|
|
14
|
+
and loses the history - an owner decision.
|
|
15
|
+
|
|
16
|
+
Paths come from content-policy-exclude.txt (written by Test-PortContentPolicy.ps1) unless
|
|
17
|
+
-PathsFile is given. Afterwards it re-runs Test-PortContentPolicy.ps1, which must exit 0, and
|
|
18
|
+
rewrites manifest.json's heads and tags so Publish's stage-drift check sees the cleaned refs.
|
|
19
|
+
#>
|
|
20
|
+
[CmdletBinding(SupportsShouldProcess)]
|
|
21
|
+
param(
|
|
22
|
+
[Parameter(Mandatory)][string]$Name,
|
|
23
|
+
[Parameter(Mandatory)][ValidateSet('Rewrite', 'FreshStart')][string]$Mode,
|
|
24
|
+
[string]$PathsFile,
|
|
25
|
+
[string]$StagingRoot,
|
|
26
|
+
[string]$ConfigPath
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
$ErrorActionPreference = 'Stop'
|
|
30
|
+
. (Join-Path $PSScriptRoot 'PortCommon.ps1')
|
|
31
|
+
|
|
32
|
+
$config = Get-PortConfig $ConfigPath
|
|
33
|
+
$stage = Join-Path (Get-PortStagingRoot $config $StagingRoot) $Name
|
|
34
|
+
$mirror = Join-Path $stage 'mirror.git'
|
|
35
|
+
$manifestPath = Join-Path $stage 'manifest.json'
|
|
36
|
+
if (-not (Test-Path -LiteralPath $mirror)) { throw "No staged mirror at $mirror." }
|
|
37
|
+
if (-not $PathsFile) { $PathsFile = Join-Path $stage 'content-policy-exclude.txt' }
|
|
38
|
+
if (-not (Test-Path -LiteralPath $PathsFile)) { throw "No paths file at $PathsFile. Run Test-PortContentPolicy.ps1 first." }
|
|
39
|
+
$paths = @(Get-Content -LiteralPath $PathsFile | Where-Object { $_ -and $_.Trim() })
|
|
40
|
+
if ($paths.Count -eq 0) { Write-PortOk 'Nothing to remove.'; exit 0 }
|
|
41
|
+
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
|
42
|
+
$branch = $manifest.defaultBranch
|
|
43
|
+
|
|
44
|
+
# git reads pathspecs as patterns; ":(literal)" makes every entry an exact path, so a file
|
|
45
|
+
# named with [ ] * ? is never widened into a glob that removes something else.
|
|
46
|
+
$specFile = Join-Path $stage 'history-clean-pathspec.txt'
|
|
47
|
+
[IO.File]::WriteAllLines($specFile, [string[]]@($paths | ForEach-Object { ":(literal)$_" }), (New-Object Text.UTF8Encoding($false)))
|
|
48
|
+
|
|
49
|
+
$before = @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/heads', 'refs/tags'))
|
|
50
|
+
if (-not $PSCmdlet.ShouldProcess($mirror, "$Mode history, removing $($paths.Count) path(s)")) { exit 0 }
|
|
51
|
+
|
|
52
|
+
if ($Mode -eq 'Rewrite') {
|
|
53
|
+
$fr = Invoke-PortGit -AllowFailure @('filter-repo', '--version')
|
|
54
|
+
if ($fr.ExitCode -eq 0) {
|
|
55
|
+
Write-PortStep "Rewriting with git filter-repo ($($paths.Count) path(s))"
|
|
56
|
+
$plain = Join-Path $stage 'history-clean-paths.txt'
|
|
57
|
+
[IO.File]::WriteAllLines($plain, [string[]]$paths, (New-Object Text.UTF8Encoding($false)))
|
|
58
|
+
Invoke-PortGit @('-C', $mirror, 'filter-repo', '--force', '--invert-paths', '--paths-from-file', $plain) | Out-Null
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
Write-PortStep "Rewriting with git filter-branch ($($paths.Count) path(s)); git filter-repo is not installed"
|
|
62
|
+
$env:FILTER_BRANCH_SQUELCH_WARNING = '1'
|
|
63
|
+
# The index filter is eval'd by a shell. Pass the path through an environment variable, so a
|
|
64
|
+
# quote in it (C:\Users\O'Brien\...) cannot break the command.
|
|
65
|
+
$env:PORT_PATHSPEC_FILE = $specFile.Replace('\', '/')
|
|
66
|
+
try {
|
|
67
|
+
$idx = 'git rm -r --cached --ignore-unmatch --quiet --pathspec-from-file="$PORT_PATHSPEC_FILE"'
|
|
68
|
+
Invoke-PortGit @('-C', $mirror, 'filter-branch', '-f', '--index-filter', $idx, '--prune-empty', '--tag-name-filter', 'cat', '--', '--all') | Out-Null
|
|
69
|
+
}
|
|
70
|
+
finally { Remove-Item Env:FILTER_BRANCH_SQUELCH_WARNING, Env:PORT_PATHSPEC_FILE -ErrorAction SilentlyContinue }
|
|
71
|
+
foreach ($r in @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname)', 'refs/original'))) {
|
|
72
|
+
if ($r) { Invoke-PortGit @('-C', $mirror, 'update-ref', '-d', $r) | Out-Null }
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
Write-PortStep "Fresh start: one commit from $branch minus $($paths.Count) path(s)"
|
|
78
|
+
$tmpIndex = Join-Path $stage 'fresh.index'
|
|
79
|
+
# git rm needs a work tree even with --cached; a bare mirror has none, so lend it an empty one.
|
|
80
|
+
$tmpWork = Join-Path $stage 'fresh-worktree'
|
|
81
|
+
New-Item -ItemType Directory -Force -Path $tmpWork | Out-Null
|
|
82
|
+
$env:GIT_INDEX_FILE = $tmpIndex
|
|
83
|
+
$env:GIT_WORK_TREE = $tmpWork
|
|
84
|
+
try {
|
|
85
|
+
Invoke-PortGit @('-C', $mirror, 'read-tree', $branch) | Out-Null
|
|
86
|
+
Invoke-PortGit @('-C', $mirror, 'rm', '-r', '--cached', '--ignore-unmatch', '--quiet', "--pathspec-from-file=$specFile") | Out-Null
|
|
87
|
+
$tree = (Invoke-PortGit @('-C', $mirror, 'write-tree')) -join ''
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
Remove-Item Env:GIT_INDEX_FILE, Env:GIT_WORK_TREE -ErrorAction SilentlyContinue
|
|
91
|
+
Remove-Item -LiteralPath $tmpIndex -ErrorAction SilentlyContinue
|
|
92
|
+
Remove-Item -LiteralPath $tmpWork -Recurse -Force -ErrorAction SilentlyContinue
|
|
93
|
+
}
|
|
94
|
+
$msg = "Initial import of $Name (fresh start: history not carried over)"
|
|
95
|
+
$body = "Source: $($manifest.source), branch $branch at $((Invoke-PortGit @('-C', $mirror, 'rev-parse', $branch)) -join ''). $($paths.Count) path(s) excluded by the content policy."
|
|
96
|
+
$commit = (Invoke-PortGit @('-C', $mirror, 'commit-tree', $tree, '-m', $msg, '-m', $body)) -join ''
|
|
97
|
+
foreach ($r in @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname)', 'refs/heads', 'refs/tags'))) {
|
|
98
|
+
if ($r) { Invoke-PortGit @('-C', $mirror, 'update-ref', '-d', $r) | Out-Null }
|
|
99
|
+
}
|
|
100
|
+
Invoke-PortGit @('-C', $mirror, 'update-ref', "refs/heads/$branch", $commit) | Out-Null
|
|
101
|
+
Invoke-PortGit @('-C', $mirror, 'symbolic-ref', 'HEAD', "refs/heads/$branch") | Out-Null
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
# Drop the unreachable objects, so a later readiness scan measures what will actually be pushed.
|
|
105
|
+
Invoke-PortGit @('-C', $mirror, 'reflog', 'expire', '--expire=now', '--all') | Out-Null
|
|
106
|
+
Invoke-PortGit @('-C', $mirror, 'gc', '--prune=now', '--quiet') | Out-Null
|
|
107
|
+
|
|
108
|
+
$after = @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/heads', 'refs/tags'))
|
|
109
|
+
$manifest.heads = @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname:short) %(objectname)', 'refs/heads'))
|
|
110
|
+
$manifest.tags = @(Invoke-PortGit @('-C', $mirror, 'for-each-ref', '--format=%(refname:short) %(objectname)', 'refs/tags'))
|
|
111
|
+
$manifest | Add-Member -NotePropertyName historyClean -NotePropertyValue ([pscustomobject]@{
|
|
112
|
+
mode = $Mode; at = (Get-Date).ToString('o'); pathsRemoved = $paths.Count
|
|
113
|
+
refsBefore = $before.Count; refsAfter = $after.Count
|
|
114
|
+
note = 'Commit SHAs changed. The source still holds the original history.'
|
|
115
|
+
}) -Force
|
|
116
|
+
ConvertTo-PortJsonFile $manifest $manifestPath
|
|
117
|
+
|
|
118
|
+
# The proof: the policy must now pass. A clean that cannot fail is not a clean.
|
|
119
|
+
Write-PortStep 'Re-checking the content policy on the cleaned history'
|
|
120
|
+
& (Join-Path $PSScriptRoot 'Test-PortContentPolicy.ps1') -Name $Name -StagingRoot $StagingRoot -ConfigPath $ConfigPath
|
|
121
|
+
if ($LASTEXITCODE -ne 0) { Write-PortFail 'Excluded content is STILL in history after the clean. Do not publish.'; exit 1 }
|
|
122
|
+
Write-PortOk ("History cleaned ({0}): refs {1} -> {2}. Re-run Test-PortReadiness.ps1 before publishing." -f $Mode, $before.Count, $after.Count)
|
|
123
|
+
exit 0
|