@caddis/cli 0.4.21 → 0.4.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,218 @@
1
+ <#
2
+ .SYNOPSIS
3
+ Check a staged repository against the source-only content policy, over the tips AND all history.
4
+
5
+ .DESCRIPTION
6
+ The policy: a repository holds application code, configuration and templates, migration
7
+ SCRIPTS, tests, and the scripts/manifests that rebuild the environment. It does NOT hold:
8
+
9
+ runtime a runtime or server distribution (MariaDB, PHP, JDK, Node, Python, Apache...)
10
+ binary compiled or packaged files (.exe .dll .so .jar .msi, archives)
11
+ db-data database data files (ibdata1, *.ibd, *.frm, *.mdf, SQLite files, dumps)
12
+ data-export data files above the size threshold (.csv .xlsx .json .xml .sql ...), or any
13
+ data file under a backups/dumps/exports folder
14
+ dependency installed dependencies (node_modules, vendor, .venv, site-packages)
15
+ backup editor and copy backups (*.bak, *.orig, *.old, *~)
16
+
17
+ Large files that match no rule are reported as 'large-review' (a decision, not a violation).
18
+
19
+ .gitignore does not remove a file that was committed before it: every excluded path is
20
+ checked in ALL history, not only in the current files.
21
+
22
+ Reads <staging>\<Name>\mirror.git. Changes nothing. Writes content-policy.json next to it.
23
+ Exit 0: compliant. Exit 1: at least one excluded path exists somewhere in history.
24
+
25
+ Project overrides (optional) in the config file, key "contentPolicy":
26
+ { "keep": ["regex", ...], "exclude": ["regex", ...], "dataThresholdMB": 5, "largeMB": 10 }
27
+ Regexes match the repository path (forward slashes).
28
+ #>
29
+ [CmdletBinding()]
30
+ param(
31
+ [Parameter(Mandatory)][string]$Name,
32
+ [string]$StagingRoot,
33
+ [string]$ConfigPath,
34
+ [double]$DataThresholdMB,
35
+ [double]$LargeMB,
36
+ # Print every finding, not only the per-folder summary.
37
+ [switch]$Detail
38
+ )
39
+
40
+ $ErrorActionPreference = 'Stop'
41
+ . (Join-Path $PSScriptRoot 'PortCommon.ps1')
42
+
43
+ $config = Get-PortConfig $ConfigPath
44
+ $stage = Join-Path (Get-PortStagingRoot $config $StagingRoot) $Name
45
+ $mirror = Join-Path $stage 'mirror.git'
46
+ if (-not (Test-Path -LiteralPath $mirror)) { throw "No staged mirror at $mirror. Run Get-PortSource.ps1 first." }
47
+ $manifest = Get-Content -LiteralPath (Join-Path $stage 'manifest.json') -Raw | ConvertFrom-Json
48
+ $branch = $manifest.defaultBranch
49
+
50
+ $policy = Get-PortConfigValue $config 'contentPolicy' $null
51
+ $keepRx = @(); $extraExcludeRx = @()
52
+ if ($policy) {
53
+ if ($policy.PSObject.Properties.Name -contains 'keep') { $keepRx = @($policy.keep) }
54
+ if ($policy.PSObject.Properties.Name -contains 'exclude') { $extraExcludeRx = @($policy.exclude) }
55
+ if (-not $DataThresholdMB -and $policy.PSObject.Properties.Name -contains 'dataThresholdMB') { $DataThresholdMB = [double]$policy.dataThresholdMB }
56
+ if (-not $LargeMB -and $policy.PSObject.Properties.Name -contains 'largeMB') { $LargeMB = [double]$policy.largeMB }
57
+ }
58
+ if (-not $DataThresholdMB) { $DataThresholdMB = 5 }
59
+ if (-not $LargeMB) { $LargeMB = 10 }
60
+ $dataBytes = [int64]($DataThresholdMB * 1MB)
61
+ $largeBytes = [int64]($LargeMB * 1MB)
62
+
63
+ # A runtime folder is the runtime's NAME, optionally followed by a version or platform tag
64
+ # (php-8.3.33, jdk-17.0.2, node-v20.11.0-win-x64, apache24). Not any folder that merely STARTS
65
+ # with one: redis-migrations/ or apache-poi-wrapper/ are source.
66
+ $ver = '([-_ ]?v?\d[\w.]*(-(win|linux|x64|x86|amd64|arm64)[\w.-]*)?)?'
67
+ $runtimeDirRx = "(?i)^(mariadb|mysql|mysql-server|php|jdk|jre|openjdk|java|python|python-embed|nodejs|node|apache|httpd|nginx|tomcat|apache-tomcat|redis|postgres|postgresql|pgsql|erlang|ruby|dotnet|go)$ver$"
68
+ $runtimeBinRx = '(?i)^(mariadbd|mysqld|mariadb|mysql|php|php-cgi|node|python|pythonw|java|javaw|httpd|nginx|redis-server|postgres|pg_ctl|erl|ruby|dotnet)\.exe$'
69
+ $dependencyDirRx = '(?i)^(node_modules|vendor|\.venv|venv|site-packages|bower_components|\.yarn|jspm_packages)$'
70
+ $binaryExtRx = '(?i)\.(exe|dll|so|dylib|msi|msix|bin|iso|img|jar|war|ear|class|pyc|pdb|lib|a|o|obj|zip|7z|tar|gz|tgz|rar|bz2|xz|cab|nupkg|whl|deb|rpm)$'
71
+ $dbDataRx = '(?i)(^|/)(ibdata\d*|ib_logfile\d*|ib_buffer_pool|aria_log[\w.]*|undo\d+)$|\.(ibd|frm|myd|myi|mdf|ldf|ndf|sqlite3?|db3|dmp)$'
72
+ $backupRx = '(?i)(\.(bak|orig|old|swp)|~)$'
73
+ $dataExtRx = '(?i)\.(csv|tsv|xlsx|xlsm|xlsb|xls|ods|parquet|avro|orc|jsonl|ndjson|json|xml|sql|dat|txt|log)$'
74
+ $dataDirRx = '(?i)^(backups?|dumps?|exports?|snapshots?|data-?dumps?)$'
75
+
76
+ function Get-Category([string]$Path, [int64]$Bytes) {
77
+ foreach ($rx in $keepRx) { if ($Path -match $rx) { return $null } }
78
+ foreach ($rx in $extraExcludeRx) { if ($Path -match $rx) { return 'project-exclude' } }
79
+ $parts = $Path -split '/'
80
+ $dirs = if ($parts.Count -gt 1) { $parts[0..($parts.Count - 2)] } else { @() }
81
+ $leaf = $parts[-1]
82
+ foreach ($d in $dirs) { if ($d -match $runtimeDirRx) { return 'runtime' } }
83
+ if ($leaf -match $runtimeBinRx) { return 'runtime' }
84
+ foreach ($d in $dirs) { if ($d -match $dependencyDirRx) { return 'dependency' } }
85
+ if ($Path -match $dbDataRx) { return 'db-data' }
86
+ if ($leaf -match $backupRx) { return 'backup' }
87
+ if ($leaf -match $binaryExtRx) { return 'binary' }
88
+ if ($leaf -match $dataExtRx) {
89
+ # Under a backups/dumps folder any data file is excluded, but a plain note (.txt/.md) is not data.
90
+ if ($leaf -notmatch '(?i)\.(txt|md)$') { foreach ($d in $dirs) { if ($d -match $dataDirRx) { return 'data-export' } } }
91
+ # Migration SCRIPTS stay by policy. A large .sql under migration(s)/ may be a script or a
92
+ # data load, so it is a decision, not an automatic exclusion. Data FILES there (csv, json,
93
+ # xlsx) are still data.
94
+ if ($leaf -match '(?i)\.sql$' -and ($dirs | Where-Object { $_ -match '(?i)^migrations?$' })) {
95
+ if ($Bytes -ge $largeBytes -or $Bytes -ge $dataBytes) { return 'large-review' }
96
+ return $null
97
+ }
98
+ if ($Bytes -ge $dataBytes) { return 'data-export' }
99
+ }
100
+ if ($Bytes -ge $largeBytes) { return 'large-review' }
101
+ return $null
102
+ }
103
+
104
+ function MGit([string[]]$a) { Invoke-PortGit (@('-C', $mirror) + $a) }
105
+
106
+ # ---- every blob in history, with every path it ever had ----------------------------------
107
+ Write-PortStep 'Reading every path and blob in history'
108
+ $pathsBySha = @{}
109
+ $submodules = New-Object System.Collections.Generic.HashSet[string]
110
+ function Add-PathForSha([string]$sha, [string]$p) {
111
+ if (-not $pathsBySha.ContainsKey($sha)) { $pathsBySha[$sha] = New-Object System.Collections.Generic.HashSet[string] }
112
+ [void]$pathsBySha[$sha].Add($p)
113
+ }
114
+ # EVERY path a blob was ever written to. rev-list --objects names each blob only ONCE, by the
115
+ # first path it meets, so identical content at a second path was invisible (found by the
116
+ # dogfood: a 3-byte test image at two paths survived a clean). log --raw lists every add or
117
+ # change, per commit, including merges (-m) and root commits.
118
+ foreach ($l in (MGit @('-c', 'core.quotePath=false', 'log', '--all', '--raw', '--no-abbrev', '--no-renames', '-m', '--root', '--format=', '--diff-filter=AMT'))) {
119
+ if (-not $l -or $l[0] -ne ':') { continue }
120
+ $meta, $p = $l -split "`t", 2
121
+ $f = $meta -split ' '
122
+ if ($f.Count -lt 4 -or -not $p -or $f[3] -match '^0+$') { continue }
123
+ # Mode 160000 is a submodule: its "blob" is a commit in ANOTHER repository, invisible to a
124
+ # size scan. Record it, so a runtime vendored as a submodule cannot pass unnoticed.
125
+ if ($f[1] -eq '160000') { [void]$submodules.Add($p); continue }
126
+ Add-PathForSha $f[3] $p
127
+ }
128
+ # Safety net: anything reachable that the log did not show.
129
+ foreach ($l in (MGit @('rev-list', '--objects', '--all'))) {
130
+ $sha, $p = $l -split ' ', 2
131
+ if ($p -and -not $pathsBySha.ContainsKey($sha)) { Add-PathForSha $sha $p }
132
+ }
133
+ $sizeBySha = @{}
134
+ foreach ($l in (MGit @('cat-file', '--batch-all-objects', '--batch-check=%(objecttype) %(objectname) %(objectsize)'))) {
135
+ $t, $o, $s = $l -split ' '
136
+ if ($t -eq 'blob' -and $pathsBySha.ContainsKey($o)) { $sizeBySha[$o] = [int64]$s }
137
+ }
138
+
139
+ # Aggregate per path: largest version, number of versions.
140
+ $byPath = @{}
141
+ foreach ($sha in $sizeBySha.Keys) {
142
+ foreach ($p in $pathsBySha[$sha]) {
143
+ if (-not $byPath.ContainsKey($p)) { $byPath[$p] = [pscustomobject]@{ path = $p; maxBytes = [int64]0; versions = 0 } }
144
+ $e = $byPath[$p]; $e.versions++
145
+ if ($sizeBySha[$sha] -gt $e.maxBytes) { $e.maxBytes = $sizeBySha[$sha] }
146
+ }
147
+ }
148
+
149
+ $tipFiles = @{}
150
+ foreach ($l in (MGit @('-c', 'core.quotePath=false', 'ls-tree', '-r', '--name-only', $branch))) { $tipFiles[$l] = $true }
151
+
152
+ # ---- classify ----------------------------------------------------------------------------------
153
+ $findings = New-Object System.Collections.Generic.List[object]
154
+ foreach ($e in $byPath.Values) {
155
+ $cat = Get-Category $e.path $e.maxBytes
156
+ if (-not $cat) { continue }
157
+ $findings.Add([pscustomobject]@{
158
+ path = $e.path; category = $cat; maxBytes = $e.maxBytes; versions = $e.versions
159
+ inDefaultBranch = [bool]$tipFiles[$e.path]
160
+ action = if ($cat -eq 'large-review') { 'review' } else { 'exclude' }
161
+ })
162
+ }
163
+ foreach ($sm in $submodules) {
164
+ $findings.Add([pscustomobject]@{ path = $sm; category = 'submodule'; maxBytes = [int64]0; versions = 1; inDefaultBranch = [bool]$tipFiles[$sm]; action = 'review' })
165
+ }
166
+ $exclude = @($findings | Where-Object { $_.action -eq 'exclude' })
167
+ $review = @($findings | Where-Object { $_.action -eq 'review' })
168
+
169
+ # Group by top two path segments so a 3,000-file vendored runtime reads as one line.
170
+ # Group by folder, at most 4 levels deep, so one vendored runtime reads as one line and a
171
+ # data folder deep in an app is still named precisely.
172
+ function Get-Group([string]$p) { $s = $p -split '/'; if ($s.Count -le 1) { return $p }; $d = $s[0..([math]::Min(3, $s.Count - 2))]; ($d -join '/') + '/' }
173
+ $groups = @($findings | Group-Object { "$($_.category)|$(Get-Group $_.path)" } | ForEach-Object {
174
+ $cat, $grp = $_.Name -split '\|', 2
175
+ [pscustomobject]@{
176
+ category = $cat; group = $grp; files = $_.Count
177
+ historyMB = [math]::Round((($_.Group | Measure-Object maxBytes -Sum).Sum) / 1MB, 1)
178
+ inDefaultBranch = @($_.Group | Where-Object { $_.inDefaultBranch }).Count
179
+ }
180
+ } | Sort-Object historyMB -Descending)
181
+
182
+ $totalBytes = [int64]0; foreach ($v in $sizeBySha.Values) { $totalBytes += $v }
183
+ # Count each blob once, even when it sits at several excluded paths (a per-path sum can exceed
184
+ # the whole history). Plain loops: Measure-Object on NO input has no .Sum under strict mode.
185
+ $exPaths = New-Object System.Collections.Generic.HashSet[string]
186
+ foreach ($x in $exclude) { [void]$exPaths.Add($x.path) }
187
+ $exBytes = [int64]0
188
+ foreach ($sha in $sizeBySha.Keys) { foreach ($p in $pathsBySha[$sha]) { if ($exPaths.Contains($p)) { $exBytes += $sizeBySha[$sha]; break } } }
189
+ $report = [ordered]@{
190
+ name = $Name; defaultBranch = $branch; checkedAt = (Get-Date).ToString('o')
191
+ thresholds = [ordered]@{ dataThresholdMB = $DataThresholdMB; largeMB = $LargeMB }
192
+ historyBlobMB = [math]::Round($totalBytes / 1MB, 1)
193
+ excludedPaths = $exclude.Count; excludedMB = [math]::Round($exBytes / 1MB, 1)
194
+ reviewPaths = $review.Count
195
+ compliant = ($exclude.Count -eq 0)
196
+ groups = $groups
197
+ findings = @($findings | Sort-Object maxBytes -Descending)
198
+ }
199
+ ConvertTo-PortJsonFile $report (Join-Path $stage 'content-policy.json')
200
+ # Plain path list, one per line, for Invoke-PortHistoryClean.ps1 and for .gitignore review.
201
+ Set-Content -LiteralPath (Join-Path $stage 'content-policy-exclude.txt') -Value @($exclude | Sort-Object path | ForEach-Object { $_.path }) -Encoding UTF8
202
+
203
+ Write-Host ''
204
+ Write-Host ("Content policy for {0}: {1} excluded path(s), {2} MB of history; {3} path(s) to review. History holds {4} MB of blobs in total." -f $Name, $exclude.Count, $report.excludedMB, $review.Count, $report.historyBlobMB)
205
+ if ($groups.Count -gt 0) {
206
+ Write-Host ''
207
+ Write-Host ('{0,-13} {1,-55} {2,7} {3,10} {4}' -f 'category', 'where', 'files', 'MB', 'in default branch')
208
+ foreach ($g in $groups) { Write-Host ('{0,-13} {1,-55} {2,7} {3,10} {4}' -f $g.category, $g.group, $g.files, $g.historyMB, $g.inDefaultBranch) }
209
+ }
210
+ if ($Detail) { $findings | Sort-Object maxBytes -Descending | ForEach-Object { Write-Host (' {0,-13} {1,9:N0} KB {2}' -f $_.category, ($_.maxBytes / 1KB), $_.path) } }
211
+ Write-Host " report: $(Join-Path $stage 'content-policy.json')"
212
+ Write-Host " exclude list: $(Join-Path $stage 'content-policy-exclude.txt')"
213
+ if ($exclude.Count -gt 0) {
214
+ Write-PortWarn 'Not compliant. Decide per group: keep (add a "keep" regex to contentPolicy), or remove from history with Invoke-PortHistoryClean.ps1. Adding a path to .gitignore does NOT remove it from history.'
215
+ exit 1
216
+ }
217
+ Write-PortOk 'Compliant: no excluded content anywhere in history.'
218
+ exit 0
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "name": "caddis-extras",
3
- "version": "1.3.32",
3
+ "version": "1.3.33",
4
4
  "description": "caddis-extras — the long-tail skills for the Antigravity (agy) caddis plugin; install alongside caddis for the full set."
5
5
  }
@@ -11,23 +11,32 @@ Last Updated: 2026-09-19T14:05:00Z
11
11
  Last Model Used: gemini-3.8-flash
12
12
  ---
13
13
 
14
+ <!--
15
+ GOLD STANDARD for the hld skill. Match its structure, heading numbering, diagram types and
16
+ styling, ID schemes (SYS_nn, INT_nn, REQ-nn) and table columns exactly.
17
+ ACCOUNTS_PER_RUN, BATCH_HOST, PDF_TOTAL and SECONDARY_SCHEDULES stand in for real, measured
18
+ figures and names in the source HLD; they are withheld here only. A generated HLD must state
19
+ the REAL number or name in every such place - never a placeholder, never a vague 'many'.
20
+ Exact figures are what make the SLA and parity claims checkable.
21
+ -->
22
+
14
23
  # High-Level Architecture (HLA): BriteBill Replacement Platform
15
24
 
16
25
  ## 1. Executive Summary
17
26
 
18
27
  ### 1.1 Initiative Goal
19
- The BriteBill Replacement Platform replaces the proprietary third-party BriteBill invoice composition system used for Virgin Media Ireland fixed billing.
28
+ The BriteBill Replacement Platform replaces the proprietary third-party BriteBill invoice composition system used for fixed-line customer billing.
20
29
 
21
30
  This platform achieves the following primary business goals:
22
31
  - It eliminates third-party licensing costs, proprietary dependencies, and vendor software lock-in.
23
32
  - It provides a locally controlled, deterministic, and auditable composition engine.
24
33
  - It delivers high-throughput batch generation for four recurring monthly billing cycles: `BR-07`, `BR-14`, `BR-21`, and `BR-28`.
25
- - It processes approximately 124,000 customer accounts per billing run within a four-hour service level agreement (SLA) window.
34
+ - It processes approximately `<ACCOUNTS_PER_RUN>` customer accounts per billing run within a four-hour service level agreement (SLA) window.
26
35
  - It simultaneously produces four synchronized output channels from a single canonical document model: Customer PDF, Print Spool AFP, Mailroom Index JRN, and Digital Portal XML.
27
36
  - It guarantees full financial integrity through mandatory, fail-closed automated reconciliation gates.
28
37
 
29
38
  ### 1.2 Solution Architecture Summary
30
- The solution operates as a standalone Python-based batch processing engine deployed on host `iegbcoppoc02`. It processes raw Kenan billing stream files (`.EN`), executes financial reconciliation, and renders customer-facing outputs. The engine isolates every run in an immutable staging directory, verifies cryptographic checksums, and publishes outputs atomically only after all financial and population gates pass.
39
+ The solution operates as a standalone Python-based batch processing engine deployed on the dedicated batch host `<BATCH_HOST>`. It processes raw Kenan billing stream files (`.EN`), executes financial reconciliation, and renders customer-facing outputs. The engine isolates every run in an immutable staging directory, verifies cryptographic checksums, and publishes outputs atomically only after all financial and population gates pass.
31
40
 
32
41
  ### 1.3 Business Value & Operational Outcomes
33
42
  - **Cost Reduction**: Complete decommissioning of third-party composition licenses.
@@ -40,7 +49,7 @@ The solution operates as a standalone Python-based batch processing engine deplo
40
49
  ## 2. Project Context and Scope
41
50
 
42
51
  ### 2.1 Background
43
- Virgin Media Ireland fixed customer billing produces monthly invoices across four staggered billing cycles. The legacy composition workflow relied on an external vendor system (BriteBill). This introduced high annual licensing costs and operational delays whenever regulatory or template changes occurred. The BriteBill Replacement Platform moves invoice composition entirely onto internal infrastructure using high-performance open standards.
52
+ Fixed-line customer billing produces monthly invoices across four staggered billing cycles. The legacy composition workflow relied on an external vendor system (BriteBill). This introduced high annual licensing costs and operational delays whenever regulatory or template changes occurred. The BriteBill Replacement Platform moves invoice composition entirely onto internal infrastructure using high-performance open standards.
44
53
 
45
54
  ### 2.2 Scope
46
55
 
@@ -65,7 +74,7 @@ Virgin Media Ireland fixed customer billing produces monthly invoices across fou
65
74
 
66
75
  ### 2.3 Architecture Boundaries and Key Assumptions
67
76
  - **Deterministic Processing**: All calculations and document renderings produce identical, byte-reproducible outputs for identical inputs.
68
- - **Local Filesystem Isolation**: Network-isolated execution on `iegbcoppoc02` with zero runtime external API or HTTP dependencies.
77
+ - **Local Filesystem Isolation**: Network-isolated execution on `<BATCH_HOST>` with zero runtime external API or HTTP dependencies.
69
78
  - **Fail-Closed Operations**: Any financial discrepancy, unmapped record, or population count mismatch immediately halts the run and prevents output release.
70
79
 
71
80
  ---
@@ -168,7 +177,7 @@ sequenceDiagram
168
177
  Render->>Storage: Write outputs into staging directories
169
178
 
170
179
  Note over Orch,Targets: Stage 5: Population Audit & Atomic Release
171
- Orch->>Audit: Verify population counts (PDF == XML == AFP == 124,024)
180
+ Orch->>Audit: Verify population counts (PDF == XML == AFP == ACCOUNTS_PER_RUN)
172
181
  Orch->>Audit: Seal freeze-manifest.json and compute final SHA-256 digests
173
182
  Orch->>Audit: Transition to RunLifecycle::READY_FOR_RELEASE
174
183
  Orch->>Targets: Copy verified artifacts to delivery endpoints
@@ -195,7 +204,7 @@ flowchart TD
195
204
  SYS_01["SYS_01: Kenan Billing Engine<br/>Rating & Billing Lifecycle"]:::existingStyle
196
205
  end
197
206
 
198
- subgraph CorePlatform["BriteBill Replacement Platform (iegbcoppoc02)"]
207
+ subgraph CorePlatform["BriteBill Replacement Platform (BATCH_HOST)"]
199
208
  direction LR
200
209
  SYS_02["SYS_02: BriteBill Replacement Engine<br/>Ingestion, Mapping, Reconcile & Render"]:::newStyle
201
210
  SYS_06["SYS_06: Operations Control Centre<br/>Audit Journal, Sealed Manifest & CLI"]:::newStyle
@@ -204,7 +213,7 @@ flowchart TD
204
213
  subgraph Downstream["Downstream Distribution Channels"]
205
214
  direction LR
206
215
  SYS_03["SYS_03: Print House & Mailroom<br/>Physical Print & Insertion Spool"]:::existingStyle
207
- SYS_04["SYS_04: My Virgin Media Portal<br/>Online eBilling XML Viewer"]:::existingStyle
216
+ SYS_04["SYS_04: Customer Self-Service Portal<br/>Online eBilling XML Viewer"]:::existingStyle
208
217
  SYS_05["SYS_05: Enterprise Document Archive<br/>Customer PDF Storage & Email"]:::existingStyle
209
218
  end
210
219
 
@@ -225,7 +234,7 @@ The table below catalogs all systems and platforms within the solution boundary:
225
234
  | `SYS_01` | Kenan Billing Engine (Amdocs) | Billing / Rating | Upstream rating engine producing periodic `.EN` billing stream files. | No |
226
235
  | `SYS_02` | BriteBill Replacement Composition Engine | Batch Engine | Core batch orchestration, mapping, financial reconciliation, and multi-channel rendering engine. | Yes |
227
236
  | `SYS_03` | External Print House & Mailroom Spool | Print Production | Commercial printing facility ingesting MO:DCA AFP spools and JRN insertion records. | No |
228
- | `SYS_04` | My Virgin Media Digital Self-Service Portal | Web Portal | Customer billing portal ingesting structured eBilling XML for web presentation. | No |
237
+ | `SYS_04` | Customer Digital Self-Service Portal | Web Portal | Customer billing portal ingesting structured eBilling XML for web presentation. | No |
229
238
  | `SYS_05` | Enterprise Document Archive & Dispatcher | Document Repository | Long-term customer communications repository and email dispatcher ingesting certified PDF invoices. | No |
230
239
  | `SYS_06` | Operations Control Centre & Audit Journal | Ops & Governance | Operational monitoring CLI, telemetry aggregator, and sealed run manifest validator. | Yes |
231
240
 
@@ -238,7 +247,7 @@ The core processing component implemented in Python. It executes local staging,
238
247
  #### 4.2.3 SYS_03: External Print House & Mailroom Spool
239
248
  Receives binary MO:DCA AFP print spools and JRN insertion index files for postal sorting, physical paper printing, envelope inserting, and mail dispatch.
240
249
 
241
- #### 4.2.4 SYS_04: My Virgin Media Digital Self-Service Portal
250
+ #### 4.2.4 SYS_04: Customer Digital Self-Service Portal
242
251
  The customer-facing digital web application that parses structured XML invoice streams to present interactive online bill breakdowns.
243
252
 
244
253
  #### 4.2.5 SYS_05: Enterprise Document Archive & Dispatcher
@@ -273,7 +282,7 @@ Delivers ASCII index files specifying sheet counts, account identifiers, envelop
273
282
  Transfers schema-validated XML documents containing invoice data for online customer self-service viewing.
274
283
 
275
284
  #### 4.3.5 INT_05: Deterministic PDF Invoices
276
- Transfers byte-deterministic PDF documents containing vector Virgin Media branding, billing transaction tables, and Giro payment slips.
285
+ Transfers byte-deterministic PDF documents containing vector brand artwork, billing transaction tables, and Giro payment slips.
277
286
 
278
287
  #### 4.3.6 INT_06: Telemetry & Sealed Manifest
279
288
  Emits structured JSONL telemetry and a cryptographic `release_manifest.json` sealing file counts, execution parameters, and SHA-256 hashes.
@@ -295,7 +304,7 @@ The platform provides administrative command-line surfaces for operations and sc
295
304
  ## 5. High-Level Data and Storage Architecture
296
305
 
297
306
  ### 5.1 Host & Working Storage Topology
298
- The engine executes entirely within a dedicated local filesystem hierarchy on host `iegbcoppoc02`. Every batch run is isolated by unique run and attempt identifiers:
307
+ The engine executes entirely within a dedicated local filesystem hierarchy on the dedicated batch host `<BATCH_HOST>`. Every batch run is isolated by unique run and attempt identifiers:
299
308
 
300
309
  ```
301
310
  <working_root>/<run_id>/<attempt_id>/
@@ -356,8 +365,8 @@ classDiagram
356
365
 
357
366
  ### 5.3 Multi-Channel Output Volume & Specifications
358
367
  Production metrics established during candidate certification (ATT-006 baseline):
359
- - **PDF Documents**: 124,980 total PDF files (124,024 primary invoices + 956 secondary itemized schedules).
360
- - **XML Documents**: 124,024 structured customer eBilling files.
368
+ - **PDF Documents**: `<PDF_TOTAL>` total PDF files (`<ACCOUNTS_PER_RUN>` primary invoices + `<SECONDARY_SCHEDULES>` secondary itemized schedules).
369
+ - **XML Documents**: `<ACCOUNTS_PER_RUN>` structured customer eBilling files.
361
370
  - **AFP Spools**: 32 consolidated MO:DCA print files matching the 32 upstream Kenan input batches.
362
371
  - **JRN Journals**: 32 postal index files containing 100% field parity with postal insertion machinery.
363
372
 
@@ -373,7 +382,7 @@ The platform security architecture adheres to strict controls defined in `docs/S
373
382
 
374
383
  ### 6.2 Zero Runtime Network Access
375
384
  - No module within `ingestion/`, `mapping/`, `templating/`, or `rendering/` initiates external HTTP or network requests during generation.
376
- - All fonts, styling assets, and configuration templates reside locally on disk on `iegbcoppoc02`.
385
+ - All fonts, styling assets, and configuration templates reside locally on disk on `<BATCH_HOST>`.
377
386
 
378
387
  ### 6.3 Working Storage Path Isolation & Traversal Defense
379
388
  - File discovery resolves candidate paths and rejects any file whose resolved parent directory escapes the staging root.
@@ -395,7 +404,7 @@ The platform security architecture adheres to strict controls defined in `docs/S
395
404
  | `REQ-01` | Ingest Kenan `.EN` tagged-record streams | `SYS_02` (`ingestion/en_reader.py`) | Automated parser unit tests and zero-failure batch validation. |
396
405
  | `REQ-02` | Strong financial reconciliation | `SYS_02` (`reconciliation/checks.py`) | Full-population balance reconciliation gate (`reconcile()`). |
397
406
  | `REQ-03` | Byte-deterministic PDF rendering | `SYS_02` (`rendering/mupdf_renderer.py`) | SHA-256 byte-identity verification against certified baseline. |
398
- | `REQ-04` | Multi-channel parity (PDF/XML/AFP/JRN) | `SYS_02` (`mapping/*`, `rendering/*`) | Cross-output population count verification (`124,024` exact). |
407
+ | `REQ-04` | Multi-channel parity (PDF/XML/AFP/JRN) | `SYS_02` (`mapping/*`, `rendering/*`) | Cross-output population count verification (`<ACCOUNTS_PER_RUN>` exact). |
399
408
  | `REQ-05` | Atomic release & staging isolation | `SYS_02` (`batch/orchestrator.py`) | Zero output visibility before `READY_FOR_RELEASE` and marker written last. |
400
409
  | `REQ-06` | Auditability & run sealing | `SYS_06` (`audit/manifest.py`) | Cryptographic `release_manifest.json` generation. |
401
410
 
@@ -414,6 +423,6 @@ The platform security architecture adheres to strict controls defined in `docs/S
414
423
  | Fail-Closed | An architectural principle where any detected inconsistency aborts processing before releasing outputs. |
415
424
 
416
425
  ### 8.2 Reference Documents
417
- - `\\iegbcoppoc02\G\britebill-replacement\README.md`: System operational runbook and architecture overview.
418
- - `\\iegbcoppoc02\G\britebill-replacement\docs\SECURITY_AND_PRIVACY.md`: Security controls and privacy safeguards.
419
- - `\\iegbcoppoc02\G\britebill-replacement\docs\PRODUCTION_CANDIDATE_ATT006.md`: Candidate certification report and benchmark results.
426
+ - `britebill-replacement/README.md`: System operational runbook and architecture overview.
427
+ - `britebill-replacement/docs/SECURITY_AND_PRIVACY.md`: Security controls and privacy safeguards.
428
+ - `britebill-replacement/docs/PRODUCTION_CANDIDATE_ATT006.md`: Candidate certification report and benchmark results.
@@ -1,13 +1,13 @@
1
1
  ---
2
2
  name: port-to-github
3
- description: Move (port) an app or repository to GitHub from ANY source - a Gitea/GitLab/any git server URL, a local folder, a network (UNC) share, or a folder on a remote Windows box reached over WinRM - with or without existing git history. Keeps every branch and tag, verifies by SHA, translates Gitea/Forgejo Actions CI, switches deploy checkouts to GitHub with automatic rollback, and retires the old host safely. Use when the user says "move this app to GitHub", "port X to GitHub", "migrate from Gitea", "put this folder on GitHub", "this app only exists on the server", "import the repo from the prod box", or "cut the deploy over to GitHub".
3
+ description: Move (port) an app or repository to GitHub from ANY source - a Gitea/GitLab/any git server URL, a local folder, a network (UNC) share, or a folder on a remote Windows box reached over WinRM - with or without existing git history. Keeps every branch and tag, verifies by SHA, translates Gitea/Forgejo Actions CI, enforces a source-only content policy (runtimes, binaries, data exports and dependencies stay out of the repo, over ALL history, with a runtime manifest and provisioning script so the environment is still reproducible), switches deploy checkouts to GitHub with automatic rollback, and retires the old host safely. Use when the user says "move this app to GitHub", "port X to GitHub", "migrate from Gitea", "put this folder on GitHub", "this app only exists on the server", "import the repo from the prod box", "keep binaries / MariaDB / PHP out of the repo", "check the repo only holds source", or "cut the deploy over to GitHub".
4
4
  ---
5
5
 
6
6
  # port-to-github
7
7
 
8
8
  Move one app to GitHub without losing history, leaking a secret, or breaking its deploy.
9
9
 
10
- The work is in **six phases**. Each ends in a check that can FAIL. Do not start a phase until
10
+ The work is in **six phases**, plus the content policy (2b). Each ends in a check that can FAIL. Do not start a phase until
11
11
  the one before it passed. Scripts are in `scripts/` next to this file. They are PowerShell 7
12
12
  (`pwsh`); the parts that run on a remote box are Windows PowerShell 5.1 compatible.
13
13
 
@@ -15,6 +15,7 @@ the one before it passed. Scripts are in `scripts/` next to this file. They are
15
15
  |---|---|---|
16
16
  | 1. Stage the source | `Get-PortSource.ps1` | No. The source is only read. |
17
17
  | 2. Readiness | `Test-PortReadiness.ps1` | No. |
18
+ | 2b. Content policy | `Test-PortContentPolicy.ps1`, `New-PortRuntimeManifest.ps1`, `Invoke-PortHistoryClean.ps1` | No. Rewrites only the local stage. |
18
19
  | 3. Publish | `Publish-PortToGitHub.ps1` | **Yes: creates a GitHub repo and pushes.** |
19
20
  | 4. CI | `Convert-PortWorkflow.ps1`, `Test-PortWorkflowParses.ps1` | Yes: a feature branch and a PR. |
20
21
  | 5. Cutover | `Switch-CheckoutOrigin.ps1` | **Yes: changes where a deploy host pulls from.** |
@@ -64,6 +65,9 @@ Read the output. These need a decision, not a shrug:
64
65
  - **Uncommitted changes** in a source checkout are left out unless you pass
65
66
  `-IncludeUncommitted`, which puts them on a separate branch `port/uncommitted-snapshot-*`.
66
67
  The real branches stay exactly as committed. Ask the user which they want.
68
+ - **Default branch.** From a server or a bare repository, its HEAD. From a WORKING checkout,
69
+ `main` (or `master`) when it exists, because the checked-out branch is only what someone had
70
+ open; the script says when it made that choice. Override with `-DefaultBranch <name>`.
67
71
  - **Remote-only branches**: branches the checkout only knows as `origin/x`. If the checkout is
68
72
  the only copy, re-run with `-PromoteRemoteBranches origin`. If a server has them, port from it.
69
73
  - **Likely secrets** in a plain folder or a snapshot stop the script (exit 2). Nothing has left
@@ -93,6 +97,63 @@ deploy jobs included. Readiness lists those branches; publish refuses them unles
93
97
  (the new repo will expose them), CI credentials, dependency hosts, runner labels, files that
94
98
  still name the old host. The report is in `readiness.json`.
95
99
 
100
+ ## Phase 2b - content policy: source only
101
+
102
+ **The rule.** The repository holds application code, configuration and templates, migration
103
+ SCRIPTS, tests, and the scripts and manifests that rebuild the environment. Runtime and server
104
+ distributions (MariaDB, PHP, JDK, Node, Python, Apache...), binaries and archives, database data
105
+ files, data exports and installed dependencies stay OUT, and are installed or provisioned
106
+ separately. This is the default for every port; a project narrows or widens it with a
107
+ `contentPolicy` block in its config (`keep` / `exclude` regexes, `dataThresholdMB`, `largeMB`).
108
+
109
+ ```powershell
110
+ & $s/Test-PortContentPolicy.ps1 -Name <repo-name>
111
+ ```
112
+
113
+ It checks the default branch AND every blob in history, because **`.gitignore` does not remove a
114
+ file that was committed before it**. Exit 1 means excluded content exists somewhere in history.
115
+ The output groups findings by folder; `content-policy.json` has every path.
116
+
117
+ For each group, decide with the user:
118
+
119
+ 1. **It belongs** (a small fixture, a vendored library the team maintains) -> add a `keep` regex
120
+ to the project config and re-run.
121
+ 2. **It does not** -> remove it from the STAGE's history. Two ways, and it is the owner's call:
122
+ ```powershell
123
+ & $s/Invoke-PortHistoryClean.ps1 -Name <repo-name> -Mode Rewrite # keep history, drop the paths
124
+ & $s/Invoke-PortHistoryClean.ps1 -Name <repo-name> -Mode FreshStart # one commit of today's files
125
+ ```
126
+ `Rewrite` keeps every branch, tag and commit but CHANGES commit SHAs from the first affected
127
+ commit on; anything quoting an old SHA stops matching. `FreshStart` is smallest and drops all
128
+ history, other branches and tags. Both re-run the policy check and exit 1 if anything
129
+ excluded survived. The source keeps its original history either way.
130
+
131
+ **Keep the environment reproducible.** Excluding a runtime is only safe if the repo says how to
132
+ get it back. Scan the SOURCE FOLDER (runtimes usually live in an ignored `tools/`):
133
+
134
+ ```powershell
135
+ & $s/New-PortRuntimeManifest.ps1 -Path <source-folder> [-ComputerName <box>] -OutDir <clone>\environment
136
+ ```
137
+
138
+ It writes `runtimes.json` (kind, exact version, binary SHA-256, data folders), `provision.ps1`
139
+ (downloads and unpacks each runtime with a known public source and FAILS for any it cannot
140
+ provision), `README.md`, and `config-templates/`: configuration found inside runtime folders
141
+ (`my.ini`, `php.ini`, `config/`, `metadata/`). Configuration is app content even when the runtime
142
+ around it is not. Files that look like they hold a credential are withheld and listed; make
143
+ those templates by hand with placeholders. **The credential check is a heuristic: read every
144
+ copied template yourself before committing it.** The dogfood found one it missed (a SAML test
145
+ user written as `'name:password' => array(...)`); the rule now covers that form, but the next
146
+ form may be new. Nothing it finds is executed; versions come from file
147
+ metadata. Commit `environment/` on the phase 4 branch, by file name.
148
+
149
+ Two things it cannot decide, so you must:
150
+ - **UNCLASSIFIED folders** beside a runtime (a deployed web tree, a library with no binary).
151
+ Decide each: provision it, rebuild it from repo code, or keep it in the repo.
152
+ - **Data folders** (e.g. a database `data/` directory). This is state, not code. The repo carries
153
+ the schema and migrations; the data comes from a backup. Say where that backup lives.
154
+
155
+ Then add every excluded folder to `.gitignore` on the phase 4 branch, so it cannot come back.
156
+
96
157
  ## Phase 3 - publish (ask first)
97
158
 
98
159
  Tell the user: the repo name, owner, visibility, branch and tag counts. Then:
@@ -101,7 +162,7 @@ Tell the user: the repo name, owner, visibility, branch and tag counts. Then:
101
162
  & $s/Publish-PortToGitHub.ps1 -Name <repo-name> [-Owner <org>] [-Visibility private]
102
163
  ```
103
164
 
104
- It first checks the stage still holds exactly the refs phase 1 recorded (anything else would be
165
+ It refuses a stage whose content policy is not compliant (run phase 2b; `-AcceptContentPolicy` records an explicit owner exception instead). It then checks the stage still holds exactly the refs phase 1 recorded (anything else would be
105
166
  pushed and then "verified" against itself), creates the repo EMPTY, pushes `refs/heads/*` and
106
167
  `refs/tags/*` with an explicit refspec, sets the default branch, and compares every ref by SHA
107
168
  (exit 1 on any difference). Do not run other git commands inside the stage; if you did,
@@ -37,6 +37,9 @@ param(
37
37
  # What to do when a plain folder or an uncommitted snapshot contains a likely secret.
38
38
  [ValidateSet('Stop', 'Exclude')][string]$OnSecret = 'Stop',
39
39
  [string]$InitialBranch = 'main',
40
+ # The GitHub default branch. Default: the server's HEAD; for a working checkout, main/master
41
+ # when present (the checked-out branch is only what someone had open).
42
+ [string]$DefaultBranch,
40
43
  # Replace an existing staging folder for this Name.
41
44
  [switch]$Force
42
45
  )
@@ -315,6 +318,7 @@ switch ($kind) {
315
318
  }
316
319
  }
317
320
  $r.isRepo = $isRepo
321
+ $r.isBare = $isBare
318
322
  if ($isRepo) {
319
323
  $bundle = Join-Path $tmp 'repo.bundle'
320
324
  & $git -c safe.directory=* -C $Path bundle create $bundle --all 2>&1 | Out-Null
@@ -362,7 +366,13 @@ switch ($kind) {
362
366
  Invoke-PortGit @('bundle', 'verify', $localBundle) | Out-Null
363
367
  Invoke-PortGit @('clone', '-q', '--mirror', $localBundle, $mirror) | Out-Null
364
368
  $manifest.hadGitHistory = $true
365
- $manifest.sourceHead = if ($remote.head) { $remote.head } else { $remote.headSha }
369
+ # A bare repository's HEAD is authoritative, like a server's; only a WORKING checkout's
370
+ # HEAD is 'whatever was open'. Leaving sourceHead empty keeps the main-line preference off.
371
+ $manifest.sourceHead = if ($remote.isBare) { $null } elseif ($remote.head) { $remote.head } else { $remote.headSha }
372
+ # A bundle records HEAD as a commit, not a branch name, so a clone of it GUESSES the
373
+ # default when two branches share that commit. For a bare repo the name read on the
374
+ # far side is authoritative; carry it across.
375
+ if ($remote.isBare -and $remote.head) { $authHead = $remote.head }
366
376
  $manifest.dirtyFiles = @($remote.dirty)
367
377
  if ($manifest.dirtyFiles.Count -gt 0) {
368
378
  if ($IncludeUncommitted -and $remote.dirtyZip) {
@@ -430,11 +440,29 @@ if ($heads.Count -eq 0) { throw "The source produced no branches. Nothing to por
430
440
 
431
441
  $symHead = Invoke-PortGit -AllowFailure @('-C', $mirror, 'symbolic-ref', '--short', 'HEAD')
432
442
  $def = if ($symHead.ExitCode -eq 0) { $symHead.Output -join '' } else { $null }
443
+ $mainLine = @('main', 'master') | Where-Object { $headNames -contains $_ } | Select-Object -First 1
444
+ if (-not (Get-Variable -Name authHead -ErrorAction SilentlyContinue)) { $authHead = $null }
445
+ # The branch the source checkout was really on (a bundle's HEAD is only a guess), so the
446
+ # main-line choice below can say what it replaced.
447
+ if ($manifest.sourceHead -and ($headNames -contains $manifest.sourceHead)) { $def = $manifest.sourceHead }
448
+ if ($DefaultBranch) {
449
+ if (-not ($headNames -contains $DefaultBranch)) { throw "-DefaultBranch '$DefaultBranch' is not a branch in the source." }
450
+ $def = $DefaultBranch
451
+ }
452
+ elseif ($authHead -and ($headNames -contains $authHead)) {
453
+ $def = $authHead
454
+ }
455
+ elseif ($kind -ne 'url' -and $manifest.sourceHead -and $mainLine -and $def -ne $mainLine) {
456
+ # A server's HEAD IS its default branch. A working checkout's HEAD is only whatever branch
457
+ # someone had open, so prefer the main line when there is one, and say so.
458
+ $manifest.warnings += "The checkout was on '$def', but '$mainLine' exists, so '$mainLine' is the default branch on GitHub. Pass -DefaultBranch to choose another."
459
+ $def = $mainLine
460
+ }
433
461
  if (-not $def -or -not ($headNames -contains $def)) {
434
462
  $def = @('main', 'master', 'develop') | Where-Object { $headNames -contains $_ } | Select-Object -First 1
435
463
  if (-not $def) { $def = $headNames[0] }
436
- Invoke-PortGit @('-C', $mirror, 'symbolic-ref', 'HEAD', "refs/heads/$def") | Out-Null
437
464
  }
465
+ Invoke-PortGit @('-C', $mirror, 'symbolic-ref', 'HEAD', "refs/heads/$def") | Out-Null
438
466
  $manifest.defaultBranch = $def
439
467
 
440
468
  if ($manifest.remoteOnlyBranches.Count -gt 0) {