@caddis/cli 0.4.18 → 0.4.20

package/bundles/antigravity-plugin/skills/port-to-github/scripts/Test-PortReadiness.ps1

@@ -0,0 +1,218 @@
+<#
+.SYNOPSIS
+  Inspect a staged repository and list what will break or leak when it moves to GitHub.
+
+.DESCRIPTION
+  Reads <staging>\<Name>\mirror.git (written by Get-PortSource.ps1). Changes nothing.
+  Writes <staging>\<Name>\readiness.json and prints a report.
+
+  BLOCKERS (exit 1): a file over 100 MB anywhere in history, Git LFS content, a workflow
+  containing an empty expression, a likely live credential in the current tree.
+  WARNINGS: everything a human must decide: CI credentials, dependency hosts, runner labels,
+  hardcoded references to the old host, secret-looking files in past history.
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory)][string]$Name,
+    [string]$StagingRoot,
+    [string]$ConfigPath
+)
+
+$ErrorActionPreference = 'Stop'
+. (Join-Path $PSScriptRoot 'PortCommon.ps1')
+
+$config = Get-PortConfig $ConfigPath
+$stage = Join-Path (Get-PortStagingRoot $config $StagingRoot) $Name
+$mirror = Join-Path $stage 'mirror.git'
+if (-not (Test-Path -LiteralPath $mirror)) { throw "No staged mirror at $mirror. Run Get-PortSource.ps1 first." }
+$manifest = Get-Content -LiteralPath (Join-Path $stage 'manifest.json') -Raw | ConvertFrom-Json
+$branch = $manifest.defaultBranch
+
+$blockers = New-Object System.Collections.Generic.List[string]
+$warnings = New-Object System.Collections.Generic.List[string]
+$report = [ordered]@{ name = $Name; defaultBranch = $branch; checkedAt = (Get-Date).ToString('o') }
+
+function MGit([string[]]$a) { Invoke-PortGit (@('-C', $mirror) + $a) }
+function MGitTry([string[]]$a) { Invoke-PortGit -AllowFailure (@('-C', $mirror) + $a) }
+
+# ---- 1. Size: GitHub refuses any blob over 100 MB, in ANY commit, not just the tip ----------
+Write-PortStep 'Checking object sizes across all history'
+$big = @()
+foreach ($line in (MGit @('cat-file', '--batch-all-objects', '--batch-check=%(objecttype) %(objectsize) %(objectname)'))) {
+    $t, $s, $o = $line -split ' '
+    if ($t -eq 'blob' -and [int64]$s -gt 50MB) { $big += [pscustomobject]@{ sha = $o; bytes = [int64]$s } }
+}
+if ($big.Count -gt 0) {
+    $paths = @{}
+    foreach ($l in (MGit @('rev-list', '--objects', '--all'))) {
+        $sha, $p = $l -split ' ', 2
+        if ($p -and ($big.sha -contains $sha)) { $paths[$sha] = $p }
+    }
+    foreach ($b in $big) {
+        $p = $paths[$b.sha]
+        $mb = [math]::Round($b.bytes / 1MB, 1)
+        if ($b.bytes -gt 100MB) { $blockers.Add("File over 100 MB in history: $p ($mb MB). GitHub rejects the push. Rewrite history (git filter-repo) or move it to LFS/releases first.") }
+        else { $warnings.Add("Large file in history: $p ($mb MB). GitHub warns above 50 MB.") }
+    }
+}
+$report.largeBlobs = $big.Count
+
+# ---- 2. LFS ---------------------------------------------------------------------------------
+$attrs = MGitTry @('show', "${branch}:.gitattributes")
+if ($attrs.ExitCode -eq 0 -and (($attrs.Output -join "`n") -match 'filter=lfs')) {
+    $blockers.Add('The repository uses Git LFS. LFS objects are not in a git mirror; push them separately (git lfs fetch --all from the source, git lfs push --all to GitHub) before declaring the port done.')
+}
+
+# ---- 3. Secrets ---------------------------------------------------------------------------------
+Write-PortStep 'Scanning for credentials'
+$liveSecretPatterns = @('gh[pousr]_[A-Za-z0-9]{36,}', 'github_pat_[A-Za-z0-9_]{40,}', 'AKIA[0-9A-Z]{16}', '-----BEGIN [A-Z ]*PRIVATE KEY-----', 'xox[baprs]-[A-Za-z0-9-]{10,}')
+$grepArgs = @('grep', '-I', '-l', '-E')
+$liveHits = @()
+foreach ($p in $liveSecretPatterns) {
+    $g = MGitTry ($grepArgs + @('-e', $p, $branch))
+    if ($g.ExitCode -eq 0) { $liveHits += $g.Output }
+}
+foreach ($h in ($liveHits | Sort-Object -Unique)) { $blockers.Add("Likely live credential in the current tree: $h. Remove it and ROTATE it; history still holds it.") }
+
+# The same patterns across ALL history. A token committed once and deleted later is gone from
+# the tip but still readable by anyone who can clone the new repo. -G matches commits whose
+# diff adds or removes a matching line (extended regex).
+$histPattern = $liveSecretPatterns -join '|'
+$histOut = @(MGit @('log', '--all', "-G$histPattern", '--format=@@%h', '--name-only'))
+$cur = $null
+$histHits = @{}
+foreach ($l in $histOut) {
+    if (-not $l) { continue }
+    if ($l.StartsWith('@@')) { $cur = $l.Substring(2); continue }
+    if (-not $histHits.ContainsKey($l)) { $histHits[$l] = @() }
+    $histHits[$l] += $cur
+}
+foreach ($file in ($histHits.Keys | Sort-Object)) {
+    $tipName = "${branch}:$file"
+    if ($liveHits -contains $tipName) { continue }   # already reported as a current-tree blocker
+    $blockers.Add("Likely credential in PAST history: $file (commit(s) $(($histHits[$file] | Select-Object -Unique) -join ', ')). It is gone from the tip but ships with the history. ROTATE it, then either accept that or rewrite history (git filter-repo) before publishing.")
+}
+$report.historySecretFiles = @($histHits.Keys)
+
+$pwHits = MGitTry @('grep', '-I', '-l', '-i', '-E', '-e', '(password|passwd|pwd|api[_-]?key|secret)[[:space:]]*[:=][[:space:]]*[''"][^''" ]{8,}[''"]', $branch)
+if ($pwHits.ExitCode -eq 0) {
+    foreach ($h in $pwHits.Output) { $warnings.Add("Possible hardcoded password or key: $h. Check it is a placeholder or test value.") }
+}
+
+$treeFiles = @(MGit @('ls-tree', '-r', '--name-only', $branch))
+foreach ($f in $treeFiles) {
+    if (Test-PortSecretName (Split-Path $f -Leaf)) { $warnings.Add("Secret-type file tracked in the current tree: $f. Check it holds no real value.") }
+}
+$histNames = @(MGit @('log', '--all', '--format=', '--name-only') | Where-Object { $_ } | Sort-Object -Unique)
+$histSecret = @($histNames | Where-Object { (Test-PortSecretName (Split-Path $_ -Leaf)) -and ($treeFiles -notcontains $_) })
+foreach ($f in $histSecret) { $warnings.Add("Secret-type file existed in PAST history: $f. It will be public to anyone who can read the new repo. Rotate what it held, or rewrite history first.") }
+
+# ---- 4. CI ----------------------------------------------------------------------------------
+Write-PortStep 'Inventorying CI'
+$ciFiles = @($treeFiles | Where-Object {
+        $_ -match '^\.(gitea|github|forgejo)/workflows/[^/]+\.ya?ml$' -or
+        $_ -in @('.gitlab-ci.yml', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml', '.drone.yml', '.woodpecker.yml')
+    })
+$report.ciFiles = $ciFiles
+$workflows = @()
+foreach ($f in $ciFiles) {
+    $text = (MGit @('show', "${branch}:$f")) -join "`n"
+    $lines = $text -split "`n"
+    $wf = [ordered]@{ file = $f; emptyExpressions = @(); credentialLines = @(); runsOn = @(); hosts = @(); dispatchOnly = $false }
+    for ($i = 0; $i -lt $lines.Count; $i++) {
+        $ln = $lines[$i]
+        if ($ln -match '\$\{\{\s*\}\}') { $wf.emptyExpressions += ($i + 1) }
+        if ($ln -match 'secrets\.|github\.token|(?i)token\s*:|_TOKEN\b|_USER\b|PASSWORD') { $wf.credentialLines += ("{0}: {1}" -f ($i + 1), $ln.Trim()) }
+        if ($ln -match '^\s*runs-on:\s*(.+)$') { $wf.runsOn += $Matches[1].Trim() }
+        foreach ($m in [regex]::Matches($ln, 'https?://([A-Za-z0-9.-]+(:\d+)?)')) { $wf.hosts += $m.Groups[1].Value }
+    }
+    $wf.runsOn = @($wf.runsOn | Sort-Object -Unique)
+    $wf.hosts = @($wf.hosts | Where-Object { $_ -notmatch '^(github\.com|api\.github\.com|localhost|127\.0\.0\.1)' } | Sort-Object -Unique)
+    if ($text -match '(?m)^on:\s*(\[\s*)?workflow_dispatch' -and $text -notmatch '(?m)^\s+(push|pull_request)\s*:') { $wf.dispatchOnly = $true }
+    if ($wf.emptyExpressions.Count -gt 0) {
+        $msg = "$f has an empty `${{ }} on line(s) $($wf.emptyExpressions -join ', '). GitHub rejects the WHOLE file (startup_failure, zero jobs), even inside a comment in a run: block."
+        # GitHub parses only .github/workflows. A .gitea/.forgejo file is inert on GitHub until
+        # phase 4 copies it, so it is a warning here and a blocker there.
+        if ($f -like '.github/*') { $blockers.Add($msg) } else { $warnings.Add("$msg Fix it when translating (phase 4).") }
+    }
+    if ($wf.dispatchOnly) { $warnings.Add("$f triggers only on workflow_dispatch. If this is a FROZEN copy, generate the GitHub workflow from the last live version in history instead.") }
+    if ($wf.hosts.Count -gt 0) { $warnings.Add("$f references non-GitHub host(s): $($wf.hosts -join ', '). Each needs a credential that works FROM GitHub's runners, or the step breaks.") }
+    $workflows += [pscustomobject]$wf
+}
+$report.workflows = $workflows
+# Workflows already under .github/workflows RUN on GitHub the moment their branch is pushed.
+# Check every branch, not only the default one: a push of any branch fires its own workflows.
+$liveOnPush = @()
+foreach ($h in @(MGit @('for-each-ref', '--format=%(refname:short)', 'refs/heads'))) {
+    $ls = MGitTry @('ls-tree', '--name-only', "${h}:.github/workflows")
+    if ($ls.ExitCode -eq 0 -and ($ls.Output -join '')) { $liveOnPush += $h }
+}
+$report.branchesWithGithubWorkflows = $liveOnPush
+if ($liveOnPush.Count -gt 0) {
+    $warnings.Add("$($liveOnPush.Count) branch(es) already contain .github/workflows ($($liveOnPush -join ', ')). Publishing PUSHES them, which fires every push-triggered workflow - including any deploy job. Publish refuses unless you pass -AllowWorkflowRuns after reading those workflows.")
+}
+
+$otherCi = @($ciFiles | Where-Object { $_ -notmatch '^\.(gitea|github|forgejo)/workflows/' })
+if ($otherCi.Count -gt 0) {
+    $warnings.Add("Non-Actions CI found ($($otherCi -join ', ')). It will not run on GitHub; it must be rewritten as a GitHub Actions workflow.")
+}
+if ($ciFiles.Count -eq 0) { $warnings.Add('No CI found. After the port nothing builds, tests or deploys this app automatically.') }
+
+# ---- 5. Dependencies that live on another host --------------------------------------------------
+Write-PortStep 'Checking dependency pins'
+$depHosts = @()
+foreach ($f in ($treeFiles | Where-Object { $_ -match '(^|/)(pyproject\.toml|requirements[^/]*\.txt|setup\.cfg|package\.json|\.npmrc|\.pypirc|pip\.conf|go\.mod|Cargo\.toml)$' })) {
+    $text = (MGit @('show', "${branch}:$f")) -join "`n"
+    foreach ($m in [regex]::Matches($text, '(git\+)?(https?|ssh)://(?:[^@/\s]+@)?([A-Za-z0-9.-]+(:\d+)?)/')) {
+        $h = $m.Groups[3].Value
+        if ($h -match '^(pypi\.org|files\.pythonhosted\.org|registry\.npmjs\.org|github\.com)$') {
+            if ($h -eq 'github.com' -and $m.Groups[1].Value) { $depHosts += [pscustomobject]@{ file = $f; host = $h } }
+            continue
+        }
+        $depHosts += [pscustomobject]@{ file = $f; host = $h }
+    }
+    if ($text -match '(?m)^\s*registry\s*=') { $warnings.Add("$f sets a custom package registry. CI on GitHub needs a credential for it, or the install fails.") }
+}
+$report.dependencyHosts = $depHosts
+foreach ($h in ($depHosts | Group-Object host)) {
+    $warnings.Add("Dependencies pinned from $($h.Name) in: $((($h.Group.file) | Sort-Object -Unique) -join ', '). CI and every deploy host need a read credential for THAT host, chosen per host.")
+}
+
+# ---- 6. References to the old host --------------------------------------------------------------
+$oldHost = $null
+if ($manifest.sourceKind -eq 'url') {
+    if ($manifest.source -match '^[a-z]+://(?:[^@/]+@)?(?<h>[^/:]+)') { $oldHost = $Matches.h }
+    elseif ($manifest.source -match '^[^@\s]+@(?<h>[^:\s]+):') { $oldHost = $Matches.h }   # scp form: git@host:org/repo.git
+}
+if ($oldHost) {
+    $ref = MGitTry @('grep', '-I', '-l', '-F', '-e', $oldHost, $branch)
+    if ($ref.ExitCode -eq 0) {
+        $files = @($ref.Output | ForEach-Object { $_ -replace "^${branch}:", '' })
+        $report.oldHostReferences = $files
+        $warnings.Add("$($files.Count) file(s) still name the old host $oldHost ($($files[0..([math]::Min(4, $files.Count - 1))] -join ', ')$(if ($files.Count -gt 5) { ', ...' })). Decide per file: a dependency pin (keep, needs a credential) or an origin reference (change).")
+    }
+}
+
+# ---- 7. Carry forward what the source step found -----------------------------------------------
+foreach ($w in $manifest.warnings) { $warnings.Add($w) }
+
+$report.blockers = @($blockers)
+$report.warnings = @($warnings)
+$report.ready = ($blockers.Count -eq 0)
+ConvertTo-PortJsonFile $report (Join-Path $stage 'readiness.json')
+
+Write-Host ''
+Write-Host "Readiness for $Name (branch $branch): $($blockers.Count) blocker(s), $($warnings.Count) warning(s)"
+foreach ($b in $blockers) { Write-PortFail $b }
+foreach ($w in $warnings) { Write-PortWarn $w }
+foreach ($wf in $workflows) {
+    if ($wf.credentialLines.Count -gt 0) {
+        Write-Host ''
+        Write-Host "Credential lines in $($wf.file) - classify EACH as ORIGIN (becomes github.token) or DEPENDENCY (keeps a token for that host):"
+        $wf.credentialLines | ForEach-Object { Write-Host "    $_" }
+    }
+    if ($wf.runsOn.Count -gt 0) { Write-Host "Runner labels in $($wf.file): $($wf.runsOn -join ' | ') - confirm GitHub has runners with these labels." }
+}
+Write-Host "    report: $(Join-Path $stage 'readiness.json')"
+if ($blockers.Count -gt 0) { exit 1 }
+exit 0

package/bundles/antigravity-plugin/skills/port-to-github/scripts/Test-PortWorkflowParses.ps1

@@ -0,0 +1,62 @@
+<#
+.SYNOPSIS
+  After pushing a feature branch, prove GitHub could PARSE its workflows.
+
+.DESCRIPTION
+  GitHub cannot evaluate the triggers of a workflow file it cannot parse, so an invalid file
+  produces a failed run with ZERO jobs ("startup_failure") on every push, even on a branch its
+  triggers would never match. A valid file whose triggers do not match produces no run at all.
+
+  So, for the pushed commit:
+    a run with conclusion startup_failure, or failure with zero jobs  -> INVALID (exit 1)
+    no run, or runs with real jobs                                     -> parsed (exit 0)
+
+  Waits -WaitSeconds for GitHub to react, because an invalid file's run appears within seconds
+  and "no run" is only meaningful after that window.
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory)][string]$Owner,
+    [Parameter(Mandatory)][string]$Repo,
+    [Parameter(Mandatory)][string]$Branch,
+    [string]$Sha,
+    [int]$WaitSeconds = 60
+)
+
+$ErrorActionPreference = 'Stop'
+. (Join-Path $PSScriptRoot 'PortCommon.ps1')
+$full = "$Owner/$Repo"
+
+if (-not $Sha) {
+    $Sha = (& gh api "repos/$full/commits/$Branch" --jq .sha 2>&1)
+    if ($LASTEXITCODE -ne 0) { throw "Branch $Branch not found on ${full}: $Sha" }
+}
+Write-PortStep "Watching $full @ $($Sha.Substring(0,12)) on $Branch for $WaitSeconds s"
+
+$deadline = (Get-Date).AddSeconds($WaitSeconds)
+$invalid = @()
+$runs = @()
+do {
+    Start-Sleep -Seconds 10
+    $json = & gh api "repos/$full/actions/runs?head_sha=$Sha&per_page=50" 2>&1
+    if ($LASTEXITCODE -ne 0) { throw "Cannot list runs: $json" }
+    $runs = @(($json | ConvertFrom-Json).workflow_runs)
+    $invalid = @()
+    foreach ($r in $runs) {
+        if ($r.conclusion -eq 'startup_failure') { $invalid += $r; continue }
+        if ($r.status -eq 'completed' -and $r.conclusion -eq 'failure') {
+            $jobs = (& gh api "repos/$full/actions/runs/$($r.id)/jobs" --jq .total_count 2>$null)
+            if ("$jobs" -eq '0') { $invalid += $r }
+        }
+    }
+    if ($invalid.Count -gt 0) { break }
+} while ((Get-Date) -lt $deadline)
+
+if ($invalid.Count -gt 0) {
+    foreach ($r in $invalid) { Write-PortFail "INVALID workflow: '$($r.name)' ($($r.path)) - $($r.html_url)" }
+    Write-Host '    GitHub gives no line number. Look first for an empty ${{ }}; then bisect by removing jobs from the END (removing a middle job breaks needs: and fails the same way).'
+    exit 1
+}
+if ($runs.Count -eq 0) { Write-PortOk "No run for this commit: every workflow parsed and none matched a branch push. This is the expected result for main-only triggers." }
+else { foreach ($r in $runs) { Write-PortOk "Run '$($r.name)' has real jobs ($($r.status)/$($r.conclusion)): the file parsed. $($r.html_url)" } }
+exit 0

package/bundles/antigravity-plugin-extras/plugin.json

@@ -1,5 +1,5 @@
 {
   "name": "caddis-extras",
-  "version": "1.3.29",
+  "version": "1.3.31",
   "description": "caddis-extras — the long-tail skills for the Antigravity (agy) caddis plugin; install alongside caddis for the full set."
 }

package/bundles/antigravity-plugin-extras/skills/_registry.md

@@ -59,8 +59,11 @@
 | Skill | Path | When to Use |
 |-------|------|-------------|
 | Architecture Document | `architecture-document/` | Generate professional enterprise-grade HLD and LLD documents from the living Architecture.md. Outputs DOCX (editable) or PPTX (executive summary) using existing document skills. |
+| Data Lineage | `data-lineage/` | Generates End-to-End Data Lineage and Telemetry Specifications linking UI pages, API routes, database datasets, and ETL/SP producers for watch-sight observability. |
 | Doc Coauthoring | `doc-coauthoring/` | Guide users through a structured workflow for co-authoring documentation. Use when user wants to write documentation, proposals, technical specs, decision docs, or similar structured content. This workflow helps users efficiently transfer context, refine content through iteration, and verify the doc works for readers. Trigger when user mentions writing docs, creating proposals, drafting specs, or similar documentation tasks. |
 | Documentation Analyzer | `documentation-analyzer/` | Analyze codebases, explain code functionality, and generate comprehensive documentation. Use when documenting a project, creating README files, or understanding complex code. |
+| Hld | `hld/` | Generates High-Level Design (HLD) Markdown documentation and converts to Confluence Storage Format XHTML with native PlantUML diagrams and auto-generated legends. |
+| Lld | `lld/` | Generates Lower-Level Design (LLD) Markdown documentation and converts to Confluence Storage Format XHTML with native PlantUML diagrams and auto-generated legends. |
 | Naming Analyzer | `naming-analyzer/` | Suggest better variable, function, and class names based on context and conventions. |
 | Prd To Code | `prd-to-code/` | Transform PRD documents into working application code using a systematic 6-phase methodology. Use when starting projects from requirements or converting specs to implementation. |
 

package/bundles/antigravity-plugin-extras/skills/data-lineage/SKILL.md

@@ -0,0 +1,158 @@
+---
+name: data-lineage
+description: Generates End-to-End Data Lineage and Telemetry Specifications linking UI pages, API routes, database datasets, and ETL/SP producers for watch-sight observability.
+---
+
+# Data Lineage Skill
+
+This skill generates End-to-End Data Lineage and Telemetry Specifications for enterprise applications. It bridges the gap between frontend user surfaces, backend API routes, relational database objects, and upstream data producers (ETLs, Stored Procedures, and SQL Agent jobs).
+
+The resulting specifications serve as the ground truth for estate-wide observability in **watch-sight** to enable automated dependency graphs, health probing, and blast-radius impact analysis.
+
+---
+
+## Mandatory Workflow Steps
+
+### Step 1: Codebase & Infrastructure Discovery
+Conduct targeted research across the target application and observability repositories:
+1. **Frontend Surfaces:** Inspect `frontend/src/routes/` or router configurations to identify all user-facing pages, views, and navigation tabs.
+2. **API Routes:** Inspect `src/api/routes/` or API routers to map every HTTP endpoint invoked by each frontend page.
+3. **Database Repositories & SQL Queries:** Trace API controllers into data services and repositories to identify exact SQL tables, views, and columns accessed.
+4. **Data Producers & Execution Timings (DB Server SPs & ETLs):** Identify the scheduled jobs, batch Python scripts, and SQL Server Stored Procedures that populate, calculate, or refresh each dataset. Discover their exact execution timings, run frequencies, and triggers (e.g. SQL Server Agent schedules, Windows Task Scheduler jobs, cron expressions, email triggers, or polling loops from runbooks and pipeline schedule references).
+5. **Physical Estate Host Resolution:** Query `E:\Projects\watch-sight\config\monitor-targets.json` and `docs/runbooks/estate-pipelines/` to resolve physical server hostnames, IP addresses, ports, database instances, and service accounts.
+
+### Step 2: Markdown Document Authoring
+Draft the canonical Data Lineage Markdown file:
+`docs/<app>/<app>-data-lineage.md`
+
+Follow the standard 5-section lineage structure detailed below.
+
+### Step 3: Confluence XHTML Compilation & Verification
+Compile the Markdown document into Confluence Storage Format XHTML using:
+```powershell
+python scripts/convert_doc_to_confluence.py docs/<app>/<app>-data-lineage.md --output docs/<app>/<app>-data-lineage.html --diagram-type plantuml
+```
+Verify that the output compiles as well-formed XML without errors.
+
+---
+
+## Standard Lineage Document Structure (Markdown)
+
+Every lineage document must follow this standardized 5-section structure:
+
+```markdown
+---
+type: lineage
+status: current
+feature: <app>-data-lineage
+creation-agent: caddis
+Creating Model: <active model ID, e.g. gemini-3.8-flash>
+---
+
+# <App Title> — End-to-End Data Lineage & Telemetry Specification
+
+## 1. Physical Estate & Host Infrastructure
+Detail the physical hosts, IP addresses, ports, database instances, and network access paths:
+- Application Host (Web server, reverse proxy, ports)
+- Primary Database Host (SQL Server, database instance, port, auth mechanism)
+- Ingestion & Staging Host (Batch runner, scheduled jobs, agent services)
+- External Feeds & Gateways (SFTP drops, upstream BSS/mediation systems)
+
+## 2. End-to-End Dependency & Impact Flowchart
+Provide a colorful, professional Mermaid diagram (`flowchart TD` or `flowchart LR`) illustrating the 5-tier dependency pipeline:
+Upstream Ingestion -> Processing / DB Stored Procedures -> Database Tables/Views -> API Routes -> Frontend UI Pages.
+- Use explicit classDef pastel styles (never raw yellow boxes).
+- Group layers into clear subgraphs.
+
+## 3. Comprehensive Database Lineage Matrix
+A comprehensive table specifying the complete end-to-end data lineage across the following mandatory columns:
+| App | UI Page | API Route | Dataset (Tables / Views) | Producer (ETL / SP / Job) | Execution Timing / Trigger | Database | Physical Host |
+|---|---|---|---|---|---|---|---|
+| ... | ... | ... | ... | ... | ... | ... | ... |
+
+Column specifications:
+- **App:** Application identifier (e.g. `rev-sight`).
+- **UI Page:** User interface page path and title (e.g. `/runs (Run Monitor)`).
+- **API Route:** HTTP method and endpoint (e.g. `GET /api/runs`).
+- **Dataset:** Schema-qualified tables or views read (e.g. `[dbo].[CDR_RUN_LOG]`).
+- **Producer:** Exact SQL Agent job name, batch ETL script, or DB Stored Procedure that populates or refreshes the dataset.
+- **Execution Timing / Trigger:** Exact schedule, frequency, and trigger mechanism (e.g. `Daily 07:00 UTC (SQL Agent)`, `Nightly 08:00 UTC (Task Scheduler)`, `Every 5 min polling (Background Service)`).
+- **Database:** Target database name (e.g. `CDR`, `CERILLION`).
+- **Physical Host:** Hostname and IP (e.g. `iegew3ccdr01 (100.126.185.4)`).
+
+## 4. Stored Procedures & Producer Execution Catalog
+Catalog the database server stored procedures and scheduled jobs that populate the datamart:
+- Sequential execution order reference table detailing every job in execution order, execution time, host machine, scheduler, and dependencies.
+- Detailed procedure profiles: Procedure Name, database instance, trigger mechanism, exact schedule, parameters, source tables read, and target tables written.
+
+## 5. watch-sight Telemetry Ingestion Contract
+Provide a machine-readable JSON specification block aligned with `watch-sight/config/monitor-targets.json` implementing the **Three-Tier Real-Time Schedule Discovery Hierarchy**:
+- **All Producers Represented:** Every single producer (ETL script, SQL Agent job, Stored Procedure) cataloged in Section 4 must be represented in the telemetry contract.
+- **Tier 1 (Live Scheduler Discovery):** Define authoritative real-time queries against OS and DB schedulers (`msdb.dbo.sysschedules` / `msdb.dbo.sysjobschedules` for SQL Agent jobs; PowerShell `Get-ScheduledTaskInfo` for Windows Task Scheduler) so monitoring automatically tracks schedule changes made by operators in real time without documentation drift.
+- **Tier 2 (Execution Audit Telemetry):** Define queries against application execution audit tables where available (`[dbo].[ETL_AUDIT_LOG]`, `[dbo].[CDR_LOAD_LOG]`, `[dbo].[cdr_validation_run]`, `msdb.dbo.sysjobhistory`) for actual runtime watermarks, durations, and status. If an application lacks audit tables, Tier 2 is omitted and Tier 1 scheduler status is used.
+- **Tier 3 (Static Baseline Fallback):** Define static baseline fallback schedules and maximum allowable runtimes as a safety guardrail when live probes are momentarily unreachable.
+- `hosts` & `db_ping_targets`
+- `pipeline_producers` (with Tier 1, Tier 2, and Tier 3 blocks)
+- `continuous_services`
+- `application_components`
+- `blast_radius_rules`
+```
+
+---
+
+## Diagram and Conversion Standards (Expert Systems Architect & Mermaid Designer)
+
+### 🎨 Design & Styling Principles
+1. **Color Palette & Semantic Styling (Standard/Impact/Existing):**
+   - Use desaturated, modern executive colors. Do not use highly saturated primary colors.
+   - Define exactly 3-4 standard semantic class styles at the top of every diagram:
+     - `existingStyle`: Clean Blue (`fill:#EFF6FF,stroke:#2563EB,stroke-width:2px,color:#1E40AF`) for legacy/established components.
+     - `newStyle`: Warm Red/Crimson (`fill:#FEF2F2,stroke:#DC2626,stroke-width:2px,color:#991B1B`) for greenfield/new components.
+     - `impactedStyle`: Amber/Gold (`fill:#FFFBEB,stroke:#D97706,stroke-width:2px,color:#92400E`) for modified/impacted systems.
+   - Apply classes consistently to the nodes using the triple colon notation (e.g., `NodeName:::newStyle`).
+2. **Diagram Scalability Settings (Initialization Block):**
+   - Always prepend diagrams with a configuration initialization block to control sizing and font scales:
+     ```mermaid
+     %%{init: { 
+       'theme': 'default', 
+       'themeVariables': { 'fontSize': '14px', 'subgraphTitleSize': '16px' }, 
+       'flowchart': { 'useMaxWidth': false, 'htmlLabels': true, 'curve': 'basis' } 
+     } }%%
+     ```
+3. **Readable Text Wrapping:**
+   - Never let a single node have a long horizontal text block. Use `<br/>` tags to stack text vertically.
+   - Format nodes with clear visual hierarchy: Use bolding for titles, bracket notation for IDs, and italics for status:
+     `NodeName["📅 <b>Windows Scheduler</b><br/>[SYS_03]<br/><i>New Component</i>"]`
+
+### 📐 Layout & Real Estate Rules
+1. **The 3-Column Kanban Layout (Side-by-Side):**
+   - When sequential phases are required (e.g., Phase 1 | Phase 2 | Phase 3) but need to prevent horizontal text compression, utilize a "Hybrid Columnar Layout".
+   - **Rule:** Set the top-level flow to Left-to-Right (`flowchart LR`) to align parent containers horizontally, but enforce vertical stacking (`direction TB`) *inside* each subgraph container.
+   - **Example Structure:**
+     ```mermaid
+     flowchart LR
+         subgraph Phase1["Phase 1"]
+             direction TB
+             A --> B --> C
+         end
+         subgraph Phase2["Phase 2"]
+             direction TB
+             D --> E --> F
+         end
+         Phase1 --> Phase2
+     ```
+2. **The Two-Line Grid Layout (Row Stacking):**
+   - When a 3-pillar horizontal layout spans too wide, stack them into rows by setting the top-level structure to Top-to-Bottom (`flowchart TD`).
+   - Group the lower pillars (Phase 2 and Phase 3) side-by-side inside a helper subgraph container that uses `direction LR`. This forces Phase 2 to wrap onto a new line directly below Phase 1.
+3. **Component Types:**
+   - Use standard flowchart shapes to represent structural entities:
+     - `[(Database Node)]` for persistent storage, tables, or file outputs.
+     - `["Application Node"]` for processing scripts, APIs, or services.
+     - `["🖥️ UI Node"]` for user interfaces.
+4. **Dynamic Layout & Sizing:** Do not fix exclusively to Left-to-Right (`flowchart LR`) or Top-to-Bottom (`flowchart TD`). Choose the layout dynamically based on component count and readability:
+   - Use `flowchart LR` for simple linear flows with few components.
+   - Use `flowchart TD` or mixed subgraphs when there are many layers or components to prevent excessive horizontal stretching.
+5. **PlantUML Engine & Crisp Proportioning:** Diagrams convert via `convert_doc_to_confluence.py` using `scale 1.2` for flowcharts, `scale max 2000 width` for sequence diagrams, `skinparam defaultFontSize 11` (no `skinparam dpi 200` to prevent blowout), natural curved splines (omit `polyline` / `ortho`), tightened whitespace (`nodesep 24`, `ranksep 28`, package `Padding 10`), distinct 1.5px border outlines (`#FEF2F2;line:DC2626` for crimson red impacted borders, `#EFF6FF;line:2563EB` for deep blue existing borders), and cross-package `norank` connectors to guarantee sharp, readable, properly proportioned text in Confluence.
+6. **No Cluttered Labels or Overlapping Text:** Omit stereotypes (`<<impacted>>`, `<<service>>`) and status tags (`(New Component)`, `(Existing Platform)`) from component boxes and arrow labels. Multi-line edge labels must be wrapped at ~24 characters.
+7. **No Leaked Tags:** Never emit `<#ghost>` or unsupported formatting tags into Confluence macros. Component and connection classification is handled by the bottom legend.
+8. **Simplified Technical English (ASD-STE100):** Short sentences, active voice, and clear technical descriptions.