fhirsmith 0.11.2 → 0.13.0

package/utilities/merge-audit.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""merge-audit.py — find content silently lost in merge commits.
+
+For every merge commit in range, compares the *recorded* merge result against a
+mechanical re-merge of its parents (git's --remerge-diff). Lines the mechanical
+merge would have kept but the recorded merge dropped are "vanished" content —
+either a deliberate conflict resolution, or an accident nobody chose.
+
+For each vanished line the script then checks:
+  1. is it still absent from the tip of the branch? (if it came back, ignore)
+  2. did any *later regular commit* touch that content? (if someone deliberately
+     removed/reworded it afterwards, it's probably intentional — the explaining
+     commit is shown so a human can judge)
+
+Findings are classified:
+  SILENT   - the file was NOT listed in the merge commit's "# Conflicts:" trailer:
+             git merged it cleanly, yet the recorded result differs. Nobody chose
+             this. Highest suspicion.
+  CONFLICT - the file was listed as conflicted; content was dropped during manual
+             resolution. May be intentional, still worth review.
+
+Requirements: git >= 2.36 (for --remerge-diff), python3. Run against a full
+(non-shallow) clone for best results; on a shallow clone, history checks near the
+boundary will attribute old content to the boundary commit.
+
+Usage:
+  python3 merge-audit.py /path/to/repo --since 2026-01-01 [--tip master]
+  python3 merge-audit.py /path/to/repo --range v6.0..master
+  python3 merge-audit.py /path/to/repo --since 2026-06-01 --path-prefix source/
+
+Exit code is 1 if any SILENT finding with still-missing content exists (useful
+as a CI guard on new merges: --range 'origin/master@{1}..origin/master').
+"""
+import argparse, json, re, subprocess, sys
+
+MARKER = re.compile(r"^(<{7}|={7}|>{7}|\|{7})")
+
+def norm(s):
+    return "".join(s.split())
+
+def git(repo, *args, check=True):
+    r = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True)
+    if check and r.returncode != 0:
+        sys.exit(f"git {' '.join(args[:3])}... failed: {r.stderr.strip()[:300]}")
+    return r
+
+def parse_remerge_log(text):
+    merges, cur, cur_file = [], None, None
+    def close_file():
+        nonlocal cur_file
+        if cur is not None and cur_file is not None:
+            slot = cur["files"].setdefault(cur_file["path"], {"removed": [], "added": []})
+            slot["removed"] += cur_file["removed"]
+            slot["added"] += cur_file["added"]
+        cur_file = None
+    def close_merge():
+        nonlocal cur
+        close_file()
+        if cur is not None:
+            merges.append(cur)
+        cur = None
+    for line in text.splitlines():
+        m = re.match(r"^commit ([0-9a-f]{40})", line)
+        if m:
+            close_merge()
+            cur = {"sha": m.group(1), "author": "", "date": "", "subject": None,
+                   "conflicts": set(), "files": {}}
+            continue
+        if cur is None:
+            continue
+        if line.startswith("Author: "):
+            cur["author"] = line[8:]; continue
+        if line.startswith("Date: "):
+            cur["date"] = line[6:].strip(); continue
+        if line.startswith("diff --git "):
+            close_file()
+            m2 = re.match(r"^diff --git a/(.*) b/(.*)$", line)
+            cur_file = {"path": m2.group(2) if m2 else line, "removed": [], "added": []}
+            continue
+        if cur_file is not None:
+            if line.startswith("-") and not line.startswith("---"):
+                body = line[1:]
+                if not MARKER.match(body.lstrip()):
+                    cur_file["removed"].append(body)
+            elif line.startswith("+") and not line.startswith("+++"):
+                cur_file["added"].append(line[1:])
+        else:
+            st = line.strip()
+            if cur["subject"] is None and st and not st.startswith("Merge: "):
+                cur["subject"] = st
+            if st.startswith("#") and "/" in st and "Conflicts" not in st:
+                cur["conflicts"].add(st.lstrip("#").strip())
+    close_merge()
+    return merges
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("repo")
+    ap.add_argument("--since", help="only merges after this date, e.g. 2026-01-01")
+    ap.add_argument("--range", dest="range_", help="rev range, e.g. v6.0..master")
+    ap.add_argument("--tip", default="master", help="branch whose tip defines 'still missing' (default master)")
+    ap.add_argument("--path-prefix", default="", help="only report findings under this path")
+    ap.add_argument("--min-line", type=int, default=4, help="ignore vanished lines shorter than this (normalized)")
+    ap.add_argument("--json", dest="json_out", help="also write findings as JSON to this file")
+    args = ap.parse_args()
+
+    ver = git(args.repo, "version").stdout.strip()
+    m = re.search(r"(\d+)\.(\d+)", ver)
+    if m and (int(m.group(1)), int(m.group(2))) < (2, 36):
+        sys.exit(f"{ver}: --remerge-diff needs git >= 2.36")
+
+    log_args = ["log", "--merges", "--remerge-diff", "--no-color"]
+    if args.since:
+        log_args.append(f"--since={args.since}")
+    log_args.append(args.range_ if args.range_ else args.tip)
+    print(f"re-merging all merge commits in range (this can take a few minutes)...", file=sys.stderr)
+    merges = parse_remerge_log(git(args.repo, *log_args).stdout)
+
+    tip_cache = {}
+    def tip_content(path):
+        if path not in tip_cache:
+            r = git(args.repo, "show", f"{args.tip}:{path}", check=False)
+            tip_cache[path] = r.stdout if r.returncode == 0 else None
+        return tip_cache[path]
+
+    findings = []
+    for mg in merges:
+        for path, fd in mg["files"].items():
+            if args.path_prefix and not path.startswith(args.path_prefix):
+                continue
+            added_norms = {norm(a) for a in fd["added"]}
+            vanished = [r for r in fd["removed"]
+                        if norm(r) not in added_norms and len(norm(r)) > args.min_line]
+            if not vanished:
+                continue
+            tip = tip_content(path)
+            if tip is None:
+                still, note = vanished, "file absent at tip"
+            else:
+                tip_norms = {norm(l) for l in tip.splitlines()}
+                still = [v for v in vanished if norm(v) not in tip_norms]
+                note = ""
+            if not still:
+                continue
+            probe = max(still, key=lambda s: len(norm(s))).strip()
+            hist = git(args.repo, "log", "--format=%h %an %ad %s", "--date=short",
+                       "-S", probe, args.tip, "--", path, check=False).stdout.splitlines()
+            hist = [h for h in hist if not h.startswith(mg["sha"][:8])]
+            findings.append({
+                "sha": mg["sha"][:10], "date": mg["date"], "author": mg["author"],
+                "subject": mg["subject"] or "", "path": path,
+                "kind": "CONFLICT" if path in mg["conflicts"] else "SILENT",
+                "vanished": len(vanished), "still_missing": len(still), "note": note,
+                "history": hist[:4], "lines": [s.strip() for s in still[:12]],
+            })
+
+    findings.sort(key=lambda x: (x["kind"] != "SILENT", -x["still_missing"]))
+    print(f"\n# Merge-loss audit: {len(merges)} merges scanned, "
+          f"{len(findings)} file-level findings with content still missing at {args.tip}\n")
+    for f in findings:
+        print("=" * 100)
+        print(f"{f['kind']:8s} merge {f['sha']}  {f['date']}  {f['author']}")
+        print(f"         {f['subject'][:100]}")
+        print(f"         {f['path']}  (still missing {f['still_missing']} of {f['vanished']} dropped lines) {f['note']}")
+        if f["history"]:
+            print("         later commits touching this content (may explain an intentional removal):")
+            for h in f["history"]:
+                print(f"           {h[:140]}")
+        for l in f["lines"]:
+            print(f"    - {l[:160]}")
+        print(f"    inspect: git show --remerge-diff {f['sha']} -- '{f['path']}'")
+    if args.json_out:
+        with open(args.json_out, "w") as fh:
+            json.dump(findings, fh, indent=1)
+    sys.exit(1 if any(f["kind"] == "SILENT" for f in findings) else 0)
+
+if __name__ == "__main__":
+    main()

package/utilities/why-merge-audit.md

@@ -0,0 +1,39 @@
+# Why add a merge-loss audit to your repository
+
+## The problem
+
+When someone merges one branch into another and resolves conflicts by hand, they can accidentally revert other people's work — most commonly by keeping their whole version of a file, which throws away changes in regions of the file that were never in conflict. Git records whatever resolution they commit, with no distinction between "what the merge algorithm produced" and "what the human changed afterwards."
+
+This class of error is uniquely invisible. The lost content doesn't appear in the pull request's "files changed" view, or appears buried in a list of hundreds of incidentally-touched files that no reviewer can realistically audit. Standard forensics miss it too: `git log -S` and `git blame` skip merge commits by default, so even when someone later notices content is gone, the search for *when* and *who* comes up empty. Worse, the blame usually lands on the wrong person — whoever's later, entirely innocent merge first makes the absence visible in a diff.
+
+Any repository where humans (or, increasingly, coding agents) resolve merge conflicts has this exposure. That includes every workflow where contributors maintain local clones and pull with divergence — regardless of whether the project merges PRs by merge commit, squash, or rebase.
+
+## A real incident
+
+In HL7/fhir, a validation invariant (qrs-4, FHIR-55973) was added to the QuestionnaireResponse resource in April 2026. In June, a routine "merge master into my branch" commit silently reverted it — the file wasn't even listed as conflicted in the merge. The loss rode into master inside an unrelated PR, went unnoticed for two months, and was eventually blamed on the author of a *different* PR whose merge happened to make the absence visible. A retrospective audit of that repository found the same merge had silently reverted content in ~25 files, and found several additional losses from other merges, including one introduced by an AI coding agent resolving PR conflicts.
+
+Every one of those losses was detectable, mechanically, on the day it happened.
+
+## What the check does
+
+Git 2.36 added `--remerge-diff`: for any merge commit, git redoes the merge mechanically and shows only what the *recorded* result changed beyond that. An empty result means the merge was exactly what the algorithm produced. A non-empty result is precisely the set of decisions the human made — a handful of lines, instead of a hundred-file merge diff.
+
+The audit script (`tools/merge-audit.py`) runs this over a range of commits and classifies each deviation. Content dropped from a file that was **not** conflicted is flagged as a *silent drop* — nobody chose it, and it is almost always an accident. Content dropped during a genuine conflict resolution is listed for review with lower urgency. The accompanying GitHub Actions workflow runs the script on each push to the main branch, over just the newly pushed commits, and posts a warning and job summary when it finds a silent drop. It is advisory only — it never fails a build — and it also supports manual runs over a longer window as a periodic sweep.
+
+For contributors, there is a one-line self-check before pushing any hand-resolved merge:
+
+    git show --remerge-diff HEAD
+
+If that shows anything you didn't consciously decide, the merge is wrong.
+
+## The cost
+
+Near zero. The CI job needs a full-history checkout and a few minutes of compute per push; the script has no dependencies beyond git ≥ 2.36 and python3. False positives are rare and self-explaining: deliberate resolutions show up with the commit that explains them, and are waived by reading the warning. There is nothing to configure beyond the path prefix that scopes which files matter (e.g. `source/`).
+
+## Why this isn't already standard practice
+
+`--remerge-diff` is recent (2022) and little-known, and no packaged CI action popularized it the way linters and secret-scanners were popularized. The failure mode is rare per-merge, invisible when it occurs, and expensive to diagnose months later — exactly the profile of a problem that never causes enough pain in any one place to produce standard tooling. That is an argument for adopting the check, not against it: the first time it fires, it will have paid for itself.
+
+## Adoption
+
+Copy `tools/merge-audit.py` into the repository, add the workflow as `.github/workflows/merge-audit.yml`, and set the `--path-prefix` argument to the directory tree whose content matters. For an initial baseline, run it once by hand over the repository's recent history (`python3 tools/merge-audit.py . --since <date>`) — it will tell you whether you already have losses you don't know about.

package/tx/data/snomed-subset.txt

@@ -1,84 +0,0 @@
-900000000000526001 |REPLACED BY association reference set (foundation metadata concept)|
-1539003 |Acquired trigger finger (disorder)|
-1204474000 |Product containing precisely barium sulfate 600 milligram/1 milliliter conventional release oral suspension (clinical drug)|
-86299006 |Tetralogy of Fallot (disorder)|
-430983003 |Brucella abortus, vaccinal strain RB51 (organism)|
-608785007 |Open reduction of fracture of radius and ulna with internal fixation (procedure)|
-370049004 |No tumor invasion (finding)|
-370050004 |No tumor invasion of adjacent tissue (finding)|
-174826008 |Arterial switch operation (procedure)|
-11687002 |Gestational diabetes mellitus (disorder)|
-897148007 |Alcoholic beverage intake (observable entity)|
-816080008 |International Patient Summary (foundation metadata concept)|
-303248007 |Blood film specimen (specimen)|
-162980001 |Cardiovascular system not examined (situation)|
-297250002 |No family history of stroke (situation)|
-162001003 |No cardiovascular symptom (situation)|
-281302008 |Above reference range (qualifier value)|
-281300000 |Below reference range (qualifier value)|
-281301001 |Within reference range (qualifier value)|
-268677005 |[X]Other delirium (disorder)|
-155728006 |Appendicitis (disorder)|
-74400008 |Appendicitis (disorder)|
-155729003 |Appendicitis (disorder)|
-307530000 |Appendicitis NOS (disorder)|
-42463004 |Genus Opisthorchis (organism)|
-71650008 |Opisthorchis viverrini (organism)|
-116676008 |Associated morphology (attribute)|
-128045006 |Cellulitis (disorder)|
-20946005 |Fracture, closed (morphologic abnormality)|
-28012007 |Closed fracture of shaft of tibia (disorder)|
-363698007 |Finding site (attribute)|
-447139008 |Closed fracture of tibia (disorder)|
-52687003 |Bone structure of shaft of tibia (body structure)|
-56459004 |Foot structure (body structure)|
-64572001 |Disease (disorder)|
-6990005 |Fracture of shaft of tibia (disorder)|
-<<128241005 |Inflammatory disease of liver (disorder)|
-<<10200004 |Liver structure (body structure)|
-<<90140006 |Structure of intrahepatic biliary tree (body structure)|
-<<364159005 |Liver observable (observable entity)|
-<<227014008 |Liver pate (substance)|
-<<249565005 |Liver finding (finding)|
-<<406459008 |Product containing halibut liver oil (medicinal product)|
-<<776168003 |Product containing only halibut liver oil (medicinal product)|
-<<309399009 |Hospital-based dietitian (occupation)|
-<<421813008 |Class Hepaticopsida (organism)|
-<<441802002 |Computed tomography of liver (procedure)|
-<<119383005 |Specimen from liver (specimen)|
-129433002 |Inspection|
-367430006 |Repair of tendon of hand (procedure)|
-119657005 |Hand repair (procedure)|
-274059009 |Hand tendon operation (procedure)|
-281760001 |Repair of tendon of upper limb (procedure)|
-18701002 |Repair of tendon of hand with graft (procedure)|
-214433003 |Repair of ruptured tendon of hand (procedure)|
-243234005 |Repair of tendon of hand by transfer or transplantation (procedure)|
-26731003 |Repair of tendon of hand by suture (procedure)|
-27106001 |Reconstruction of tendon pulley of hand (procedure)|
-45810006 |Reattachment of tendon of hand (procedure)|
-709291000 |Repair of flexor tendon of hand (procedure)|
-712638006 |Repair of extensor tendon of hand (procedure)|
-76340004 |Plication of tendon of hand (procedure)|
-90650008 |Lengthening of tendon of hand (procedure)|
-90907001 |Shortening of tendon of hand (procedure)|
-91092007 |Myotenontoplasty of hand (procedure)|
-118632007 |Structure of tendon within hand (body structure)|
-13024002 |Tendon structure (body structure)|
-245098002 |Structure of tendon within upper limb (body structure)|
-85562004 |Hand structure (body structure)|
-#Referenced in request
-734139008 |Anatomy structure and part association reference set (foundation metadata concept)|
-#For snomed-procedure-in-display test
-#Required by: validate-code-isa-in-response.json
-41271000119108 |Infective arthritis caused by virus and co-occurrent with hepatitis (disorder)|
-#Required by: expand-ecl-ancOrSelf-response.json, expand-ecl-ancestors-response.json, expand-ecl-parents-response.json, expand-ecl-parentsOrSelf-response.json
-303270005 |Liver and biliary structure (body structure)|
-699602007 |Structure of viscus of abdomen proper (body structure)|
-#Required by: expand-ecl-ancOrSelf-response.json, expand-ecl-ancestors-response.json
-420473002 |Structure of subregion of trunk (body structure)|
-49596003 |Digestive organ (body structure)|
-86762007 |Digestive system (body structure)|
-1197039003 |Structure of lymphatic vessel of liver (body structure)|
-#Required by: lookup-procedure-pc-response.json
-24028007 |Right (qualifier value)|