fhirsmith 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -2
- package/config-template.json +18 -1
- package/package.json +1 -1
- package/publisher/publisher.js +153 -5
- package/publisher/validation.js +198 -0
- package/server.js +12 -12
- package/stats-db.js +127 -0
- package/stats.js +172 -6
- package/translations/Messages.properties +4 -1
- package/tx/cs/cs-api.md +1 -1
- package/tx/cs/cs-snomed.js +103 -54
- package/tx/library/conceptmap.js +39 -0
- package/tx/library/parameter-check.js +139 -0
- package/tx/operation-context.js +220 -10
- package/tx/params.js +226 -175
- package/tx/sct/expressions.js +485 -0
- package/tx/tx.fhir.org.yml +1 -1
- package/tx/tx.js +57 -50
- package/tx/workers/batch-validate.js +5 -2
- package/tx/workers/cache-control.js +82 -6
- package/tx/workers/expand.js +80 -23
- package/tx/workers/lookup.js +14 -1
- package/tx/workers/translate.js +461 -30
- package/tx/workers/validate.js +7 -1
- package/tx/workers/worker.js +22 -10
- package/tx/xml/xml-base.js +21 -1
- package/tx/xversion/xv-conceptmap.js +81 -0
- package/tx/xversion/xv-parameters.js +8 -0
- package/utilities/merge-audit.py +178 -0
- package/utilities/why-merge-audit.md +39 -0
- package/tx/data/snomed-subset.txt +0 -84
package/tx/workers/worker.js
CHANGED
|
@@ -137,9 +137,10 @@ class TerminologyWorker {
|
|
|
137
137
|
}
|
|
138
138
|
}
|
|
139
139
|
const found = matches[latest];
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
140
|
+
// enable for logging
|
|
141
|
+
// if (this.additionalResourcesCacheId && this.log) {
|
|
142
|
+
// this.log.info(`cache-id '${this.additionalResourcesCacheId}': using cached ${found.resourceType} ${found.url}${found.version ? '|' + found.version : ''} for lookup of ${url}${version ? '|' + version : ''}`);
|
|
143
|
+
// }
|
|
143
144
|
return found;
|
|
144
145
|
}
|
|
145
146
|
}
|
|
@@ -462,7 +463,9 @@ class TerminologyWorker {
|
|
|
462
463
|
/**
|
|
463
464
|
* Process a ValueSet, recording context and extracting embedded expansion parameters
|
|
464
465
|
* @param {Object} vs - ValueSet resource (raw JSON)
|
|
465
|
-
* @param {
|
|
466
|
+
* @param {TxParameters} params - the parameters to add what the ValueSet declares to.
|
|
467
|
+
* Note: this is the caller's parameters, not this.params - the worker's own
|
|
468
|
+
* params is null when expanding an imported ValueSet
|
|
466
469
|
*/
|
|
467
470
|
seeValueSet(vs, params) {
|
|
468
471
|
// Build canonical URL from url and version
|
|
@@ -482,12 +485,14 @@ class TerminologyWorker {
|
|
|
482
485
|
if (nameExt && valueExt) {
|
|
483
486
|
const name = nameExt.valueString || nameExt.valueCode;
|
|
484
487
|
if (name) {
|
|
485
|
-
|
|
488
|
+
params.seeParameter(name, valueExt, false);
|
|
486
489
|
}
|
|
487
490
|
}
|
|
488
491
|
}
|
|
489
492
|
}
|
|
490
493
|
}
|
|
494
|
+
// R6's ValueSet.compose.property
|
|
495
|
+
params.seeCompose(vs.jsonObj.compose);
|
|
491
496
|
if (!params.FHTTPLanguages && vs.jsonObj.language) {
|
|
492
497
|
params.HTTPLanguages = Languages.fromAcceptLanguage(vs.jsonObj.language, this.languages, !this.isValidating());
|
|
493
498
|
}
|
|
@@ -666,9 +671,12 @@ class TerminologyWorker {
|
|
|
666
671
|
// The cache must already exist: caches are created explicitly via
|
|
667
672
|
// $cache-control?mode=start, which is the only thing that mints a cache-id.
|
|
668
673
|
// A cache-id the server doesn't know is an unambiguous, server-authoritative
|
|
669
|
-
// error condition
|
|
670
|
-
//
|
|
671
|
-
//
|
|
674
|
+
// error condition - report it with a specific coded issue rather than silently
|
|
675
|
+
// auto-creating a fresh cache and then failing obscurely later when a
|
|
676
|
+
// by-reference resource can't be found. The cache keeps a tombstone for every
|
|
677
|
+
// id it has retired, so describeMissing() names which fate this id met
|
|
678
|
+
// (never issued here / closed by the client / idle-expired) instead of
|
|
679
|
+
// offering the client a list of possibilities to guess between.
|
|
672
680
|
if (!this.opContext.resourceCache.has(cacheId)) {
|
|
673
681
|
// A comma in the cache-id means the request carried multiple X-Cache-Id
|
|
674
682
|
// headers (Node joins duplicate headers with ", "): server-minted ids are
|
|
@@ -684,8 +692,12 @@ class TerminologyWorker {
|
|
|
684
692
|
'started a second cache on a connection that already had one, without removing the first header.',
|
|
685
693
|
'cache-id-duplicate', 400);
|
|
686
694
|
}
|
|
687
|
-
|
|
688
|
-
|
|
695
|
+
// The issue *code* stays cache-id-unknown for every fate: clients switch on
|
|
696
|
+
// the coding, and all three mean the same thing to them (this cache is gone,
|
|
697
|
+
// don't retry). Only the human-readable diagnostics differ.
|
|
698
|
+
const { messageId, params } = this.opContext.resourceCache.describeMissing(cacheId);
|
|
699
|
+
throw new Issue('error', 'not-found', null, messageId,
|
|
700
|
+
this.i18n.translate(messageId, this.opContext.langs, params),
|
|
689
701
|
'cache-id-unknown', 404);
|
|
690
702
|
}
|
|
691
703
|
|
package/tx/xml/xml-base.js
CHANGED
|
@@ -486,12 +486,32 @@ class FhirXmlBase {
|
|
|
486
486
|
|
|
487
487
|
/**
|
|
488
488
|
* Escape special characters for XML
|
|
489
|
+
*
|
|
490
|
+
* XML 1.0 cannot represent the C0 control characters at all - not literally,
|
|
491
|
+
* and not as numeric character references either - so a value carrying one
|
|
492
|
+
* cannot be serialised as XML by any means. Tab, line feed and carriage
|
|
493
|
+
* return are the three exceptions; DEL (U+007F) is legal in XML 1.0 and is
|
|
494
|
+
* left alone.
|
|
495
|
+
*
|
|
496
|
+
* Rather than emit a document no conformant parser will read, this rejects
|
|
497
|
+
* the value. Anything reaching here with a control character in it came from
|
|
498
|
+
* outside (a remote API, an imported file) and should have been caught at
|
|
499
|
+
* the boundary.
|
|
500
|
+
*
|
|
489
501
|
* @param {*} value - Value to escape
|
|
490
502
|
* @returns {string}
|
|
503
|
+
* @throws {Error} if the value contains a character XML cannot carry
|
|
491
504
|
*/
|
|
492
505
|
static escapeXml(value) {
|
|
493
506
|
if (value === null || value === undefined) return '';
|
|
494
|
-
|
|
507
|
+
const str = String(value);
|
|
508
|
+
// eslint-disable-next-line no-control-regex
|
|
509
|
+
const illegal = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/.exec(str);
|
|
510
|
+
if (illegal) {
|
|
511
|
+
const code = illegal[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
|
|
512
|
+
throw new Error(`Cannot serialise this content as XML: it contains U+${code} at offset ${illegal.index}, a control character XML does not allow`);
|
|
513
|
+
}
|
|
514
|
+
return str
|
|
495
515
|
.replace(/&/g, '&')
|
|
496
516
|
.replace(/</g, '<')
|
|
497
517
|
.replace(/>/g, '>')
|
|
@@ -1,5 +1,63 @@
|
|
|
1
1
|
const {VersionUtilities} = require("../../library/version-utilities");
|
|
2
2
|
|
|
3
|
+
// R4/R3 have no ConceptMap.group.unmapped.valueSet or .relationship. The Java version
|
|
4
|
+
// convertors carry them across as these cross-version extensions, so read them back.
|
|
5
|
+
const EXT_UNMAPPED_VALUESET = 'http://hl7.org/fhir/5.0/StructureDefinition/extension-ConceptMap.group.unmapped.valueSet';
|
|
6
|
+
const EXT_UNMAPPED_RELATIONSHIP = 'http://hl7.org/fhir/5.0/StructureDefinition/extension-ConceptMap.group.unmapped.relationship';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* R4 spells group.unmapped differently: the target map is 'url' rather than 'otherMap',
|
|
10
|
+
* and the 'provided' mode is called 'use-source-code' in R5.
|
|
11
|
+
*/
|
|
12
|
+
function unmappedToR5(unmapped) {
|
|
13
|
+
if (!unmapped) {
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
if (unmapped.mode === 'provided') {
|
|
17
|
+
unmapped.mode = 'use-source-code';
|
|
18
|
+
}
|
|
19
|
+
if (unmapped.url !== undefined && unmapped.otherMap === undefined) {
|
|
20
|
+
unmapped.otherMap = unmapped.url;
|
|
21
|
+
delete unmapped.url;
|
|
22
|
+
}
|
|
23
|
+
for (const ext of unmapped.extension || []) {
|
|
24
|
+
if (ext.url === EXT_UNMAPPED_VALUESET && unmapped.valueSet === undefined) {
|
|
25
|
+
unmapped.valueSet = ext.valueCanonical || ext.valueUri;
|
|
26
|
+
}
|
|
27
|
+
if (ext.url === EXT_UNMAPPED_RELATIONSHIP && unmapped.relationship === undefined) {
|
|
28
|
+
unmapped.relationship = ext.valueCode || ext.valueString;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (unmapped.extension) {
|
|
32
|
+
unmapped.extension = unmapped.extension.filter(e => e.url !== EXT_UNMAPPED_VALUESET && e.url !== EXT_UNMAPPED_RELATIONSHIP);
|
|
33
|
+
if (unmapped.extension.length === 0) {
|
|
34
|
+
delete unmapped.extension;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The inverse of unmappedToR5. */
|
|
40
|
+
function unmappedFromR5(unmapped) {
|
|
41
|
+
if (!unmapped) {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
if (unmapped.mode === 'use-source-code') {
|
|
45
|
+
unmapped.mode = 'provided';
|
|
46
|
+
}
|
|
47
|
+
if (unmapped.otherMap !== undefined) {
|
|
48
|
+
unmapped.url = unmapped.otherMap;
|
|
49
|
+
delete unmapped.otherMap;
|
|
50
|
+
}
|
|
51
|
+
if (unmapped.valueSet !== undefined) {
|
|
52
|
+
(unmapped.extension = unmapped.extension || []).push({url: EXT_UNMAPPED_VALUESET, valueCanonical: unmapped.valueSet});
|
|
53
|
+
delete unmapped.valueSet;
|
|
54
|
+
}
|
|
55
|
+
if (unmapped.relationship !== undefined) {
|
|
56
|
+
(unmapped.extension = unmapped.extension || []).push({url: EXT_UNMAPPED_RELATIONSHIP, valueCode: unmapped.relationship});
|
|
57
|
+
delete unmapped.relationship;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
3
61
|
/**
|
|
4
62
|
* Converts input ConceptMap to R5 format (modifies input object for performance)
|
|
5
63
|
* @param {Object} jsonObj - The input ConceptMap object
|
|
@@ -45,8 +103,30 @@ function conceptMapToR5(jsonObj, sourceVersion) {
|
|
|
45
103
|
// Convert equivalence to relationship in group.element.target
|
|
46
104
|
if (jsonObj.group && Array.isArray(jsonObj.group)) {
|
|
47
105
|
jsonObj.group.forEach(group => {
|
|
106
|
+
unmappedToR5(group.unmapped);
|
|
48
107
|
if (group.element && Array.isArray(group.element)) {
|
|
49
108
|
group.element.forEach(element => {
|
|
109
|
+
// R4 has no element.noMap. A target with equivalence = 'unmatched' and no
|
|
110
|
+
// code is R4's way of saying the same thing: there is positively no map
|
|
111
|
+
// for this concept. Lift it to noMap and drop the placeholder target, so
|
|
112
|
+
// everything downstream sees one R5 shape. This mirrors what the Java
|
|
113
|
+
// convertors do (ConceptMap40_50.convertSourceElementComponent), including
|
|
114
|
+
// their reservation of 'unmatched' for noMap - a real 'not-related-to'
|
|
115
|
+
// target converts to 'disjoint' and carries a code, so it is untouched here.
|
|
116
|
+
if (element.target && Array.isArray(element.target)) {
|
|
117
|
+
const noMapTargets = element.target.filter(t => t.equivalence === 'unmatched' && !t.code);
|
|
118
|
+
if (noMapTargets.length > 0) {
|
|
119
|
+
element.noMap = true;
|
|
120
|
+
const comment = noMapTargets.find(t => t.comment);
|
|
121
|
+
if (comment && !element.comment) {
|
|
122
|
+
element.comment = comment.comment;
|
|
123
|
+
}
|
|
124
|
+
element.target = element.target.filter(t => !(t.equivalence === 'unmatched' && !t.code));
|
|
125
|
+
if (element.target.length === 0) {
|
|
126
|
+
delete element.target;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
50
130
|
if (element.target && Array.isArray(element.target)) {
|
|
51
131
|
element.target.forEach(target => {
|
|
52
132
|
if (target.equivalence && !target.relationship) {
|
|
@@ -142,6 +222,7 @@ function conceptMapR5ToR4(r5Obj) {
|
|
|
142
222
|
// Convert relationship back to equivalence in group.element.target
|
|
143
223
|
if (r5Obj.group && Array.isArray(r5Obj.group)) {
|
|
144
224
|
r5Obj.group.forEach(group => {
|
|
225
|
+
unmappedFromR5(group.unmapped);
|
|
145
226
|
if (group.element && Array.isArray(group.element)) {
|
|
146
227
|
group.element.forEach(element => {
|
|
147
228
|
if (element.target && Array.isArray(element.target)) {
|
|
@@ -88,6 +88,14 @@ function fixMatchParameterfor5(p) {
|
|
|
88
88
|
|
|
89
89
|
function fixMatchParameterfor4(p) {
|
|
90
90
|
if (p.part) {
|
|
91
|
+
// R4 calls originMap 'source'. originMap is reported in every version; source is added
|
|
92
|
+
// here for R4 only, and is prohibited everywhere else
|
|
93
|
+
if (!p.part.find(pp => pp.name === 'source')) {
|
|
94
|
+
let om = p.part.find(pp => pp.name === 'originMap');
|
|
95
|
+
if (om && om.valueCanonical) {
|
|
96
|
+
p.part.push({name: "source", valueCanonical: om.valueCanonical});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
91
99
|
if (!p.part.find(pp => pp.name === 'equivalence')) {
|
|
92
100
|
let rel = p.part.find(pp => pp.name === 'relationship');
|
|
93
101
|
if (rel && rel.valueCode) {
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""merge-audit.py — find content silently lost in merge commits.
|
|
3
|
+
|
|
4
|
+
For every merge commit in range, compares the *recorded* merge result against a
|
|
5
|
+
mechanical re-merge of its parents (git's --remerge-diff). Lines the mechanical
|
|
6
|
+
merge would have kept but the recorded merge dropped are "vanished" content —
|
|
7
|
+
either a deliberate conflict resolution, or an accident nobody chose.
|
|
8
|
+
|
|
9
|
+
For each vanished line the script then checks:
|
|
10
|
+
1. is it still absent from the tip of the branch? (if it came back, ignore)
|
|
11
|
+
2. did any *later regular commit* touch that content? (if someone deliberately
|
|
12
|
+
removed/reworded it afterwards, it's probably intentional — the explaining
|
|
13
|
+
commit is shown so a human can judge)
|
|
14
|
+
|
|
15
|
+
Findings are classified:
|
|
16
|
+
SILENT - the file was NOT listed in the merge commit's "# Conflicts:" trailer:
|
|
17
|
+
git merged it cleanly, yet the recorded result differs. Nobody chose
|
|
18
|
+
this. Highest suspicion.
|
|
19
|
+
CONFLICT - the file was listed as conflicted; content was dropped during manual
|
|
20
|
+
resolution. May be intentional, still worth review.
|
|
21
|
+
|
|
22
|
+
Requirements: git >= 2.36 (for --remerge-diff), python3. Run against a full
|
|
23
|
+
(non-shallow) clone for best results; on a shallow clone, history checks near the
|
|
24
|
+
boundary will attribute old content to the boundary commit.
|
|
25
|
+
|
|
26
|
+
Usage:
|
|
27
|
+
python3 merge-audit.py /path/to/repo --since 2026-01-01 [--tip master]
|
|
28
|
+
python3 merge-audit.py /path/to/repo --range v6.0..master
|
|
29
|
+
python3 merge-audit.py /path/to/repo --since 2026-06-01 --path-prefix source/
|
|
30
|
+
|
|
31
|
+
Exit code is 1 if any SILENT finding with still-missing content exists (useful
|
|
32
|
+
as a CI guard on new merges: --range 'origin/master@{1}..origin/master').
|
|
33
|
+
"""
|
|
34
|
+
import argparse, json, re, subprocess, sys
|
|
35
|
+
|
|
36
|
+
MARKER = re.compile(r"^(<{7}|={7}|>{7}|\|{7})")
|
|
37
|
+
|
|
38
|
+
def norm(s):
|
|
39
|
+
return "".join(s.split())
|
|
40
|
+
|
|
41
|
+
def git(repo, *args, check=True):
|
|
42
|
+
r = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True)
|
|
43
|
+
if check and r.returncode != 0:
|
|
44
|
+
sys.exit(f"git {' '.join(args[:3])}... failed: {r.stderr.strip()[:300]}")
|
|
45
|
+
return r
|
|
46
|
+
|
|
47
|
+
def parse_remerge_log(text):
|
|
48
|
+
merges, cur, cur_file = [], None, None
|
|
49
|
+
def close_file():
|
|
50
|
+
nonlocal cur_file
|
|
51
|
+
if cur is not None and cur_file is not None:
|
|
52
|
+
slot = cur["files"].setdefault(cur_file["path"], {"removed": [], "added": []})
|
|
53
|
+
slot["removed"] += cur_file["removed"]
|
|
54
|
+
slot["added"] += cur_file["added"]
|
|
55
|
+
cur_file = None
|
|
56
|
+
def close_merge():
|
|
57
|
+
nonlocal cur
|
|
58
|
+
close_file()
|
|
59
|
+
if cur is not None:
|
|
60
|
+
merges.append(cur)
|
|
61
|
+
cur = None
|
|
62
|
+
for line in text.splitlines():
|
|
63
|
+
m = re.match(r"^commit ([0-9a-f]{40})", line)
|
|
64
|
+
if m:
|
|
65
|
+
close_merge()
|
|
66
|
+
cur = {"sha": m.group(1), "author": "", "date": "", "subject": None,
|
|
67
|
+
"conflicts": set(), "files": {}}
|
|
68
|
+
continue
|
|
69
|
+
if cur is None:
|
|
70
|
+
continue
|
|
71
|
+
if line.startswith("Author: "):
|
|
72
|
+
cur["author"] = line[8:]; continue
|
|
73
|
+
if line.startswith("Date: "):
|
|
74
|
+
cur["date"] = line[6:].strip(); continue
|
|
75
|
+
if line.startswith("diff --git "):
|
|
76
|
+
close_file()
|
|
77
|
+
m2 = re.match(r"^diff --git a/(.*) b/(.*)$", line)
|
|
78
|
+
cur_file = {"path": m2.group(2) if m2 else line, "removed": [], "added": []}
|
|
79
|
+
continue
|
|
80
|
+
if cur_file is not None:
|
|
81
|
+
if line.startswith("-") and not line.startswith("---"):
|
|
82
|
+
body = line[1:]
|
|
83
|
+
if not MARKER.match(body.lstrip()):
|
|
84
|
+
cur_file["removed"].append(body)
|
|
85
|
+
elif line.startswith("+") and not line.startswith("+++"):
|
|
86
|
+
cur_file["added"].append(line[1:])
|
|
87
|
+
else:
|
|
88
|
+
st = line.strip()
|
|
89
|
+
if cur["subject"] is None and st and not st.startswith("Merge: "):
|
|
90
|
+
cur["subject"] = st
|
|
91
|
+
if st.startswith("#") and "/" in st and "Conflicts" not in st:
|
|
92
|
+
cur["conflicts"].add(st.lstrip("#").strip())
|
|
93
|
+
close_merge()
|
|
94
|
+
return merges
|
|
95
|
+
|
|
96
|
+
def main():
|
|
97
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
98
|
+
ap.add_argument("repo")
|
|
99
|
+
ap.add_argument("--since", help="only merges after this date, e.g. 2026-01-01")
|
|
100
|
+
ap.add_argument("--range", dest="range_", help="rev range, e.g. v6.0..master")
|
|
101
|
+
ap.add_argument("--tip", default="master", help="branch whose tip defines 'still missing' (default master)")
|
|
102
|
+
ap.add_argument("--path-prefix", default="", help="only report findings under this path")
|
|
103
|
+
ap.add_argument("--min-line", type=int, default=4, help="ignore vanished lines shorter than this (normalized)")
|
|
104
|
+
ap.add_argument("--json", dest="json_out", help="also write findings as JSON to this file")
|
|
105
|
+
args = ap.parse_args()
|
|
106
|
+
|
|
107
|
+
ver = git(args.repo, "version").stdout.strip()
|
|
108
|
+
m = re.search(r"(\d+)\.(\d+)", ver)
|
|
109
|
+
if m and (int(m.group(1)), int(m.group(2))) < (2, 36):
|
|
110
|
+
sys.exit(f"{ver}: --remerge-diff needs git >= 2.36")
|
|
111
|
+
|
|
112
|
+
log_args = ["log", "--merges", "--remerge-diff", "--no-color"]
|
|
113
|
+
if args.since:
|
|
114
|
+
log_args.append(f"--since={args.since}")
|
|
115
|
+
log_args.append(args.range_ if args.range_ else args.tip)
|
|
116
|
+
print(f"re-merging all merge commits in range (this can take a few minutes)...", file=sys.stderr)
|
|
117
|
+
merges = parse_remerge_log(git(args.repo, *log_args).stdout)
|
|
118
|
+
|
|
119
|
+
tip_cache = {}
|
|
120
|
+
def tip_content(path):
|
|
121
|
+
if path not in tip_cache:
|
|
122
|
+
r = git(args.repo, "show", f"{args.tip}:{path}", check=False)
|
|
123
|
+
tip_cache[path] = r.stdout if r.returncode == 0 else None
|
|
124
|
+
return tip_cache[path]
|
|
125
|
+
|
|
126
|
+
findings = []
|
|
127
|
+
for mg in merges:
|
|
128
|
+
for path, fd in mg["files"].items():
|
|
129
|
+
if args.path_prefix and not path.startswith(args.path_prefix):
|
|
130
|
+
continue
|
|
131
|
+
added_norms = {norm(a) for a in fd["added"]}
|
|
132
|
+
vanished = [r for r in fd["removed"]
|
|
133
|
+
if norm(r) not in added_norms and len(norm(r)) > args.min_line]
|
|
134
|
+
if not vanished:
|
|
135
|
+
continue
|
|
136
|
+
tip = tip_content(path)
|
|
137
|
+
if tip is None:
|
|
138
|
+
still, note = vanished, "file absent at tip"
|
|
139
|
+
else:
|
|
140
|
+
tip_norms = {norm(l) for l in tip.splitlines()}
|
|
141
|
+
still = [v for v in vanished if norm(v) not in tip_norms]
|
|
142
|
+
note = ""
|
|
143
|
+
if not still:
|
|
144
|
+
continue
|
|
145
|
+
probe = max(still, key=lambda s: len(norm(s))).strip()
|
|
146
|
+
hist = git(args.repo, "log", "--format=%h %an %ad %s", "--date=short",
|
|
147
|
+
"-S", probe, args.tip, "--", path, check=False).stdout.splitlines()
|
|
148
|
+
hist = [h for h in hist if not h.startswith(mg["sha"][:8])]
|
|
149
|
+
findings.append({
|
|
150
|
+
"sha": mg["sha"][:10], "date": mg["date"], "author": mg["author"],
|
|
151
|
+
"subject": mg["subject"] or "", "path": path,
|
|
152
|
+
"kind": "CONFLICT" if path in mg["conflicts"] else "SILENT",
|
|
153
|
+
"vanished": len(vanished), "still_missing": len(still), "note": note,
|
|
154
|
+
"history": hist[:4], "lines": [s.strip() for s in still[:12]],
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
findings.sort(key=lambda x: (x["kind"] != "SILENT", -x["still_missing"]))
|
|
158
|
+
print(f"\n# Merge-loss audit: {len(merges)} merges scanned, "
|
|
159
|
+
f"{len(findings)} file-level findings with content still missing at {args.tip}\n")
|
|
160
|
+
for f in findings:
|
|
161
|
+
print("=" * 100)
|
|
162
|
+
print(f"{f['kind']:8s} merge {f['sha']} {f['date']} {f['author']}")
|
|
163
|
+
print(f" {f['subject'][:100]}")
|
|
164
|
+
print(f" {f['path']} (still missing {f['still_missing']} of {f['vanished']} dropped lines) {f['note']}")
|
|
165
|
+
if f["history"]:
|
|
166
|
+
print(" later commits touching this content (may explain an intentional removal):")
|
|
167
|
+
for h in f["history"]:
|
|
168
|
+
print(f" {h[:140]}")
|
|
169
|
+
for l in f["lines"]:
|
|
170
|
+
print(f" - {l[:160]}")
|
|
171
|
+
print(f" inspect: git show --remerge-diff {f['sha']} -- '{f['path']}'")
|
|
172
|
+
if args.json_out:
|
|
173
|
+
with open(args.json_out, "w") as fh:
|
|
174
|
+
json.dump(findings, fh, indent=1)
|
|
175
|
+
sys.exit(1 if any(f["kind"] == "SILENT" for f in findings) else 0)
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__":
|
|
178
|
+
main()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Why add a merge-loss audit to your repository
|
|
2
|
+
|
|
3
|
+
## The problem
|
|
4
|
+
|
|
5
|
+
When someone merges one branch into another and resolves conflicts by hand, they can accidentally revert other people's work — most commonly by keeping their whole version of a file, which throws away changes in regions of the file that were never in conflict. Git records whatever resolution they commit, with no distinction between "what the merge algorithm produced" and "what the human changed afterwards."
|
|
6
|
+
|
|
7
|
+
This class of error is uniquely invisible. The lost content doesn't appear in the pull request's "files changed" view, or appears buried in a list of hundreds of incidentally-touched files that no reviewer can realistically audit. Standard forensics miss it too: `git log -S` and `git blame` skip merge commits by default, so even when someone later notices content is gone, the search for *when* and *who* comes up empty. Worse, the blame usually lands on the wrong person — whoever's later, entirely innocent merge first makes the absence visible in a diff.
|
|
8
|
+
|
|
9
|
+
Any repository where humans (or, increasingly, coding agents) resolve merge conflicts has this exposure. That includes every workflow where contributors maintain local clones and pull with divergence — regardless of whether the project merges PRs by merge commit, squash, or rebase.
|
|
10
|
+
|
|
11
|
+
## A real incident
|
|
12
|
+
|
|
13
|
+
In HL7/fhir, a validation invariant (qrs-4, FHIR-55973) was added to the QuestionnaireResponse resource in April 2026. In June, a routine "merge master into my branch" commit silently reverted it — the file wasn't even listed as conflicted in the merge. The loss rode into master inside an unrelated PR, went unnoticed for two months, and was eventually blamed on the author of a *different* PR whose merge happened to make the absence visible. A retrospective audit of that repository found the same merge had silently reverted content in ~25 files, and found several additional losses from other merges, including one introduced by an AI coding agent resolving PR conflicts.
|
|
14
|
+
|
|
15
|
+
Every one of those losses was detectable, mechanically, on the day it happened.
|
|
16
|
+
|
|
17
|
+
## What the check does
|
|
18
|
+
|
|
19
|
+
Git 2.36 added `--remerge-diff`: for any merge commit, git redoes the merge mechanically and shows only what the *recorded* result changed beyond that. An empty result means the merge was exactly what the algorithm produced. A non-empty result is precisely the set of decisions the human made — a handful of lines, instead of a hundred-file merge diff.
|
|
20
|
+
|
|
21
|
+
The audit script (`tools/merge-audit.py`) runs this over a range of commits and classifies each deviation. Content dropped from a file that was **not** conflicted is flagged as a *silent drop* — nobody chose it, and it is almost always an accident. Content dropped during a genuine conflict resolution is listed for review with lower urgency. The accompanying GitHub Actions workflow runs the script on each push to the main branch, over just the newly pushed commits, and posts a warning and job summary when it finds a silent drop. It is advisory only — it never fails a build — and it also supports manual runs over a longer window as a periodic sweep.
|
|
22
|
+
|
|
23
|
+
For contributors, there is a one-line self-check before pushing any hand-resolved merge:
|
|
24
|
+
|
|
25
|
+
git show --remerge-diff HEAD
|
|
26
|
+
|
|
27
|
+
If that shows anything you didn't consciously decide, the merge is wrong.
|
|
28
|
+
|
|
29
|
+
## The cost
|
|
30
|
+
|
|
31
|
+
Near zero. The CI job needs a full-history checkout and a few minutes of compute per push; the script has no dependencies beyond git ≥ 2.36 and python3. False positives are rare and self-explaining: deliberate resolutions show up with the commit that explains them, and are waived by reading the warning. There is nothing to configure beyond the path prefix that scopes which files matter (e.g. `source/`).
|
|
32
|
+
|
|
33
|
+
## Why this isn't already standard practice
|
|
34
|
+
|
|
35
|
+
`--remerge-diff` is recent (2022) and little-known, and no packaged CI action popularized it the way linters and secret-scanners were popularized. The failure mode is rare per-merge, invisible when it occurs, and expensive to diagnose months later — exactly the profile of a problem that never causes enough pain in any one place to produce standard tooling. That is an argument for adopting the check, not against it: the first time it fires, it will have paid for itself.
|
|
36
|
+
|
|
37
|
+
## Adoption
|
|
38
|
+
|
|
39
|
+
Copy `tools/merge-audit.py` into the repository, add the workflow as `.github/workflows/merge-audit.yml`, and set the `--path-prefix` argument to the directory tree whose content matters. For an initial baseline, run it once by hand over the repository's recent history (`python3 tools/merge-audit.py . --since <date>`) — it will tell you whether you already have losses you don't know about.
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
900000000000526001 |REPLACED BY association reference set (foundation metadata concept)|
|
|
2
|
-
1539003 |Acquired trigger finger (disorder)|
|
|
3
|
-
1204474000 |Product containing precisely barium sulfate 600 milligram/1 milliliter conventional release oral suspension (clinical drug)|
|
|
4
|
-
86299006 |Tetralogy of Fallot (disorder)|
|
|
5
|
-
430983003 |Brucella abortus, vaccinal strain RB51 (organism)|
|
|
6
|
-
608785007 |Open reduction of fracture of radius and ulna with internal fixation (procedure)|
|
|
7
|
-
370049004 |No tumor invasion (finding)|
|
|
8
|
-
370050004 |No tumor invasion of adjacent tissue (finding)|
|
|
9
|
-
174826008 |Arterial switch operation (procedure)|
|
|
10
|
-
11687002 |Gestational diabetes mellitus (disorder)|
|
|
11
|
-
897148007 |Alcoholic beverage intake (observable entity)|
|
|
12
|
-
816080008 |International Patient Summary (foundation metadata concept)|
|
|
13
|
-
303248007 |Blood film specimen (specimen)|
|
|
14
|
-
162980001 |Cardiovascular system not examined (situation)|
|
|
15
|
-
297250002 |No family history of stroke (situation)|
|
|
16
|
-
162001003 |No cardiovascular symptom (situation)|
|
|
17
|
-
281302008 |Above reference range (qualifier value)|
|
|
18
|
-
281300000 |Below reference range (qualifier value)|
|
|
19
|
-
281301001 |Within reference range (qualifier value)|
|
|
20
|
-
268677005 |[X]Other delirium (disorder)|
|
|
21
|
-
155728006 |Appendicitis (disorder)|
|
|
22
|
-
74400008 |Appendicitis (disorder)|
|
|
23
|
-
155729003 |Appendicitis (disorder)|
|
|
24
|
-
307530000 |Appendicitis NOS (disorder)|
|
|
25
|
-
42463004 |Genus Opisthorchis (organism)|
|
|
26
|
-
71650008 |Opisthorchis viverrini (organism)|
|
|
27
|
-
116676008 |Associated morphology (attribute)|
|
|
28
|
-
128045006 |Cellulitis (disorder)|
|
|
29
|
-
20946005 |Fracture, closed (morphologic abnormality)|
|
|
30
|
-
28012007 |Closed fracture of shaft of tibia (disorder)|
|
|
31
|
-
363698007 |Finding site (attribute)|
|
|
32
|
-
447139008 |Closed fracture of tibia (disorder)|
|
|
33
|
-
52687003 |Bone structure of shaft of tibia (body structure)|
|
|
34
|
-
56459004 |Foot structure (body structure)|
|
|
35
|
-
64572001 |Disease (disorder)|
|
|
36
|
-
6990005 |Fracture of shaft of tibia (disorder)|
|
|
37
|
-
<<128241005 |Inflammatory disease of liver (disorder)|
|
|
38
|
-
<<10200004 |Liver structure (body structure)|
|
|
39
|
-
<<90140006 |Structure of intrahepatic biliary tree (body structure)|
|
|
40
|
-
<<364159005 |Liver observable (observable entity)|
|
|
41
|
-
<<227014008 |Liver pate (substance)|
|
|
42
|
-
<<249565005 |Liver finding (finding)|
|
|
43
|
-
<<406459008 |Product containing halibut liver oil (medicinal product)|
|
|
44
|
-
<<776168003 |Product containing only halibut liver oil (medicinal product)|
|
|
45
|
-
<<309399009 |Hospital-based dietitian (occupation)|
|
|
46
|
-
<<421813008 |Class Hepaticopsida (organism)|
|
|
47
|
-
<<441802002 |Computed tomography of liver (procedure)|
|
|
48
|
-
<<119383005 |Specimen from liver (specimen)|
|
|
49
|
-
129433002 |Inspection|
|
|
50
|
-
367430006 |Repair of tendon of hand (procedure)|
|
|
51
|
-
119657005 |Hand repair (procedure)|
|
|
52
|
-
274059009 |Hand tendon operation (procedure)|
|
|
53
|
-
281760001 |Repair of tendon of upper limb (procedure)|
|
|
54
|
-
18701002 |Repair of tendon of hand with graft (procedure)|
|
|
55
|
-
214433003 |Repair of ruptured tendon of hand (procedure)|
|
|
56
|
-
243234005 |Repair of tendon of hand by transfer or transplantation (procedure)|
|
|
57
|
-
26731003 |Repair of tendon of hand by suture (procedure)|
|
|
58
|
-
27106001 |Reconstruction of tendon pulley of hand (procedure)|
|
|
59
|
-
45810006 |Reattachment of tendon of hand (procedure)|
|
|
60
|
-
709291000 |Repair of flexor tendon of hand (procedure)|
|
|
61
|
-
712638006 |Repair of extensor tendon of hand (procedure)|
|
|
62
|
-
76340004 |Plication of tendon of hand (procedure)|
|
|
63
|
-
90650008 |Lengthening of tendon of hand (procedure)|
|
|
64
|
-
90907001 |Shortening of tendon of hand (procedure)|
|
|
65
|
-
91092007 |Myotenontoplasty of hand (procedure)|
|
|
66
|
-
118632007 |Structure of tendon within hand (body structure)|
|
|
67
|
-
13024002 |Tendon structure (body structure)|
|
|
68
|
-
245098002 |Structure of tendon within upper limb (body structure)|
|
|
69
|
-
85562004 |Hand structure (body structure)|
|
|
70
|
-
#Referenced in request
|
|
71
|
-
734139008 |Anatomy structure and part association reference set (foundation metadata concept)|
|
|
72
|
-
#For snomed-procedure-in-display test
|
|
73
|
-
#Required by: validate-code-isa-in-response.json
|
|
74
|
-
41271000119108 |Infective arthritis caused by virus and co-occurrent with hepatitis (disorder)|
|
|
75
|
-
#Required by: expand-ecl-ancOrSelf-response.json, expand-ecl-ancestors-response.json, expand-ecl-parents-response.json, expand-ecl-parentsOrSelf-response.json
|
|
76
|
-
303270005 |Liver and biliary structure (body structure)|
|
|
77
|
-
699602007 |Structure of viscus of abdomen proper (body structure)|
|
|
78
|
-
#Required by: expand-ecl-ancOrSelf-response.json, expand-ecl-ancestors-response.json
|
|
79
|
-
420473002 |Structure of subregion of trunk (body structure)|
|
|
80
|
-
49596003 |Digestive organ (body structure)|
|
|
81
|
-
86762007 |Digestive system (body structure)|
|
|
82
|
-
1197039003 |Structure of lymphatic vessel of liver (body structure)|
|
|
83
|
-
#Required by: lookup-procedure-pc-response.json
|
|
84
|
-
24028007 |Right (qualifier value)|
|