@kevlns/xlmerge 2.0.1 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,565 +1,660 @@
1
- """Safe, Excel-backed formula materialisation for the conflict resolver.
2
-
3
- This module deliberately has a small trust boundary. Excel is only ever given a
4
- temporary copy, with links and macros disabled. The result produced by Excel is
5
- read back as cached values and written into a copy of the original package with
6
- openpyxl. We do not use ``data_only`` as a source workbook: a stale cache is not
7
- an acceptable answer for a conflict resolver.
8
- """
9
-
10
- from __future__ import annotations
11
-
12
- import json
13
- import math
14
- import os
15
- import re
16
- import shutil
17
- import subprocess
18
- import tempfile
19
- import time
20
- import zipfile
21
- from datetime import date, datetime, time as datetime_time
22
- from pathlib import Path
23
- from typing import Any, Iterable
24
- from xml.etree import ElementTree
25
-
26
- from openpyxl import load_workbook
27
- from openpyxl.cell.rich_text import CellRichText, InlineFont, TextBlock
28
- from openpyxl.formula import Tokenizer
29
- from openpyxl.utils.cell import range_boundaries
30
-
31
-
32
- class FormulaValueError(RuntimeError):
33
- """Raised when a workbook cannot be evaluated and verified safely."""
34
-
35
-
36
- DEFAULT_TIMEOUT_SECONDS = 120
37
- _ERROR_VALUES = {"#NULL!", "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A", "#GETTING_DATA"}
38
- _VOLATILE_FUNCTIONS = {
39
- "NOW", "TODAY", "RAND", "RANDBETWEEN", "RANDARRAY", "RTD", "OFFSET", "INDIRECT",
40
- "CELL", "INFO",
41
- }
42
- _DYNAMIC_FUNCTIONS = {
43
- "FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE", "RANDARRAY", "TRANSPOSE",
44
- "TOCOL", "TOROW", "TAKE", "DROP", "CHOOSECOLS", "CHOOSEROWS", "EXPAND",
45
- "VSTACK", "HSTACK", "WRAPROWS", "WRAPCOLS", "XLOOKUP", "XMATCH", "LAMBDA",
46
- }
47
-
48
- # Function names are checked before opening Excel. This is intentionally a
49
- # broad list of ordinary worksheet functions, but unknown calls are rejected so
50
- # a disabled VBA UDF can never be silently accepted merely because it returned a
51
- # plausible value in a cached workbook.
52
- _BUILTIN_FUNCTIONS = {
53
- name
54
- for name in """
55
- ABS ACOS ACOSH ACOT ACOTH AGGREGATE ARABIC ASIN ASINH ATAN ATAN2 ATANH BASE
56
- CEILING CEILING.MATH CEILING.PRECISE COMBIN COMBINA COS COSH COT COTH CSC CSCH
57
- DECIMAL DEGREES EVEN EXP FACT FACTDOUBLE FLOOR FLOOR.MATH FLOOR.PRECISE GCD
58
- INT ISO.CEILING LCM LN LOG LOG10 MDETERM MINVERSE MMULT MOD MROUND MULTINOMIAL
59
- MUNIT ODD PI POWER PRODUCT QUOTIENT RADIANS ROMAN ROUND ROUNDDOWN ROUNDUP SEC
60
- SECH SIGN SIN SINH SQRT SUM SUMIF SUMIFS SUMPRODUCT SUMSQ TAN TANH TRUNC
61
- ACCRINT ACCRINTM AMORDEGRC AMORLINC COUPDAYBS COUPDAYS COUPDAYSNC COUPNCD
62
- COUPNUM COUPPCD CUMIPMT CUMPRINC DB DDB DISC DOLLARDE DOLLARFR DURATION EFFECT
63
- FV FVSCHEDULE INTRATE IPMT IRR ISPMT MDURATION MIRR NOMINAL NPER NPV PDURATION
64
- PMT PPMT PRICE PRICEDISC PRICEMAT PV RATE RECEIVED RRI SLN SYD TBILLEQ TBILLPRICE
65
- TBILLYIELD VDB XIRR XNPV YIELD YIELDDISC YIELDMAT
66
- AND FALSE IF IFERROR IFNA NOT OR TRUE XOR SWITCH IFS CHOOSE
67
- ADDRESS AREAS COLUMN COLUMNS FORMULATEXT HLOOKUP HYPERLINK INDEX INDIRECT
68
- LOOKUP MATCH ROW ROWS TRANSPOSE VLOOKUP OFFSET
69
- CELL ERROR.TYPE INFO ISBLANK ISERR ISERROR ISEVEN ISFORMULA ISLOGICAL ISNA
70
- ISNONTEXT ISNUMBER ISODD ISREF ISTEXT N TYPE NA
71
- CONCAT CONCATENATE EXACT FIND FINDB LEFT LEFTB LEN LENB LOWER MID MIDB NUMBERVALUE
72
- PHONETIC PROPER REPLACE REPLACEB REPT RIGHT RIGHTB SEARCH SEARCHB SUBSTITUTE T
73
- TEXT TEXTAFTER TEXTBEFORE TEXTJOIN TEXTSPLIT TRIM UNICHAR UNICODE UPPER VALUE
74
- ASC CHAR CLEAN CODE DBCS FIXED JIS LOWER UPPER
75
- DATE DATEVALUE DAY DAYS DAYS360 EDATE EOMONTH HOUR ISOWEEKNUM MINUTE MONTH
76
- NETWORKDAYS NETWORKDAYS.INTL SECOND TIME TIMEVALUE WEEKDAY WEEKNUM WORKDAY
77
- WORKDAY.INTL YEAR YEARFRAC
78
- COUNT COUNTA COUNTBLANK COUNTIF COUNTIFS MAX MAXIFS MIN MINIFS AVERAGE
79
- AVERAGEIF AVERAGEIFS MEDIAN MODE.MULT MODE.SNGL LARGE SMALL RANK.EQ RANK.AVG
80
- PERCENTILE.EXC PERCENTILE.INC PERCENTRANK.EXC PERCENTRANK.INC QUARTILE.EXC
81
- QUARTILE.INC STANDARDIZE STDEV.P STDEV.S STDEVA STDEVPA VAR.P VAR.S VARA VARPA
82
- CORREL COVARIANCE.P COVARIANCE.S GEOMEAN HARMEAN KURT SKEW SKEW.P SLOPE
83
- INTERCEPT FORECAST FORECAST.LINEAR FREQUENCY GROWTH LINEST LOGEST PEARSON RSQ
84
- TREND
85
- DEC2BIN DEC2HEX DEC2OCT HEX2BIN HEX2DEC HEX2OCT OCT2BIN OCT2DEC OCT2HEX
86
- BITAND BITLSHIFT BITOR BITRSHIFT BITXOR CONVERT DELTA ERF ERF.PRECISE ERFC
87
- ERFC.PRECISE GESTEP IMSUM IMABS IMAGINARY IMARGUMENT IMCONJUGATE IMCOS IMCOSH
88
- IMCOT IMCSC IMCSCH IMDIV IMEXP IMLN IMLOG10 IMLOG2 IMPOWER IMPRODUCT IMREAL
89
- IMSEC IMSECH IMSIN IMSINH IMSQRT IMSUB IMTAN IMTANH COMPLEX
90
- CUBEMEMBER CUBEVALUE GETPIVOTDATA
91
- LET LAMBDA MAP REDUCE SCAN MAKEARRAY BYROW BYCOL
92
- """.split()
93
- }
94
-
95
- _FUNCTION_CALL_RE = re.compile(r"(?<![A-Za-z0-9_.])([A-Za-z_][A-Za-z0-9_.]*)\s*\(")
96
- _EXTERNAL_REF_RE = re.compile(r"\[[^\]]+\][^!]*!")
97
- _DYNAMIC_SPILL_RE = re.compile(r"(?:[A-Z]{1,3}\$?\d+|\))#(?=$|[\s,;)])", re.IGNORECASE)
98
-
99
-
100
- def _powershell_executable() -> str:
101
- # Windows PowerShell is part of the supported host. Do not fall back to a
102
- # different Python/COM package; pywin32 is intentionally not required.
103
- return shutil.which("powershell.exe") or shutil.which("powershell") or "powershell.exe"
104
-
105
-
106
- def _formula_text(value: Any) -> str | None:
107
- if isinstance(value, str) and value.startswith("="):
108
- return value
109
- return None
110
-
111
-
112
- def _without_string_literals(formula: str) -> str:
113
- """Blank double-quoted Excel string literals before syntax/security scans."""
114
- out: list[str] = []
115
- quoted = False
116
- index = 0
117
- while index < len(formula):
118
- char = formula[index]
119
- if char == '"':
120
- if quoted and index + 1 < len(formula) and formula[index + 1] == '"':
121
- out.extend(" ")
122
- index += 2
123
- continue
124
- quoted = not quoted
125
- out.append(" ")
126
- else:
127
- out.append(" " if quoted else char)
128
- index += 1
129
- return "".join(out)
130
-
131
-
132
- def _iter_formula_cells(workbook: Any) -> Iterable[tuple[Any, Any]]:
133
- for worksheet in workbook.worksheets:
134
- for row in worksheet.iter_rows():
135
- for cell in row:
136
- # A literal string beginning with '=' is data_type='s' and is
137
- # deliberately not a formula. Only Excel's formula cell type
138
- # (including openpyxl ArrayFormula objects) enters this path.
139
- if cell.data_type == "f":
140
- yield worksheet, cell
141
-
142
-
143
- def _scan_package(path: Path) -> None:
144
- """Reject package features that can cause I/O or code execution in Excel."""
145
- try:
146
- with zipfile.ZipFile(path) as archive:
147
- archive_names = {name.replace("\\", "/").lower(): name for name in archive.namelist()}
148
- names = set(archive_names)
149
- dangerous_parts = (
150
- "xl/externallinks/", "xl/connections.xml", "xl/querytables/",
151
- "xl/ddelinks/", "xl/macrosheets/", "xl/dialogsheets/",
152
- )
153
- for part in dangerous_parts:
154
- if any(name.startswith(part) or name == part.rstrip("/") for name in names):
155
- raise FormulaValueError(f"Workbook contains unsupported external or executable part: {part}")
156
-
157
- for name in names:
158
- if not name.endswith(".xml") and not name.endswith(".rels"):
159
- continue
160
- raw = archive.read(archive_names[name])
161
- lowered = raw.lower()
162
- if b"<ddelink" in lowered or b"<olelink" in lowered:
163
- raise FormulaValueError(f"Workbook contains DDE/OLE link data: {name}")
164
- if name.endswith(".rels") and b'targetmode="external"' in lowered:
165
- # External hyperlinks are harmless, but an external target
166
- # in workbook/externalLink relationships is not. Reject
167
- # only relationship files whose type names indicate a link
168
- # or connection, avoiding false positives for user URLs.
169
- if b"externallink" in lowered or b"externalconnection" in lowered:
170
- raise FormulaValueError(f"Workbook contains an external relationship: {name}")
171
- # Shared and array formulas are represented by attributes that are
172
- # not faithfully round-trippable through openpyxl. Refuse the
173
- # whole workbook rather than replacing only the anchor and leaving
174
- # a silently wrong spill/shared range.
175
- for name in names:
176
- if not name.startswith("xl/worksheets/") or not name.endswith(".xml"):
177
- continue
178
- root = ElementTree.fromstring(archive.read(archive_names[name]))
179
- for formula_node in root.iter():
180
- if formula_node.tag.rsplit("}", 1)[-1] != "f":
181
- continue
182
- attrs = {key.rsplit("}", 1)[-1]: value for key, value in formula_node.attrib.items()}
183
- kind = attrs.get("t", "").lower()
184
- if kind in {"shared", "array", "datatable"} or "ref" in attrs:
185
- raise FormulaValueError(f"Unsupported shared/array formula in {name}")
186
- if formula_node.text is None and kind:
187
- raise FormulaValueError(f"Unsupported formula without body in {name}")
188
- except ElementTree.ParseError as exc:
189
- raise FormulaValueError(f"Cannot inspect workbook XML safely: {path}") from exc
190
- except (OSError, zipfile.BadZipFile) as exc:
191
- raise FormulaValueError(f"Cannot inspect workbook package: {path}") from exc
192
-
193
-
194
- def _scan_formulas(workbook: Any) -> list[tuple[str, str, str]]:
195
- formulas: list[tuple[str, str, str]] = []
196
- for worksheet, cell in _iter_formula_cells(workbook):
197
- formula = cell.value.text if hasattr(cell.value, "text") else cell.value
198
- if not isinstance(formula, str):
199
- raise FormulaValueError(f"Unsupported formula object at {worksheet.title}!{cell.coordinate}")
200
- if not formula.startswith("="):
201
- formula = "=" + formula
202
- scan_formula = _without_string_literals(formula)
203
- upper = scan_formula.upper()
204
- if _EXTERNAL_REF_RE.search(scan_formula) or re.search(r"\bDDE\s*\(", scan_formula, re.IGNORECASE):
205
- raise FormulaValueError(f"External workbook/DDE reference at {worksheet.title}!{cell.coordinate}")
206
- if any(token in upper for token in ("_XLFN.", "_XLWS.")):
207
- raise FormulaValueError(f"Unsupported modern/dynamic formula at {worksheet.title}!{cell.coordinate}")
208
- if _DYNAMIC_SPILL_RE.search(scan_formula) or "@" in scan_formula:
209
- raise FormulaValueError(f"Dynamic/implicit-array formula at {worksheet.title}!{cell.coordinate}")
210
- if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(name)}\s*\(", upper) for name in _VOLATILE_FUNCTIONS):
211
- raise FormulaValueError(f"Volatile formula is not reproducible at {worksheet.title}!{cell.coordinate}")
212
- if "#REF!" in upper:
213
- raise FormulaValueError(f"Broken formula reference at {worksheet.title}!{cell.coordinate}")
214
- for match in _FUNCTION_CALL_RE.finditer(scan_formula):
215
- name = match.group(1).upper()
216
- if name in _DYNAMIC_FUNCTIONS:
217
- raise FormulaValueError(f"Dynamic-array formula at {worksheet.title}!{cell.coordinate}")
218
- if name.startswith("_XLFN.") or name.startswith("_XLWS.") or name not in _BUILTIN_FUNCTIONS:
219
- raise FormulaValueError(f"Unknown/UDF formula function {name} at {worksheet.title}!{cell.coordinate}")
220
- formulas.append((worksheet.title, cell.coordinate, formula))
221
- _detect_formula_cycles(workbook)
222
- return formulas
223
-
224
-
225
- def _detect_formula_cycles(workbook: Any) -> None:
226
- """Fail closed on cycles or references we cannot statically account for."""
227
- nodes = {(ws.title, cell.coordinate): cell for ws, cell in _iter_formula_cells(workbook)}
228
- graph = {}
229
- sheet_names = {ws.title.casefold(): ws.title for ws in workbook.worksheets}
230
- global_names = {name.casefold(): value for name, value in workbook.defined_names.items()}
231
- local_names = {ws.title: {name.casefold(): value for name, value in ws.defined_names.items()}
232
- for ws in workbook.worksheets}
233
- by_sheet = {}
234
- for key, cell in nodes.items():
235
- by_sheet.setdefault(key[0], []).append((key, cell.row, cell.column))
236
- ref_re = re.compile(r"^(?:\$?[A-Z]{1,3}\$?\d+(?::\$?[A-Z]{1,3}\$?\d+)?|\$?[A-Z]{1,3}:\$?[A-Z]{1,3}|\$?\d+:\$?\d+)$", re.I)
237
-
238
- def dependencies(formula, context_sheet, name_stack=()):
239
- try:
240
- tokens = Tokenizer(formula if formula.startswith('=') else '=' + formula).items
241
- except Exception as exc:
242
- raise FormulaValueError(f"Unable to parse formula dependencies in {context_sheet}") from exc
243
- result = set()
244
- for token in tokens:
245
- if token.type != "OPERAND" or token.subtype != "RANGE":
246
- continue
247
- raw = token.value.strip()
248
- if "!" in raw:
249
- sheet_text, raw_ref = raw.rsplit("!", 1)
250
- if ":" in sheet_text:
251
- raise FormulaValueError(f"Unsupported 3-D formula reference: {raw}")
252
- if sheet_text.startswith("'") and sheet_text.endswith("'"):
253
- sheet_text = sheet_text[1:-1].replace("''", "'")
254
- target_sheet = sheet_names.get(sheet_text.casefold())
255
- if target_sheet is None:
256
- raise FormulaValueError(f"Unknown formula reference sheet: {raw}")
257
- else:
258
- raw_ref, target_sheet = raw, context_sheet
259
- if ref_re.fullmatch(raw_ref):
260
- # Relative references in defined names are anchored to Excel's
261
- # active-cell context, not necessarily the formula cell. Do
262
- # not guess that context when checking dependencies.
263
- if name_stack and not re.fullmatch(
264
- r'(?:\$[A-Z]{1,3}\$\d+(?::\$[A-Z]{1,3}\$\d+)?|\$[A-Z]{1,3}:\$[A-Z]{1,3}|\$\d+:\$\d+)',
265
- raw_ref, re.I,
266
- ):
267
- raise FormulaValueError(f"Relative defined-name reference is unsupported: {raw}")
268
- min_col, min_row, max_col, max_row = range_boundaries(raw_ref.upper())
269
- min_col, min_row = min_col or 1, min_row or 1
270
- max_col, max_row = max_col or 16384, max_row or 1048576
271
- if not (1 <= min_col <= max_col <= 16384 and 1 <= min_row <= max_row <= 1048576):
272
- raise FormulaValueError(f"Invalid formula range: {raw}")
273
- result.update(key for key, row, col in by_sheet.get(target_sheet, [])
274
- if min_col <= col <= max_col and min_row <= row <= max_row)
275
- else:
276
- name_key = raw_ref.casefold()
277
- defined = local_names.get(target_sheet, {}).get(name_key) or global_names.get(name_key)
278
- if defined is None:
279
- raise FormulaValueError(f"Unsupported or unknown formula reference: {raw}")
280
- identity = (target_sheet, name_key)
281
- if identity in name_stack or len(name_stack) >= 100:
282
- raise FormulaValueError(f"Circular or excessively nested defined name: {raw}")
283
- result.update(dependencies(str(defined.attr_text or ''), target_sheet, (*name_stack, identity)))
284
- return result
285
-
286
- for node, cell in nodes.items():
287
- formula = cell.value.text if hasattr(cell.value, "text") else str(cell.value)
288
- graph[node] = dependencies(formula, node[0])
289
- # Iterative topological elimination supports long formula chains.
290
- reverse = {node: set() for node in graph}
291
- counts = {node: len(edges) for node, edges in graph.items()}
292
- for node, edges in graph.items():
293
- for edge in edges:
294
- reverse[edge].add(node)
295
- ready = [node for node, count in counts.items() if count == 0]
296
- visited = 0
297
- while ready:
298
- node = ready.pop()
299
- visited += 1
300
- for dependent in reverse[node]:
301
- counts[dependent] -= 1
302
- if counts[dependent] == 0:
303
- ready.append(dependent)
304
- if visited != len(graph):
305
- node = next(node for node, count in counts.items() if count)
306
- raise FormulaValueError(f"Circular formula dependency detected at {node[0]}!{node[1]}")
307
-
308
-
309
- def _scan_defined_names(workbook: Any) -> None:
310
- """Reject names which smuggle an external link or executable reference."""
311
- all_names = [*workbook.defined_names.values(),
312
- *(name for ws in workbook.worksheets for name in ws.defined_names.values())]
313
- for defined_name in all_names:
314
- text = str(getattr(defined_name, "attr_text", "") or "")
315
- scan_text = _without_string_literals(text)
316
- scan_upper = scan_text.upper()
317
- if _EXTERNAL_REF_RE.search(scan_text) or re.search(r"\bDDE\s*\(", scan_text, re.IGNORECASE) or "#REF!" in scan_upper:
318
- name = getattr(defined_name, "name", "<unnamed>")
319
- raise FormulaValueError(f"Defined name contains an unsafe reference: {name}")
320
- name = getattr(defined_name, "name", "<unnamed>")
321
- if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(function)}\s*\(", scan_upper)
322
- for function in _VOLATILE_FUNCTIONS):
323
- raise FormulaValueError(f"Defined name contains a volatile formula: {name}")
324
- if _DYNAMIC_SPILL_RE.search(scan_text) or "@" in scan_text:
325
- raise FormulaValueError(f"Defined name contains a dynamic-array formula: {name}")
326
- for match in _FUNCTION_CALL_RE.finditer(scan_text):
327
- function = match.group(1).upper()
328
- if function in _DYNAMIC_FUNCTIONS or function.startswith(("_XLFN.", "_XLWS.")) \
329
- or function not in _BUILTIN_FUNCTIONS:
330
- raise FormulaValueError(f"Defined name contains unknown/UDF function {function}: {name}")
331
-
332
-
333
- def _read_state(path: Path) -> dict[str, Any]:
334
- try:
335
- text = path.read_text(encoding="utf-8-sig")
336
- value = json.loads(text)
337
- return value if isinstance(value, dict) else {}
338
- except (OSError, ValueError, json.JSONDecodeError):
339
- return {}
340
-
341
-
342
- def _cleanup_excel_process(state_path: Path) -> None:
343
- """Stop only the Excel PID and start time recorded by our PS harness."""
344
- state = _read_state(state_path)
345
- pid = state.get("pid")
346
- start_filetime = state.get("startFileTimeUtc")
347
- if not isinstance(pid, int) or not isinstance(start_filetime, int):
348
- return
349
- helper = Path(__file__).with_name("recalculate_workbook.ps1")
350
- try:
351
- # Pass ownership data as JSON, never as PowerShell source/arguments.
352
- # The helper validates both PID and process creation time before stop.
353
- subprocess.run(
354
- [_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
355
- "-File", str(helper), "-CleanupParametersPath", str(state_path)],
356
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=False,
357
- )
358
- except (OSError, subprocess.TimeoutExpired):
359
- pass
360
-
361
-
362
- def _run_excel_recalculate(source: Path, result: Path, timeout_seconds: int) -> None:
363
- script = Path(__file__).with_name("recalculate_workbook.ps1")
364
- if not script.is_file():
365
- raise FormulaValueError(f"Excel recalculation helper is missing: {script}")
366
- with tempfile.TemporaryDirectory(prefix="xlmerge-excel-state-") as state_dir:
367
- state_path = Path(state_dir) / "state.json"
368
- params_path = Path(state_dir) / "params.json"
369
- params_path.write_text(json.dumps({
370
- "source": str(source), "result": str(result), "state": str(state_path),
371
- "timeoutSeconds": int(timeout_seconds),
372
- }), encoding="utf-8")
373
- command = [_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
374
- "-File", str(script), "-ParametersPath", str(params_path)]
375
- try:
376
- completed = subprocess.run(command, capture_output=True, text=True,
377
- encoding="utf-8", errors="replace",
378
- timeout=max(1, int(timeout_seconds) + 20), check=False)
379
- except subprocess.TimeoutExpired as exc:
380
- _cleanup_excel_process(state_path)
381
- raise FormulaValueError("Excel recalculation exceeded the hard timeout") from exc
382
- finally:
383
- # The helper normally closes Excel itself. This also handles a
384
- # PowerShell error path without ever scanning/killing user Excel.
385
- if state_path.exists() and _read_state(state_path).get("status") not in {"completed", "failed"}:
386
- _cleanup_excel_process(state_path)
387
- if completed.returncode != 0:
388
- state = _read_state(state_path)
389
- detail = state.get("error") or completed.stderr.strip() or completed.stdout.strip()
390
- raise FormulaValueError(f"Excel could not safely calculate workbook: {detail or 'unknown error'}")
391
- if not result.is_file() or result.stat().st_size == 0:
392
- raise FormulaValueError("Excel reported success but did not export a result workbook")
393
-
394
-
395
- def _set_literal(cell: Any, value: Any) -> None:
396
- """Assign a value without allowing a string beginning '=' to become a formula."""
397
- if isinstance(value, str):
398
- # openpyxl classifies strings such as '#N/A' as error cells and strings
399
- # beginning '=' as formulas. Cached formula text is data, regardless
400
- # of its spelling, so force the serialized string type for all text.
401
- cell.data_type = "s"
402
- if value == "":
403
- # openpyxl normally omits an empty string cell entirely on write,
404
- # which reloads as None. A zero-length rich-text run is a valid
405
- # inline string and round-trips as a typed empty string.
406
- cell._value = CellRichText(TextBlock(InlineFont(), ""))
407
- else:
408
- cell._value = value
409
- else:
410
- cell.value = value
411
-
412
-
413
- def _validate_result(value: Any, worksheet: str, coordinate: str, data_type: str | None = None) -> None:
414
- # Use the serialized cell type, not the displayed text: a formula may
415
- # intentionally return the literal string "#N/A", while an Excel error is
416
- # represented by cell type ``e``.
417
- if data_type == "e":
418
- raise FormulaValueError(f"Excel returned an error value at {worksheet}!{coordinate}: {value}")
419
- if isinstance(value, float) and not math.isfinite(value):
420
- raise FormulaValueError(f"Excel returned non-finite numeric value at {worksheet}!{coordinate}")
421
- if value is not None and not isinstance(value, (str, bool, int, float, date, datetime, datetime_time)):
422
- raise FormulaValueError(f"Unsupported Excel result type at {worksheet}!{coordinate}: {type(value).__name__}")
423
-
424
-
425
- def _formula_cache_presence(path: Path) -> dict[tuple[str, str], bool]:
426
- """Return whether each formula cell has a serialized ``<v>`` cache.
427
-
428
- Excel writes an empty-string result as ``<v></v>`` (which openpyxl exposes
429
- as ``None`` on some Excel versions), whereas an uncalculated formula has no
430
- ``<v>`` element. Keeping this distinction is essential for fail-closed
431
- calculation verification.
432
- """
433
- result: dict[tuple[str, str], bool] = {}
434
- try:
435
- with zipfile.ZipFile(path) as archive:
436
- wb_root = ElementTree.fromstring(archive.read("xl/workbook.xml"))
437
- rel_root = ElementTree.fromstring(archive.read("xl/_rels/workbook.xml.rels"))
438
- rel_targets = {}
439
- for rel in rel_root:
440
- rid = rel.attrib.get("Id") or rel.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
441
- target = rel.attrib.get("Target")
442
- if rid and target:
443
- rel_targets[rid] = "xl/" + target.lstrip("/") if not target.startswith("/") else target.lstrip("/")
444
- for sheet in wb_root.iter():
445
- if sheet.tag.rsplit("}", 1)[-1] != "sheet":
446
- continue
447
- sheet_name = sheet.attrib.get("name")
448
- rid = sheet.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
449
- target = rel_targets.get(rid)
450
- if not sheet_name or not target or target not in archive.namelist():
451
- continue
452
- root = ElementTree.fromstring(archive.read(target))
453
- for cell in root.iter():
454
- if cell.tag.rsplit("}", 1)[-1] != "c" or not cell.attrib.get("r"):
455
- continue
456
- if any(child.tag.rsplit("}", 1)[-1] == "f" for child in cell):
457
- result[(sheet_name, cell.attrib["r"])] = any(
458
- child.tag.rsplit("}", 1)[-1] == "v" for child in cell
459
- )
460
- except (OSError, KeyError, zipfile.BadZipFile, ElementTree.ParseError):
461
- return {}
462
- return result
463
-
464
-
465
- def _materialize_from_result(source: Path, calculated: Path, output: Path,
466
- formula_keys: set[tuple[str, str]]) -> Path:
467
- keep_vba = source.suffix.lower() == ".xlsm"
468
- try:
469
- original = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
470
- values = load_workbook(calculated, data_only=True, keep_links=False, keep_vba=keep_vba)
471
- except Exception as exc:
472
- raise FormulaValueError(f"Cannot read Excel calculation result: {exc}") from exc
473
- try:
474
- cache_presence = _formula_cache_presence(calculated)
475
- for sheet_name, coordinate in formula_keys:
476
- if sheet_name not in values.sheetnames or sheet_name not in original.sheetnames:
477
- raise FormulaValueError(f"Calculation result lost sheet {sheet_name!r}")
478
- source_cell = original[sheet_name][coordinate]
479
- result_cell = values[sheet_name][coordinate]
480
- value = result_cell.value
481
- _validate_result(value, sheet_name, coordinate, result_cell.data_type)
482
- # A missing <v> cache is different from a legitimate cached empty
483
- # string. Excel's data_only reader returns None in both unusual
484
- # cases; normal formulas returning "" have a string cache and are
485
- # accepted. Missing cache is rejected by the post-save formula
486
- # count/result verification below if Excel did not calculate it.
487
- if value is None:
488
- if result_cell.data_type not in ('s', 'str') or not cache_presence.get((sheet_name, coordinate), False):
489
- raise FormulaValueError(f"Excel did not provide a cached result at {sheet_name}!{coordinate}")
490
- # Excel's empty-string cache can be represented by an empty
491
- # <v> with no Python value. It is still a string result.
492
- value = ""
493
- _set_literal(source_cell, value)
494
-
495
- output.parent.mkdir(parents=True, exist_ok=True)
496
- temp_output = output.with_name(f".{output.stem}.tmp-{os.getpid()}-{time.time_ns()}{output.suffix}")
497
- try:
498
- original.save(temp_output)
499
- check = load_workbook(temp_output, data_only=False, keep_links=False, keep_vba=keep_vba)
500
- try:
501
- leftovers = [(ws.title, c.coordinate) for ws, c in _iter_formula_cells(check)]
502
- finally:
503
- check.close()
504
- if leftovers:
505
- raise FormulaValueError(f"Formula values remained after materialization: {leftovers[:3]}")
506
- os.replace(temp_output, output)
507
- finally:
508
- if temp_output.exists():
509
- temp_output.unlink(missing_ok=True)
510
- finally:
511
- original.close()
512
- values.close()
513
- return output
514
-
515
-
516
- def materialize_values(source_path: Path | str, output_path: Path | str) -> Path:
517
- """Calculate every formula in ``source_path`` and emit a literal snapshot.
518
-
519
- The source is never opened by Excel and is never modified. A workbook with
520
- no formulas is returned unchanged, as an explicit fast path.
521
- """
522
- source = Path(source_path).resolve()
523
- output = Path(output_path).resolve()
524
- if not source.is_file():
525
- raise FormulaValueError(f"Workbook does not exist: {source}")
526
- if source == output:
527
- raise FormulaValueError("Output path must differ from source path")
528
- if source.suffix.lower() not in {".xlsx", ".xlsm"}:
529
- raise FormulaValueError(f"Unsupported workbook extension: {source.suffix}")
530
- if output.suffix.lower() != source.suffix.lower():
531
- raise FormulaValueError("Output extension must match source extension")
532
-
533
- _scan_package(source)
534
- keep_vba = source.suffix.lower() == ".xlsm"
535
- try:
536
- workbook = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
537
- except Exception as exc:
538
- raise FormulaValueError(f"Cannot open workbook for formula inspection: {exc}") from exc
539
- try:
540
- formulas = _scan_formulas(workbook)
541
- _scan_defined_names(workbook)
542
- finally:
543
- workbook.close()
544
- if not formulas:
545
- return source
546
-
547
- timeout_setting = os.environ.get("XLMERGE_EXCEL_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))
548
- try:
549
- timeout_seconds = max(1, int(timeout_setting))
550
- except ValueError as exc:
551
- raise FormulaValueError("XLMERGE_EXCEL_TIMEOUT_SECONDS must be an integer") from exc
552
-
553
- temp_dir = Path(tempfile.mkdtemp(prefix="xlmerge-formula-values-"))
554
- temporary_source = temp_dir / source.name
555
- temporary_result = temp_dir / ("calculated" + source.suffix.lower())
556
- try:
557
- shutil.copy2(source, temporary_source)
558
- _run_excel_recalculate(temporary_source, temporary_result, timeout_seconds)
559
- return _materialize_from_result(source, temporary_result, output,
560
- {(sheet, coordinate) for sheet, coordinate, _ in formulas})
561
- finally:
562
- shutil.rmtree(temp_dir, ignore_errors=True)
563
-
564
-
565
- __all__ = ["FormulaValueError", "materialize_values"]
1
+ """Safe, Excel-backed formula materialisation for the conflict resolver.
2
+
3
+ This module deliberately has a small trust boundary. Excel is only ever given a
4
+ temporary copy, with links and macros disabled. The result produced by Excel is
5
+ read back as cached values and written into a copy of the original package with
6
+ openpyxl. We do not use ``data_only`` as a source workbook: a stale cache is not
7
+ an acceptable answer for a conflict resolver.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import math
14
+ import os
15
+ import re
16
+ import shutil
17
+ import subprocess
18
+ import tempfile
19
+ import time
20
+ import zipfile
21
+ from datetime import date, datetime, time as datetime_time
22
+ from pathlib import Path
23
+ from typing import Any, Iterable
24
+ from xml.etree import ElementTree
25
+
26
+ from openpyxl import load_workbook
27
+ from openpyxl.cell.rich_text import CellRichText, InlineFont, TextBlock
28
+ from openpyxl.formula import Tokenizer
29
+ from openpyxl.formula.translate import Translator
30
+ from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries
31
+
32
+
33
+ class FormulaValueError(RuntimeError):
34
+ """Raised when a workbook cannot be evaluated and verified safely."""
35
+
36
+
37
+ DEFAULT_TIMEOUT_SECONDS = 120
38
+ _ERROR_VALUES = {"#NULL!", "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A", "#GETTING_DATA"}
39
+ _VOLATILE_FUNCTIONS = {
40
+ "NOW", "TODAY", "RAND", "RANDBETWEEN", "RANDARRAY", "RTD", "OFFSET", "INDIRECT",
41
+ "CELL", "INFO",
42
+ }
43
+ _DYNAMIC_FUNCTIONS = {
44
+ "FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE", "RANDARRAY", "TRANSPOSE",
45
+ "TOCOL", "TOROW", "TAKE", "DROP", "CHOOSECOLS", "CHOOSEROWS", "EXPAND",
46
+ "VSTACK", "HSTACK", "WRAPROWS", "WRAPCOLS", "XLOOKUP", "XMATCH", "LAMBDA",
47
+ }
48
+
49
+ # Function names are checked before opening Excel. This is intentionally a
50
+ # broad list of ordinary worksheet functions, but unknown calls are rejected so
51
+ # a disabled VBA UDF can never be silently accepted merely because it returned a
52
+ # plausible value in a cached workbook.
53
+ _BUILTIN_FUNCTIONS = {
54
+ name
55
+ for name in """
56
+ ABS ACOS ACOSH ACOT ACOTH AGGREGATE ARABIC ASIN ASINH ATAN ATAN2 ATANH BASE
57
+ CEILING CEILING.MATH CEILING.PRECISE COMBIN COMBINA COS COSH COT COTH CSC CSCH
58
+ DECIMAL DEGREES EVEN EXP FACT FACTDOUBLE FLOOR FLOOR.MATH FLOOR.PRECISE GCD
59
+ INT ISO.CEILING LCM LN LOG LOG10 MDETERM MINVERSE MMULT MOD MROUND MULTINOMIAL
60
+ MUNIT ODD PI POWER PRODUCT QUOTIENT RADIANS ROMAN ROUND ROUNDDOWN ROUNDUP SEC
61
+ SECH SIGN SIN SINH SQRT SUM SUMIF SUMIFS SUMPRODUCT SUMSQ TAN TANH TRUNC
62
+ ACCRINT ACCRINTM AMORDEGRC AMORLINC COUPDAYBS COUPDAYS COUPDAYSNC COUPNCD
63
+ COUPNUM COUPPCD CUMIPMT CUMPRINC DB DDB DISC DOLLARDE DOLLARFR DURATION EFFECT
64
+ FV FVSCHEDULE INTRATE IPMT IRR ISPMT MDURATION MIRR NOMINAL NPER NPV PDURATION
65
+ PMT PPMT PRICE PRICEDISC PRICEMAT PV RATE RECEIVED RRI SLN SYD TBILLEQ TBILLPRICE
66
+ TBILLYIELD VDB XIRR XNPV YIELD YIELDDISC YIELDMAT
67
+ AND FALSE IF IFERROR IFNA NOT OR TRUE XOR SWITCH IFS CHOOSE
68
+ ADDRESS AREAS COLUMN COLUMNS FORMULATEXT HLOOKUP HYPERLINK INDEX INDIRECT
69
+ LOOKUP MATCH ROW ROWS TRANSPOSE VLOOKUP OFFSET
70
+ CELL ERROR.TYPE INFO ISBLANK ISERR ISERROR ISEVEN ISFORMULA ISLOGICAL ISNA
71
+ ISNONTEXT ISNUMBER ISODD ISREF ISTEXT N TYPE NA
72
+ CONCAT CONCATENATE EXACT FIND FINDB LEFT LEFTB LEN LENB LOWER MID MIDB NUMBERVALUE
73
+ PHONETIC PROPER REPLACE REPLACEB REPT RIGHT RIGHTB SEARCH SEARCHB SUBSTITUTE T
74
+ TEXT TEXTAFTER TEXTBEFORE TEXTJOIN TEXTSPLIT TRIM UNICHAR UNICODE UPPER VALUE
75
+ ASC CHAR CLEAN CODE DBCS FIXED JIS LOWER UPPER
76
+ DATE DATEVALUE DAY DAYS DAYS360 EDATE EOMONTH HOUR ISOWEEKNUM MINUTE MONTH
77
+ NETWORKDAYS NETWORKDAYS.INTL SECOND TIME TIMEVALUE WEEKDAY WEEKNUM WORKDAY
78
+ WORKDAY.INTL YEAR YEARFRAC
79
+ COUNT COUNTA COUNTBLANK COUNTIF COUNTIFS MAX MAXIFS MIN MINIFS AVERAGE
80
+ AVERAGEIF AVERAGEIFS MEDIAN MODE.MULT MODE.SNGL LARGE SMALL RANK.EQ RANK.AVG
81
+ PERCENTILE.EXC PERCENTILE.INC PERCENTRANK.EXC PERCENTRANK.INC QUARTILE.EXC
82
+ QUARTILE.INC STANDARDIZE STDEV.P STDEV.S STDEVA STDEVPA VAR.P VAR.S VARA VARPA
83
+ CORREL COVARIANCE.P COVARIANCE.S GEOMEAN HARMEAN KURT SKEW SKEW.P SLOPE
84
+ INTERCEPT FORECAST FORECAST.LINEAR FREQUENCY GROWTH LINEST LOGEST PEARSON RSQ
85
+ TREND
86
+ DEC2BIN DEC2HEX DEC2OCT HEX2BIN HEX2DEC HEX2OCT OCT2BIN OCT2DEC OCT2HEX
87
+ BITAND BITLSHIFT BITOR BITRSHIFT BITXOR CONVERT DELTA ERF ERF.PRECISE ERFC
88
+ ERFC.PRECISE GESTEP IMSUM IMABS IMAGINARY IMARGUMENT IMCONJUGATE IMCOS IMCOSH
89
+ IMCOT IMCSC IMCSCH IMDIV IMEXP IMLN IMLOG10 IMLOG2 IMPOWER IMPRODUCT IMREAL
90
+ IMSEC IMSECH IMSIN IMSINH IMSQRT IMSUB IMTAN IMTANH COMPLEX
91
+ CUBEMEMBER CUBEVALUE GETPIVOTDATA
92
+ LET LAMBDA MAP REDUCE SCAN MAKEARRAY BYROW BYCOL
93
+ """.split()
94
+ }
95
+
96
+ _FUNCTION_CALL_RE = re.compile(r"(?<![A-Za-z0-9_.])([A-Za-z_][A-Za-z0-9_.]*)\s*\(")
97
+ _EXTERNAL_REF_RE = re.compile(r"\[[^\]]+\][^!]*!")
98
+ _DYNAMIC_SPILL_RE = re.compile(r"(?:[A-Z]{1,3}\$?\d+|\))#(?=$|[\s,;)])", re.IGNORECASE)
99
+
100
+
101
+ def _powershell_executable() -> str:
102
+ # Windows PowerShell is part of the supported host. Do not fall back to a
103
+ # different Python/COM package; pywin32 is intentionally not required.
104
+ return shutil.which("powershell.exe") or shutil.which("powershell") or "powershell.exe"
105
+
106
+
107
+ def _formula_text(value: Any) -> str | None:
108
+ if isinstance(value, str) and value.startswith("="):
109
+ return value
110
+ return None
111
+
112
+
113
+ def _without_string_literals(formula: str) -> str:
114
+ """Blank double-quoted Excel string literals before syntax/security scans."""
115
+ out: list[str] = []
116
+ quoted = False
117
+ index = 0
118
+ while index < len(formula):
119
+ char = formula[index]
120
+ if char == '"':
121
+ if quoted and index + 1 < len(formula) and formula[index + 1] == '"':
122
+ out.extend(" ")
123
+ index += 2
124
+ continue
125
+ quoted = not quoted
126
+ out.append(" ")
127
+ else:
128
+ out.append(" " if quoted else char)
129
+ index += 1
130
+ return "".join(out)
131
+
132
+
133
+ def _iter_formula_cells(workbook: Any) -> Iterable[tuple[Any, Any]]:
134
+ for worksheet in workbook.worksheets:
135
+ # These are normal, fully loaded worksheets. Inspect stored cells,
136
+ # rather than constructing the rectangular formatted UsedRange.
137
+ for cell in worksheet._cells.values():
138
+ if cell.data_type == "f":
139
+ yield worksheet, cell
140
+
141
+
142
+ def _scan_sheet_formulas(root: Any, sheet_part: str) -> None:
143
+ """Validate shared groups before openpyxl expands their relative references."""
144
+ groups: dict[str, list[tuple[str, Any]]] = {}
145
+ coordinates: set[str] = set()
146
+ for cell in root.iter():
147
+ if cell.tag.rsplit("}", 1)[-1] != "c":
148
+ continue
149
+ coordinate = cell.get("r", "")
150
+ formula_nodes = [node for node in cell if node.tag.rsplit("}", 1)[-1] == "f"]
151
+ if not formula_nodes:
152
+ continue
153
+ location = f"{sheet_part}!{coordinate or '<missing coordinate>'}"
154
+ if not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]*", coordinate):
155
+ raise FormulaValueError(f"Invalid formula coordinate at {location}")
156
+ row, column = coordinate_to_tuple(coordinate)
157
+ if row > 1048576 or column > 16384 or coordinate in coordinates or len(formula_nodes) != 1:
158
+ raise FormulaValueError(f"Invalid or duplicate formula cell at {location}")
159
+ coordinates.add(coordinate)
160
+ formula = formula_nodes[0]
161
+ kind = formula.get("t", "normal").lower()
162
+ if kind == "shared":
163
+ index = formula.get("si", "")
164
+ if not re.fullmatch(r"[0-9]+", index) or int(index) > 4294967295:
165
+ raise FormulaValueError(f"Invalid shared formula index at {location}")
166
+ groups.setdefault(index, []).append((coordinate, formula))
167
+ elif kind != "normal" or "ref" in formula.attrib or "si" in formula.attrib:
168
+ raise FormulaValueError(f"Unsupported {kind} formula at {location}")
169
+ elif not formula.text:
170
+ raise FormulaValueError(f"Formula has no body at {location}")
171
+
172
+ for index, members in groups.items():
173
+ masters = [(coordinate, formula) for coordinate, formula in members if formula.text]
174
+ if len(masters) != 1:
175
+ raise FormulaValueError(f"Shared formula group {index} in {sheet_part} must have exactly one master")
176
+ origin, master = masters[0]
177
+ location = f"{sheet_part}!{origin}"
178
+ # openpyxl translates followers as it reads them. A missing or late
179
+ # master would otherwise silently turn their bodies into '='.
180
+ if members[0][0] != origin:
181
+ raise FormulaValueError(f"Shared formula master follows its members at {location}")
182
+ reference = master.get("ref", "")
183
+ if not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]*(?::[A-Z]{1,3}[1-9][0-9]*)?", reference):
184
+ raise FormulaValueError(f"Invalid shared formula range at {location}: {reference}")
185
+ min_col, min_row, max_col, max_row = range_boundaries(reference)
186
+ if not (1 <= min_col <= max_col <= 16384 and 1 <= min_row <= max_row <= 1048576):
187
+ raise FormulaValueError(f"Invalid shared formula range at {location}: {reference}")
188
+ try:
189
+ translator = Translator("=" + master.text, origin=origin)
190
+ for coordinate, formula in members:
191
+ row, column = coordinate_to_tuple(coordinate)
192
+ if not (min_col <= column <= max_col and min_row <= row <= max_row):
193
+ raise FormulaValueError(f"Shared formula member outside {reference} at {sheet_part}!{coordinate}")
194
+ if coordinate != origin and "ref" in formula.attrib:
195
+ raise FormulaValueError(f"Shared formula follower declares a range at {sheet_part}!{coordinate}")
196
+ translator.translate_formula(coordinate)
197
+ except FormulaValueError:
198
+ raise
199
+ except Exception as exc:
200
+ raise FormulaValueError(f"Cannot expand shared formula at {location}") from exc
201
+
202
+
203
+ def _scan_package(path: Path) -> None:
204
+ """Reject package features that can cause I/O or code execution in Excel."""
205
+ try:
206
+ with zipfile.ZipFile(path) as archive:
207
+ archive_names = {name.replace("\\", "/").lower(): name for name in archive.namelist()}
208
+ names = set(archive_names)
209
+ dangerous_parts = (
210
+ "xl/externallinks/", "xl/connections.xml", "xl/querytables/",
211
+ "xl/ddelinks/", "xl/macrosheets/", "xl/dialogsheets/",
212
+ )
213
+ for part in dangerous_parts:
214
+ if any(name.startswith(part) or name == part.rstrip("/") for name in names):
215
+ raise FormulaValueError(f"Workbook contains unsupported external or executable part: {part}")
216
+
217
+ for name in names:
218
+ if not name.endswith(".xml") and not name.endswith(".rels"):
219
+ continue
220
+ raw = archive.read(archive_names[name])
221
+ lowered = raw.lower()
222
+ if b"<ddelink" in lowered or b"<olelink" in lowered:
223
+ raise FormulaValueError(f"Workbook contains DDE/OLE link data: {name}")
224
+ if name.endswith(".rels") and b'targetmode="external"' in lowered:
225
+ # External hyperlinks are harmless, but an external target
226
+ # in workbook/externalLink relationships is not. Reject
227
+ # only relationship files whose type names indicate a link
228
+ # or connection, avoiding false positives for user URLs.
229
+ if b"externallink" in lowered or b"externalconnection" in lowered:
230
+ raise FormulaValueError(f"Workbook contains an external relationship: {name}")
231
+ # Shared formulas are ordinary formulas compressed into a master
232
+ # and relative followers. Validate the groups before openpyxl
233
+ # expands them; array/data-table ranges still require separate
234
+ # semantics and remain unsupported.
235
+ for name in names:
236
+ if not name.startswith("xl/worksheets/") or not name.endswith(".xml"):
237
+ continue
238
+ root = ElementTree.fromstring(archive.read(archive_names[name]))
239
+ _scan_sheet_formulas(root, name)
240
+ except ElementTree.ParseError as exc:
241
+ raise FormulaValueError(f"Cannot inspect workbook XML safely: {path}") from exc
242
+ except (OSError, zipfile.BadZipFile) as exc:
243
+ raise FormulaValueError(f"Cannot inspect workbook package: {path}") from exc
244
+
245
+
246
+ def _scan_formulas(workbook: Any, *, allow_errors: bool = False) -> list[tuple[str, str, str]]:
247
+ formulas: list[tuple[str, str, str]] = []
248
+ for worksheet, cell in _iter_formula_cells(workbook):
249
+ formula = cell.value.text if hasattr(cell.value, "text") else cell.value
250
+ if not isinstance(formula, str):
251
+ raise FormulaValueError(f"Unsupported formula object at {worksheet.title}!{cell.coordinate}")
252
+ if not formula.startswith("="):
253
+ formula = "=" + formula
254
+ scan_formula = _without_string_literals(formula)
255
+ upper = scan_formula.upper()
256
+ if _EXTERNAL_REF_RE.search(scan_formula) or re.search(r"\bDDE\s*\(", scan_formula, re.IGNORECASE):
257
+ raise FormulaValueError(f"External workbook/DDE reference at {worksheet.title}!{cell.coordinate}")
258
+ if any(token in upper for token in ("_XLFN.", "_XLWS.")):
259
+ raise FormulaValueError(f"Unsupported modern/dynamic formula at {worksheet.title}!{cell.coordinate}")
260
+ if _DYNAMIC_SPILL_RE.search(scan_formula) or "@" in scan_formula:
261
+ raise FormulaValueError(f"Dynamic/implicit-array formula at {worksheet.title}!{cell.coordinate}")
262
+ if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(name)}\s*\(", upper) for name in _VOLATILE_FUNCTIONS):
263
+ raise FormulaValueError(f"Volatile formula is not reproducible at {worksheet.title}!{cell.coordinate}")
264
+ if "#REF!" in upper and not allow_errors:
265
+ raise FormulaValueError(f"Broken formula reference at {worksheet.title}!{cell.coordinate}")
266
+ for match in _FUNCTION_CALL_RE.finditer(scan_formula):
267
+ name = match.group(1).upper()
268
+ if name in _DYNAMIC_FUNCTIONS:
269
+ raise FormulaValueError(f"Dynamic-array formula at {worksheet.title}!{cell.coordinate}")
270
+ if name.startswith("_XLFN.") or name.startswith("_XLWS.") or name not in _BUILTIN_FUNCTIONS:
271
+ raise FormulaValueError(f"Unknown/UDF formula function {name} at {worksheet.title}!{cell.coordinate}")
272
+ formulas.append((worksheet.title, cell.coordinate, formula))
273
+ _detect_formula_cycles(workbook, allow_errors=allow_errors)
274
+ return formulas
275
+
276
+
277
+ def _detect_formula_cycles(workbook: Any, *, allow_errors: bool = False) -> None:
278
+ """Fail closed on cycles or references we cannot statically account for."""
279
+ nodes = {(ws.title, cell.coordinate): cell for ws, cell in _iter_formula_cells(workbook)}
280
+ graph = {}
281
+ sheet_names = {ws.title.casefold(): ws.title for ws in workbook.worksheets}
282
+ global_names = {name.casefold(): value for name, value in workbook.defined_names.items()}
283
+ local_names = {ws.title: {name.casefold(): value for name, value in ws.defined_names.items()}
284
+ for ws in workbook.worksheets}
285
+ by_sheet = {}
286
+ for key, cell in nodes.items():
287
+ by_sheet.setdefault(key[0], []).append((key, cell.row, cell.column))
288
+ ref_re = re.compile(r"^(?:\$?[A-Z]{1,3}\$?\d+(?::\$?[A-Z]{1,3}\$?\d+)?|\$?[A-Z]{1,3}:\$?[A-Z]{1,3}|\$?\d+:\$?\d+)$", re.I)
289
+
290
+ def dependencies(formula, context_sheet, name_stack=()):
291
+ try:
292
+ tokens = Tokenizer(formula if formula.startswith('=') else '=' + formula).items
293
+ except Exception as exc:
294
+ raise FormulaValueError(f"Unable to parse formula dependencies in {context_sheet}") from exc
295
+ result = set()
296
+ for token in tokens:
297
+ if token.type != "OPERAND" or token.subtype != "RANGE":
298
+ continue
299
+ raw = token.value.strip()
300
+ if allow_errors and "#REF!" in raw.upper():
301
+ # A deleted reference (including Sheet!#REF!) has no target
302
+ # node. Excel still computes and records its typed error;
303
+ # other dependencies and cycles remain fully checked.
304
+ continue
305
+ if "!" in raw:
306
+ sheet_text, raw_ref = raw.rsplit("!", 1)
307
+ if ":" in sheet_text:
308
+ raise FormulaValueError(f"Unsupported 3-D formula reference: {raw}")
309
+ if sheet_text.startswith("'") and sheet_text.endswith("'"):
310
+ sheet_text = sheet_text[1:-1].replace("''", "'")
311
+ target_sheet = sheet_names.get(sheet_text.casefold())
312
+ if target_sheet is None:
313
+ raise FormulaValueError(f"Unknown formula reference sheet: {raw}")
314
+ else:
315
+ raw_ref, target_sheet = raw, context_sheet
316
+ if ref_re.fullmatch(raw_ref):
317
+ # Relative references in defined names are anchored to Excel's
318
+ # active-cell context, not necessarily the formula cell. Do
319
+ # not guess that context when checking dependencies.
320
+ if name_stack and not re.fullmatch(
321
+ r'(?:\$[A-Z]{1,3}\$\d+(?::\$[A-Z]{1,3}\$\d+)?|\$[A-Z]{1,3}:\$[A-Z]{1,3}|\$\d+:\$\d+)',
322
+ raw_ref, re.I,
323
+ ):
324
+ raise FormulaValueError(f"Relative defined-name reference is unsupported: {raw}")
325
+ min_col, min_row, max_col, max_row = range_boundaries(raw_ref.upper())
326
+ min_col, min_row = min_col or 1, min_row or 1
327
+ max_col, max_row = max_col or 16384, max_row or 1048576
328
+ if not (1 <= min_col <= max_col <= 16384 and 1 <= min_row <= max_row <= 1048576):
329
+ raise FormulaValueError(f"Invalid formula range: {raw}")
330
+ result.update(key for key, row, col in by_sheet.get(target_sheet, [])
331
+ if min_col <= col <= max_col and min_row <= row <= max_row)
332
+ else:
333
+ name_key = raw_ref.casefold()
334
+ defined = local_names.get(target_sheet, {}).get(name_key) or global_names.get(name_key)
335
+ if defined is None:
336
+ raise FormulaValueError(f"Unsupported or unknown formula reference: {raw}")
337
+ identity = (target_sheet, name_key)
338
+ if identity in name_stack or len(name_stack) >= 100:
339
+ raise FormulaValueError(f"Circular or excessively nested defined name: {raw}")
340
+ result.update(dependencies(str(defined.attr_text or ''), target_sheet, (*name_stack, identity)))
341
+ return result
342
+
343
+ for node, cell in nodes.items():
344
+ formula = cell.value.text if hasattr(cell.value, "text") else str(cell.value)
345
+ graph[node] = dependencies(formula, node[0])
346
+ # Iterative topological elimination supports long formula chains.
347
+ reverse = {node: set() for node in graph}
348
+ counts = {node: len(edges) for node, edges in graph.items()}
349
+ for node, edges in graph.items():
350
+ for edge in edges:
351
+ reverse[edge].add(node)
352
+ ready = [node for node, count in counts.items() if count == 0]
353
+ visited = 0
354
+ while ready:
355
+ node = ready.pop()
356
+ visited += 1
357
+ for dependent in reverse[node]:
358
+ counts[dependent] -= 1
359
+ if counts[dependent] == 0:
360
+ ready.append(dependent)
361
+ if visited != len(graph):
362
+ node = next(node for node, count in counts.items() if count)
363
+ raise FormulaValueError(f"Circular formula dependency detected at {node[0]}!{node[1]}")
364
+
365
+
366
+ def _scan_defined_names(workbook: Any, *, allow_errors: bool = False) -> None:
367
+ """Reject names which smuggle an external link or executable reference."""
368
+ all_names = [*workbook.defined_names.values(),
369
+ *(name for ws in workbook.worksheets for name in ws.defined_names.values())]
370
+ for defined_name in all_names:
371
+ text = str(getattr(defined_name, "attr_text", "") or "")
372
+ scan_text = _without_string_literals(text)
373
+ scan_upper = scan_text.upper()
374
+ if _EXTERNAL_REF_RE.search(scan_text) or re.search(r"\bDDE\s*\(", scan_text, re.IGNORECASE) or ("#REF!" in scan_upper and not allow_errors):
375
+ name = getattr(defined_name, "name", "<unnamed>")
376
+ raise FormulaValueError(f"Defined name contains an unsafe reference: {name}")
377
+ name = getattr(defined_name, "name", "<unnamed>")
378
+ if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(function)}\s*\(", scan_upper)
379
+ for function in _VOLATILE_FUNCTIONS):
380
+ raise FormulaValueError(f"Defined name contains a volatile formula: {name}")
381
+ if _DYNAMIC_SPILL_RE.search(scan_text) or "@" in scan_text:
382
+ raise FormulaValueError(f"Defined name contains a dynamic-array formula: {name}")
383
+ for match in _FUNCTION_CALL_RE.finditer(scan_text):
384
+ function = match.group(1).upper()
385
+ if function in _DYNAMIC_FUNCTIONS or function.startswith(("_XLFN.", "_XLWS.")) \
386
+ or function not in _BUILTIN_FUNCTIONS:
387
+ raise FormulaValueError(f"Defined name contains unknown/UDF function {function}: {name}")
388
+
389
+
390
+ def _read_state(path: Path) -> dict[str, Any]:
391
+ try:
392
+ text = path.read_text(encoding="utf-8-sig")
393
+ value = json.loads(text)
394
+ return value if isinstance(value, dict) else {}
395
+ except (OSError, ValueError, json.JSONDecodeError):
396
+ return {}
397
+
398
+
399
+ def _cleanup_excel_process(state_path: Path) -> None:
400
+ """Stop only the Excel PID and start time recorded by our PS harness."""
401
+ state = _read_state(state_path)
402
+ pid = state.get("pid")
403
+ start_filetime = state.get("startFileTimeUtc")
404
+ if not isinstance(pid, int) or not isinstance(start_filetime, int):
405
+ return
406
+ helper = Path(__file__).with_name("recalculate_workbook.ps1")
407
+ try:
408
+ # Pass ownership data as JSON, never as PowerShell source/arguments.
409
+ # The helper validates both PID and process creation time before stop.
410
+ subprocess.run(
411
+ [_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
412
+ "-File", str(helper), "-CleanupParametersPath", str(state_path)],
413
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=False,
414
+ )
415
+ except (OSError, subprocess.TimeoutExpired):
416
+ pass
417
+
418
+
419
+ def _run_excel_recalculate(source: Path, result: Path, timeout_seconds: int) -> None:
420
+ from prepare_progress import checkpoint, cancellable, PreparationCancelled
421
+ script = Path(__file__).with_name("recalculate_workbook.ps1")
422
+ if not script.is_file():
423
+ raise FormulaValueError(f"Excel recalculation helper is missing: {script}")
424
+ with tempfile.TemporaryDirectory(prefix="xlmerge-excel-state-") as state_dir:
425
+ state_path = Path(state_dir) / "state.json"
426
+ params_path = Path(state_dir) / "params.json"
427
+ params_path.write_text(json.dumps({
428
+ "source": str(source), "result": str(result), "state": str(state_path),
429
+ "timeoutSeconds": int(timeout_seconds),
430
+ }), encoding="utf-8")
431
+ command = [_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
432
+ "-File", str(script), "-ParametersPath", str(params_path)]
433
+ try:
434
+ if not cancellable():
435
+ completed = subprocess.run(command, capture_output=True, text=True,
436
+ encoding="utf-8", errors="replace",
437
+ timeout=max(1, int(timeout_seconds) + 20), check=False)
438
+ else:
439
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
440
+ text=True, encoding='utf-8', errors='replace')
441
+ deadline = time.monotonic() + max(1, int(timeout_seconds) + 20)
442
+ try:
443
+ while True:
444
+ checkpoint()
445
+ remaining = deadline - time.monotonic()
446
+ if remaining <= 0:
447
+ raise subprocess.TimeoutExpired(command, timeout_seconds)
448
+ try:
449
+ stdout, stderr = process.communicate(timeout=min(.25, remaining))
450
+ completed = subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
451
+ break
452
+ except subprocess.TimeoutExpired:
453
+ continue
454
+ except BaseException:
455
+ # Cleanup before terminating the harness; its PID/start time
456
+ # ownership record excludes Excel instances opened by users.
457
+ _cleanup_excel_process(state_path)
458
+ process.terminate()
459
+ try:
460
+ process.communicate(timeout=10)
461
+ except subprocess.TimeoutExpired:
462
+ process.kill()
463
+ process.communicate()
464
+ raise
465
+ except subprocess.TimeoutExpired as exc:
466
+ _cleanup_excel_process(state_path)
467
+ raise FormulaValueError("Excel recalculation exceeded the hard timeout") from exc
468
+ finally:
469
+ # The helper normally closes Excel itself. This also handles a
470
+ # PowerShell error path without ever scanning/killing user Excel.
471
+ if state_path.exists() and _read_state(state_path).get("status") not in {"completed", "failed"}:
472
+ _cleanup_excel_process(state_path)
473
+ if completed.returncode != 0:
474
+ state = _read_state(state_path)
475
+ detail = state.get("error") or completed.stderr.strip() or completed.stdout.strip()
476
+ raise FormulaValueError(f"Excel could not safely calculate workbook: {detail or 'unknown error'}")
477
+ if not result.is_file() or result.stat().st_size == 0:
478
+ raise FormulaValueError("Excel reported success but did not export a result workbook")
479
+
480
+
481
+ def _set_literal(cell: Any, value: Any) -> None:
482
+ """Assign a value without allowing a string beginning '=' to become a formula."""
483
+ if isinstance(value, str):
484
+ # openpyxl classifies strings such as '#N/A' as error cells and strings
485
+ # beginning '=' as formulas. Cached formula text is data, regardless
486
+ # of its spelling, so force the serialized string type for all text.
487
+ cell.data_type = "s"
488
+ if value == "":
489
+ # openpyxl normally omits an empty string cell entirely on write,
490
+ # which reloads as None. A zero-length rich-text run is a valid
491
+ # inline string and round-trips as a typed empty string.
492
+ cell._value = CellRichText(TextBlock(InlineFont(), ""))
493
+ else:
494
+ cell._value = value
495
+ else:
496
+ cell.value = value
497
+
498
+
499
+ def _validate_result(value: Any, worksheet: str, coordinate: str, data_type: str | None = None) -> None:
500
+ # Use the serialized cell type, not the displayed text: a formula may
501
+ # intentionally return the literal string "#N/A", while an Excel error is
502
+ # represented by cell type ``e``.
503
+ if data_type == "e":
504
+ raise FormulaValueError(f"Excel returned an error value at {worksheet}!{coordinate}: {value}")
505
+ if isinstance(value, float) and not math.isfinite(value):
506
+ raise FormulaValueError(f"Excel returned non-finite numeric value at {worksheet}!{coordinate}")
507
+ if value is not None and not isinstance(value, (str, bool, int, float, date, datetime, datetime_time)):
508
+ raise FormulaValueError(f"Unsupported Excel result type at {worksheet}!{coordinate}: {type(value).__name__}")
509
+
510
+
511
+ def _formula_cache_presence(path: Path) -> dict[tuple[str, str], bool]:
512
+ """Return whether each formula cell has a serialized ``<v>`` cache.
513
+
514
+ Excel writes an empty-string result as ``<v></v>`` (which openpyxl exposes
515
+ as ``None`` on some Excel versions), whereas an uncalculated formula has no
516
+ ``<v>`` element. Keeping this distinction is essential for fail-closed
517
+ calculation verification.
518
+ """
519
+ result: dict[tuple[str, str], bool] = {}
520
+ try:
521
+ with zipfile.ZipFile(path) as archive:
522
+ wb_root = ElementTree.fromstring(archive.read("xl/workbook.xml"))
523
+ rel_root = ElementTree.fromstring(archive.read("xl/_rels/workbook.xml.rels"))
524
+ rel_targets = {}
525
+ for rel in rel_root:
526
+ rid = rel.attrib.get("Id") or rel.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
527
+ target = rel.attrib.get("Target")
528
+ if rid and target:
529
+ rel_targets[rid] = "xl/" + target.lstrip("/") if not target.startswith("/") else target.lstrip("/")
530
+ for sheet in wb_root.iter():
531
+ if sheet.tag.rsplit("}", 1)[-1] != "sheet":
532
+ continue
533
+ sheet_name = sheet.attrib.get("name")
534
+ rid = sheet.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
535
+ target = rel_targets.get(rid)
536
+ if not sheet_name or not target or target not in archive.namelist():
537
+ continue
538
+ root = ElementTree.fromstring(archive.read(target))
539
+ for cell in root.iter():
540
+ if cell.tag.rsplit("}", 1)[-1] != "c" or not cell.attrib.get("r"):
541
+ continue
542
+ if any(child.tag.rsplit("}", 1)[-1] == "f" for child in cell):
543
+ result[(sheet_name, cell.attrib["r"])] = any(
544
+ child.tag.rsplit("}", 1)[-1] == "v" for child in cell
545
+ )
546
+ except (OSError, KeyError, zipfile.BadZipFile, ElementTree.ParseError):
547
+ return {}
548
+ return result
549
+
550
+
551
+ def _materialize_from_result(source: Path, calculated: Path, output: Path,
552
+ formula_keys: set[tuple[str, str]], *, allow_errors: bool = False) -> Path:
553
+ keep_vba = source.suffix.lower() == ".xlsm"
554
+ try:
555
+ original = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
556
+ values = load_workbook(calculated, data_only=True, keep_links=False, keep_vba=keep_vba)
557
+ except Exception as exc:
558
+ raise FormulaValueError(f"Cannot read Excel calculation result: {exc}") from exc
559
+ try:
560
+ cache_presence = _formula_cache_presence(calculated)
561
+ for sheet_name, coordinate in formula_keys:
562
+ if sheet_name not in values.sheetnames or sheet_name not in original.sheetnames:
563
+ raise FormulaValueError(f"Calculation result lost sheet {sheet_name!r}")
564
+ source_cell = original[sheet_name][coordinate]
565
+ result_cell = values[sheet_name][coordinate]
566
+ value = result_cell.value
567
+ if result_cell.data_type == "e" and allow_errors:
568
+ if not isinstance(value, str) or value not in _ERROR_VALUES:
569
+ raise FormulaValueError(f"Invalid Excel error result at {sheet_name}!{coordinate}: {value}")
570
+ # Historical errors are typed unknowns, never literal text or
571
+ # invented numeric results. Consumers must require review.
572
+ source_cell.value = value
573
+ source_cell.data_type = "e"
574
+ continue
575
+ _validate_result(value, sheet_name, coordinate, result_cell.data_type)
576
+ # A missing <v> cache is different from a legitimate cached empty
577
+ # string. Excel's data_only reader returns None in both unusual
578
+ # cases; normal formulas returning "" have a string cache and are
579
+ # accepted. Missing cache is rejected by the post-save formula
580
+ # count/result verification below if Excel did not calculate it.
581
+ if value is None:
582
+ if result_cell.data_type not in ('s', 'str') or not cache_presence.get((sheet_name, coordinate), False):
583
+ raise FormulaValueError(f"Excel did not provide a cached result at {sheet_name}!{coordinate}")
584
+ # Excel's empty-string cache can be represented by an empty
585
+ # <v> with no Python value. It is still a string result.
586
+ value = ""
587
+ _set_literal(source_cell, value)
588
+
589
+ output.parent.mkdir(parents=True, exist_ok=True)
590
+ temp_output = output.with_name(f".{output.stem}.tmp-{os.getpid()}-{time.time_ns()}{output.suffix}")
591
+ try:
592
+ original.save(temp_output)
593
+ check = load_workbook(temp_output, data_only=False, keep_links=False, keep_vba=keep_vba)
594
+ try:
595
+ leftovers = [(ws.title, c.coordinate) for ws, c in _iter_formula_cells(check)]
596
+ finally:
597
+ check.close()
598
+ if leftovers:
599
+ raise FormulaValueError(f"Formula values remained after materialization: {leftovers[:3]}")
600
+ os.replace(temp_output, output)
601
+ finally:
602
+ if temp_output.exists():
603
+ temp_output.unlink(missing_ok=True)
604
+ finally:
605
+ original.close()
606
+ values.close()
607
+ return output
608
+
609
+
610
+ def materialize_values(source_path: Path | str, output_path: Path | str, *, allow_errors: bool = False) -> Path:
611
+ """Calculate every formula in ``source_path`` and emit a literal snapshot.
612
+
613
+ The source is never opened by Excel and is never modified. A workbook with
614
+ no formulas is returned unchanged, as an explicit fast path.
615
+ """
616
+ source = Path(source_path).resolve()
617
+ output = Path(output_path).resolve()
618
+ if not source.is_file():
619
+ raise FormulaValueError(f"Workbook does not exist: {source}")
620
+ if source == output:
621
+ raise FormulaValueError("Output path must differ from source path")
622
+ if source.suffix.lower() not in {".xlsx", ".xlsm"}:
623
+ raise FormulaValueError(f"Unsupported workbook extension: {source.suffix}")
624
+ if output.suffix.lower() != source.suffix.lower():
625
+ raise FormulaValueError("Output extension must match source extension")
626
+
627
+ _scan_package(source)
628
+ keep_vba = source.suffix.lower() == ".xlsm"
629
+ try:
630
+ workbook = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
631
+ except Exception as exc:
632
+ raise FormulaValueError(f"Cannot open workbook for formula inspection: {exc}") from exc
633
+ try:
634
+ formulas = _scan_formulas(workbook, allow_errors=allow_errors)
635
+ _scan_defined_names(workbook, allow_errors=allow_errors)
636
+ finally:
637
+ workbook.close()
638
+ if not formulas:
639
+ return source
640
+
641
+ timeout_setting = os.environ.get("XLMERGE_EXCEL_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))
642
+ try:
643
+ timeout_seconds = max(1, int(timeout_setting))
644
+ except ValueError as exc:
645
+ raise FormulaValueError("XLMERGE_EXCEL_TIMEOUT_SECONDS must be an integer") from exc
646
+
647
+ temp_dir = Path(tempfile.mkdtemp(prefix="xlmerge-formula-values-"))
648
+ temporary_source = temp_dir / source.name
649
+ temporary_result = temp_dir / ("calculated" + source.suffix.lower())
650
+ try:
651
+ shutil.copy2(source, temporary_source)
652
+ _run_excel_recalculate(temporary_source, temporary_result, timeout_seconds)
653
+ return _materialize_from_result(source, temporary_result, output,
654
+ {(sheet, coordinate) for sheet, coordinate, _ in formulas},
655
+ allow_errors=allow_errors)
656
+ finally:
657
+ shutil.rmtree(temp_dir, ignore_errors=True)
658
+
659
+
660
+ __all__ = ["FormulaValueError", "materialize_values"]