@kevlns/xlmerge 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +169 -141
- package/LICENSE +21 -21
- package/README.md +111 -193
- package/bin/xlmerge.js +152 -152
- package/docs/documentation.md +24 -0
- package/package.json +7 -5
- package/python/xlsx_merge_engine/__init__.py +2 -2
- package/python/xlsx_merge_engine/xlsx_git_merge_bridge.py +1614 -1611
- package/python/xlsx_resolver/__init__.py +2 -2
- package/python/xlsx_resolver/__main__.py +4 -4
- package/python/xlsx_resolver/formula_values.py +660 -565
- package/python/xlsx_resolver/loading_ui.html +44 -0
- package/python/xlsx_resolver/merge_server.py +179 -126
- package/python/xlsx_resolver/merge_ui.html +1634 -1522
- package/python/xlsx_resolver/prepare_progress.py +34 -0
- package/python/xlsx_resolver/recalculate_workbook.ps1 +141 -155
- package/python/xlsx_resolver/resolve_xlsx_conflict.py +236 -222
- package/python/xlsx_resolver/xlsx_conflict.py +2317 -1994
- package/v-cli.plugin.json +248 -194
|
@@ -1,565 +1,660 @@
|
|
|
1
|
-
"""Safe, Excel-backed formula materialisation for the conflict resolver.
|
|
2
|
-
|
|
3
|
-
This module deliberately has a small trust boundary. Excel is only ever given a
|
|
4
|
-
temporary copy, with links and macros disabled. The result produced by Excel is
|
|
5
|
-
read back as cached values and written into a copy of the original package with
|
|
6
|
-
openpyxl. We do not use ``data_only`` as a source workbook: a stale cache is not
|
|
7
|
-
an acceptable answer for a conflict resolver.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from __future__ import annotations
|
|
11
|
-
|
|
12
|
-
import json
|
|
13
|
-
import math
|
|
14
|
-
import os
|
|
15
|
-
import re
|
|
16
|
-
import shutil
|
|
17
|
-
import subprocess
|
|
18
|
-
import tempfile
|
|
19
|
-
import time
|
|
20
|
-
import zipfile
|
|
21
|
-
from datetime import date, datetime, time as datetime_time
|
|
22
|
-
from pathlib import Path
|
|
23
|
-
from typing import Any, Iterable
|
|
24
|
-
from xml.etree import ElementTree
|
|
25
|
-
|
|
26
|
-
from openpyxl import load_workbook
|
|
27
|
-
from openpyxl.cell.rich_text import CellRichText, InlineFont, TextBlock
|
|
28
|
-
from openpyxl.formula import Tokenizer
|
|
29
|
-
from openpyxl.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
#
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
for
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
if
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
1
|
+
"""Safe, Excel-backed formula materialisation for the conflict resolver.
|
|
2
|
+
|
|
3
|
+
This module deliberately has a small trust boundary. Excel is only ever given a
|
|
4
|
+
temporary copy, with links and macros disabled. The result produced by Excel is
|
|
5
|
+
read back as cached values and written into a copy of the original package with
|
|
6
|
+
openpyxl. We do not use ``data_only`` as a source workbook: a stale cache is not
|
|
7
|
+
an acceptable answer for a conflict resolver.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
import tempfile
|
|
19
|
+
import time
|
|
20
|
+
import zipfile
|
|
21
|
+
from datetime import date, datetime, time as datetime_time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Iterable
|
|
24
|
+
from xml.etree import ElementTree
|
|
25
|
+
|
|
26
|
+
from openpyxl import load_workbook
|
|
27
|
+
from openpyxl.cell.rich_text import CellRichText, InlineFont, TextBlock
|
|
28
|
+
from openpyxl.formula import Tokenizer
|
|
29
|
+
from openpyxl.formula.translate import Translator
|
|
30
|
+
from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class FormulaValueError(RuntimeError):
|
|
34
|
+
"""Raised when a workbook cannot be evaluated and verified safely."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
DEFAULT_TIMEOUT_SECONDS = 120
|
|
38
|
+
_ERROR_VALUES = {"#NULL!", "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A", "#GETTING_DATA"}
|
|
39
|
+
_VOLATILE_FUNCTIONS = {
|
|
40
|
+
"NOW", "TODAY", "RAND", "RANDBETWEEN", "RANDARRAY", "RTD", "OFFSET", "INDIRECT",
|
|
41
|
+
"CELL", "INFO",
|
|
42
|
+
}
|
|
43
|
+
_DYNAMIC_FUNCTIONS = {
|
|
44
|
+
"FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE", "RANDARRAY", "TRANSPOSE",
|
|
45
|
+
"TOCOL", "TOROW", "TAKE", "DROP", "CHOOSECOLS", "CHOOSEROWS", "EXPAND",
|
|
46
|
+
"VSTACK", "HSTACK", "WRAPROWS", "WRAPCOLS", "XLOOKUP", "XMATCH", "LAMBDA",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
# Function names are checked before opening Excel. This is intentionally a
|
|
50
|
+
# broad list of ordinary worksheet functions, but unknown calls are rejected so
|
|
51
|
+
# a disabled VBA UDF can never be silently accepted merely because it returned a
|
|
52
|
+
# plausible value in a cached workbook.
|
|
53
|
+
_BUILTIN_FUNCTIONS = {
|
|
54
|
+
name
|
|
55
|
+
for name in """
|
|
56
|
+
ABS ACOS ACOSH ACOT ACOTH AGGREGATE ARABIC ASIN ASINH ATAN ATAN2 ATANH BASE
|
|
57
|
+
CEILING CEILING.MATH CEILING.PRECISE COMBIN COMBINA COS COSH COT COTH CSC CSCH
|
|
58
|
+
DECIMAL DEGREES EVEN EXP FACT FACTDOUBLE FLOOR FLOOR.MATH FLOOR.PRECISE GCD
|
|
59
|
+
INT ISO.CEILING LCM LN LOG LOG10 MDETERM MINVERSE MMULT MOD MROUND MULTINOMIAL
|
|
60
|
+
MUNIT ODD PI POWER PRODUCT QUOTIENT RADIANS ROMAN ROUND ROUNDDOWN ROUNDUP SEC
|
|
61
|
+
SECH SIGN SIN SINH SQRT SUM SUMIF SUMIFS SUMPRODUCT SUMSQ TAN TANH TRUNC
|
|
62
|
+
ACCRINT ACCRINTM AMORDEGRC AMORLINC COUPDAYBS COUPDAYS COUPDAYSNC COUPNCD
|
|
63
|
+
COUPNUM COUPPCD CUMIPMT CUMPRINC DB DDB DISC DOLLARDE DOLLARFR DURATION EFFECT
|
|
64
|
+
FV FVSCHEDULE INTRATE IPMT IRR ISPMT MDURATION MIRR NOMINAL NPER NPV PDURATION
|
|
65
|
+
PMT PPMT PRICE PRICEDISC PRICEMAT PV RATE RECEIVED RRI SLN SYD TBILLEQ TBILLPRICE
|
|
66
|
+
TBILLYIELD VDB XIRR XNPV YIELD YIELDDISC YIELDMAT
|
|
67
|
+
AND FALSE IF IFERROR IFNA NOT OR TRUE XOR SWITCH IFS CHOOSE
|
|
68
|
+
ADDRESS AREAS COLUMN COLUMNS FORMULATEXT HLOOKUP HYPERLINK INDEX INDIRECT
|
|
69
|
+
LOOKUP MATCH ROW ROWS TRANSPOSE VLOOKUP OFFSET
|
|
70
|
+
CELL ERROR.TYPE INFO ISBLANK ISERR ISERROR ISEVEN ISFORMULA ISLOGICAL ISNA
|
|
71
|
+
ISNONTEXT ISNUMBER ISODD ISREF ISTEXT N TYPE NA
|
|
72
|
+
CONCAT CONCATENATE EXACT FIND FINDB LEFT LEFTB LEN LENB LOWER MID MIDB NUMBERVALUE
|
|
73
|
+
PHONETIC PROPER REPLACE REPLACEB REPT RIGHT RIGHTB SEARCH SEARCHB SUBSTITUTE T
|
|
74
|
+
TEXT TEXTAFTER TEXTBEFORE TEXTJOIN TEXTSPLIT TRIM UNICHAR UNICODE UPPER VALUE
|
|
75
|
+
ASC CHAR CLEAN CODE DBCS FIXED JIS LOWER UPPER
|
|
76
|
+
DATE DATEVALUE DAY DAYS DAYS360 EDATE EOMONTH HOUR ISOWEEKNUM MINUTE MONTH
|
|
77
|
+
NETWORKDAYS NETWORKDAYS.INTL SECOND TIME TIMEVALUE WEEKDAY WEEKNUM WORKDAY
|
|
78
|
+
WORKDAY.INTL YEAR YEARFRAC
|
|
79
|
+
COUNT COUNTA COUNTBLANK COUNTIF COUNTIFS MAX MAXIFS MIN MINIFS AVERAGE
|
|
80
|
+
AVERAGEIF AVERAGEIFS MEDIAN MODE.MULT MODE.SNGL LARGE SMALL RANK.EQ RANK.AVG
|
|
81
|
+
PERCENTILE.EXC PERCENTILE.INC PERCENTRANK.EXC PERCENTRANK.INC QUARTILE.EXC
|
|
82
|
+
QUARTILE.INC STANDARDIZE STDEV.P STDEV.S STDEVA STDEVPA VAR.P VAR.S VARA VARPA
|
|
83
|
+
CORREL COVARIANCE.P COVARIANCE.S GEOMEAN HARMEAN KURT SKEW SKEW.P SLOPE
|
|
84
|
+
INTERCEPT FORECAST FORECAST.LINEAR FREQUENCY GROWTH LINEST LOGEST PEARSON RSQ
|
|
85
|
+
TREND
|
|
86
|
+
DEC2BIN DEC2HEX DEC2OCT HEX2BIN HEX2DEC HEX2OCT OCT2BIN OCT2DEC OCT2HEX
|
|
87
|
+
BITAND BITLSHIFT BITOR BITRSHIFT BITXOR CONVERT DELTA ERF ERF.PRECISE ERFC
|
|
88
|
+
ERFC.PRECISE GESTEP IMSUM IMABS IMAGINARY IMARGUMENT IMCONJUGATE IMCOS IMCOSH
|
|
89
|
+
IMCOT IMCSC IMCSCH IMDIV IMEXP IMLN IMLOG10 IMLOG2 IMPOWER IMPRODUCT IMREAL
|
|
90
|
+
IMSEC IMSECH IMSIN IMSINH IMSQRT IMSUB IMTAN IMTANH COMPLEX
|
|
91
|
+
CUBEMEMBER CUBEVALUE GETPIVOTDATA
|
|
92
|
+
LET LAMBDA MAP REDUCE SCAN MAKEARRAY BYROW BYCOL
|
|
93
|
+
""".split()
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
_FUNCTION_CALL_RE = re.compile(r"(?<![A-Za-z0-9_.])([A-Za-z_][A-Za-z0-9_.]*)\s*\(")
|
|
97
|
+
_EXTERNAL_REF_RE = re.compile(r"\[[^\]]+\][^!]*!")
|
|
98
|
+
_DYNAMIC_SPILL_RE = re.compile(r"(?:[A-Z]{1,3}\$?\d+|\))#(?=$|[\s,;)])", re.IGNORECASE)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _powershell_executable() -> str:
|
|
102
|
+
# Windows PowerShell is part of the supported host. Do not fall back to a
|
|
103
|
+
# different Python/COM package; pywin32 is intentionally not required.
|
|
104
|
+
return shutil.which("powershell.exe") or shutil.which("powershell") or "powershell.exe"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _formula_text(value: Any) -> str | None:
|
|
108
|
+
if isinstance(value, str) and value.startswith("="):
|
|
109
|
+
return value
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _without_string_literals(formula: str) -> str:
|
|
114
|
+
"""Blank double-quoted Excel string literals before syntax/security scans."""
|
|
115
|
+
out: list[str] = []
|
|
116
|
+
quoted = False
|
|
117
|
+
index = 0
|
|
118
|
+
while index < len(formula):
|
|
119
|
+
char = formula[index]
|
|
120
|
+
if char == '"':
|
|
121
|
+
if quoted and index + 1 < len(formula) and formula[index + 1] == '"':
|
|
122
|
+
out.extend(" ")
|
|
123
|
+
index += 2
|
|
124
|
+
continue
|
|
125
|
+
quoted = not quoted
|
|
126
|
+
out.append(" ")
|
|
127
|
+
else:
|
|
128
|
+
out.append(" " if quoted else char)
|
|
129
|
+
index += 1
|
|
130
|
+
return "".join(out)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _iter_formula_cells(workbook: Any) -> Iterable[tuple[Any, Any]]:
|
|
134
|
+
for worksheet in workbook.worksheets:
|
|
135
|
+
# These are normal, fully loaded worksheets. Inspect stored cells,
|
|
136
|
+
# rather than constructing the rectangular formatted UsedRange.
|
|
137
|
+
for cell in worksheet._cells.values():
|
|
138
|
+
if cell.data_type == "f":
|
|
139
|
+
yield worksheet, cell
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _scan_sheet_formulas(root: Any, sheet_part: str) -> None:
|
|
143
|
+
"""Validate shared groups before openpyxl expands their relative references."""
|
|
144
|
+
groups: dict[str, list[tuple[str, Any]]] = {}
|
|
145
|
+
coordinates: set[str] = set()
|
|
146
|
+
for cell in root.iter():
|
|
147
|
+
if cell.tag.rsplit("}", 1)[-1] != "c":
|
|
148
|
+
continue
|
|
149
|
+
coordinate = cell.get("r", "")
|
|
150
|
+
formula_nodes = [node for node in cell if node.tag.rsplit("}", 1)[-1] == "f"]
|
|
151
|
+
if not formula_nodes:
|
|
152
|
+
continue
|
|
153
|
+
location = f"{sheet_part}!{coordinate or '<missing coordinate>'}"
|
|
154
|
+
if not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]*", coordinate):
|
|
155
|
+
raise FormulaValueError(f"Invalid formula coordinate at {location}")
|
|
156
|
+
row, column = coordinate_to_tuple(coordinate)
|
|
157
|
+
if row > 1048576 or column > 16384 or coordinate in coordinates or len(formula_nodes) != 1:
|
|
158
|
+
raise FormulaValueError(f"Invalid or duplicate formula cell at {location}")
|
|
159
|
+
coordinates.add(coordinate)
|
|
160
|
+
formula = formula_nodes[0]
|
|
161
|
+
kind = formula.get("t", "normal").lower()
|
|
162
|
+
if kind == "shared":
|
|
163
|
+
index = formula.get("si", "")
|
|
164
|
+
if not re.fullmatch(r"[0-9]+", index) or int(index) > 4294967295:
|
|
165
|
+
raise FormulaValueError(f"Invalid shared formula index at {location}")
|
|
166
|
+
groups.setdefault(index, []).append((coordinate, formula))
|
|
167
|
+
elif kind != "normal" or "ref" in formula.attrib or "si" in formula.attrib:
|
|
168
|
+
raise FormulaValueError(f"Unsupported {kind} formula at {location}")
|
|
169
|
+
elif not formula.text:
|
|
170
|
+
raise FormulaValueError(f"Formula has no body at {location}")
|
|
171
|
+
|
|
172
|
+
for index, members in groups.items():
|
|
173
|
+
masters = [(coordinate, formula) for coordinate, formula in members if formula.text]
|
|
174
|
+
if len(masters) != 1:
|
|
175
|
+
raise FormulaValueError(f"Shared formula group {index} in {sheet_part} must have exactly one master")
|
|
176
|
+
origin, master = masters[0]
|
|
177
|
+
location = f"{sheet_part}!{origin}"
|
|
178
|
+
# openpyxl translates followers as it reads them. A missing or late
|
|
179
|
+
# master would otherwise silently turn their bodies into '='.
|
|
180
|
+
if members[0][0] != origin:
|
|
181
|
+
raise FormulaValueError(f"Shared formula master follows its members at {location}")
|
|
182
|
+
reference = master.get("ref", "")
|
|
183
|
+
if not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]*(?::[A-Z]{1,3}[1-9][0-9]*)?", reference):
|
|
184
|
+
raise FormulaValueError(f"Invalid shared formula range at {location}: {reference}")
|
|
185
|
+
min_col, min_row, max_col, max_row = range_boundaries(reference)
|
|
186
|
+
if not (1 <= min_col <= max_col <= 16384 and 1 <= min_row <= max_row <= 1048576):
|
|
187
|
+
raise FormulaValueError(f"Invalid shared formula range at {location}: {reference}")
|
|
188
|
+
try:
|
|
189
|
+
translator = Translator("=" + master.text, origin=origin)
|
|
190
|
+
for coordinate, formula in members:
|
|
191
|
+
row, column = coordinate_to_tuple(coordinate)
|
|
192
|
+
if not (min_col <= column <= max_col and min_row <= row <= max_row):
|
|
193
|
+
raise FormulaValueError(f"Shared formula member outside {reference} at {sheet_part}!{coordinate}")
|
|
194
|
+
if coordinate != origin and "ref" in formula.attrib:
|
|
195
|
+
raise FormulaValueError(f"Shared formula follower declares a range at {sheet_part}!{coordinate}")
|
|
196
|
+
translator.translate_formula(coordinate)
|
|
197
|
+
except FormulaValueError:
|
|
198
|
+
raise
|
|
199
|
+
except Exception as exc:
|
|
200
|
+
raise FormulaValueError(f"Cannot expand shared formula at {location}") from exc
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _scan_package(path: Path) -> None:
|
|
204
|
+
"""Reject package features that can cause I/O or code execution in Excel."""
|
|
205
|
+
try:
|
|
206
|
+
with zipfile.ZipFile(path) as archive:
|
|
207
|
+
archive_names = {name.replace("\\", "/").lower(): name for name in archive.namelist()}
|
|
208
|
+
names = set(archive_names)
|
|
209
|
+
dangerous_parts = (
|
|
210
|
+
"xl/externallinks/", "xl/connections.xml", "xl/querytables/",
|
|
211
|
+
"xl/ddelinks/", "xl/macrosheets/", "xl/dialogsheets/",
|
|
212
|
+
)
|
|
213
|
+
for part in dangerous_parts:
|
|
214
|
+
if any(name.startswith(part) or name == part.rstrip("/") for name in names):
|
|
215
|
+
raise FormulaValueError(f"Workbook contains unsupported external or executable part: {part}")
|
|
216
|
+
|
|
217
|
+
for name in names:
|
|
218
|
+
if not name.endswith(".xml") and not name.endswith(".rels"):
|
|
219
|
+
continue
|
|
220
|
+
raw = archive.read(archive_names[name])
|
|
221
|
+
lowered = raw.lower()
|
|
222
|
+
if b"<ddelink" in lowered or b"<olelink" in lowered:
|
|
223
|
+
raise FormulaValueError(f"Workbook contains DDE/OLE link data: {name}")
|
|
224
|
+
if name.endswith(".rels") and b'targetmode="external"' in lowered:
|
|
225
|
+
# External hyperlinks are harmless, but an external target
|
|
226
|
+
# in workbook/externalLink relationships is not. Reject
|
|
227
|
+
# only relationship files whose type names indicate a link
|
|
228
|
+
# or connection, avoiding false positives for user URLs.
|
|
229
|
+
if b"externallink" in lowered or b"externalconnection" in lowered:
|
|
230
|
+
raise FormulaValueError(f"Workbook contains an external relationship: {name}")
|
|
231
|
+
# Shared formulas are ordinary formulas compressed into a master
|
|
232
|
+
# and relative followers. Validate the groups before openpyxl
|
|
233
|
+
# expands them; array/data-table ranges still require separate
|
|
234
|
+
# semantics and remain unsupported.
|
|
235
|
+
for name in names:
|
|
236
|
+
if not name.startswith("xl/worksheets/") or not name.endswith(".xml"):
|
|
237
|
+
continue
|
|
238
|
+
root = ElementTree.fromstring(archive.read(archive_names[name]))
|
|
239
|
+
_scan_sheet_formulas(root, name)
|
|
240
|
+
except ElementTree.ParseError as exc:
|
|
241
|
+
raise FormulaValueError(f"Cannot inspect workbook XML safely: {path}") from exc
|
|
242
|
+
except (OSError, zipfile.BadZipFile) as exc:
|
|
243
|
+
raise FormulaValueError(f"Cannot inspect workbook package: {path}") from exc
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _scan_formulas(workbook: Any, *, allow_errors: bool = False) -> list[tuple[str, str, str]]:
|
|
247
|
+
formulas: list[tuple[str, str, str]] = []
|
|
248
|
+
for worksheet, cell in _iter_formula_cells(workbook):
|
|
249
|
+
formula = cell.value.text if hasattr(cell.value, "text") else cell.value
|
|
250
|
+
if not isinstance(formula, str):
|
|
251
|
+
raise FormulaValueError(f"Unsupported formula object at {worksheet.title}!{cell.coordinate}")
|
|
252
|
+
if not formula.startswith("="):
|
|
253
|
+
formula = "=" + formula
|
|
254
|
+
scan_formula = _without_string_literals(formula)
|
|
255
|
+
upper = scan_formula.upper()
|
|
256
|
+
if _EXTERNAL_REF_RE.search(scan_formula) or re.search(r"\bDDE\s*\(", scan_formula, re.IGNORECASE):
|
|
257
|
+
raise FormulaValueError(f"External workbook/DDE reference at {worksheet.title}!{cell.coordinate}")
|
|
258
|
+
if any(token in upper for token in ("_XLFN.", "_XLWS.")):
|
|
259
|
+
raise FormulaValueError(f"Unsupported modern/dynamic formula at {worksheet.title}!{cell.coordinate}")
|
|
260
|
+
if _DYNAMIC_SPILL_RE.search(scan_formula) or "@" in scan_formula:
|
|
261
|
+
raise FormulaValueError(f"Dynamic/implicit-array formula at {worksheet.title}!{cell.coordinate}")
|
|
262
|
+
if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(name)}\s*\(", upper) for name in _VOLATILE_FUNCTIONS):
|
|
263
|
+
raise FormulaValueError(f"Volatile formula is not reproducible at {worksheet.title}!{cell.coordinate}")
|
|
264
|
+
if "#REF!" in upper and not allow_errors:
|
|
265
|
+
raise FormulaValueError(f"Broken formula reference at {worksheet.title}!{cell.coordinate}")
|
|
266
|
+
for match in _FUNCTION_CALL_RE.finditer(scan_formula):
|
|
267
|
+
name = match.group(1).upper()
|
|
268
|
+
if name in _DYNAMIC_FUNCTIONS:
|
|
269
|
+
raise FormulaValueError(f"Dynamic-array formula at {worksheet.title}!{cell.coordinate}")
|
|
270
|
+
if name.startswith("_XLFN.") or name.startswith("_XLWS.") or name not in _BUILTIN_FUNCTIONS:
|
|
271
|
+
raise FormulaValueError(f"Unknown/UDF formula function {name} at {worksheet.title}!{cell.coordinate}")
|
|
272
|
+
formulas.append((worksheet.title, cell.coordinate, formula))
|
|
273
|
+
_detect_formula_cycles(workbook, allow_errors=allow_errors)
|
|
274
|
+
return formulas
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _detect_formula_cycles(workbook: Any, *, allow_errors: bool = False) -> None:
|
|
278
|
+
"""Fail closed on cycles or references we cannot statically account for."""
|
|
279
|
+
nodes = {(ws.title, cell.coordinate): cell for ws, cell in _iter_formula_cells(workbook)}
|
|
280
|
+
graph = {}
|
|
281
|
+
sheet_names = {ws.title.casefold(): ws.title for ws in workbook.worksheets}
|
|
282
|
+
global_names = {name.casefold(): value for name, value in workbook.defined_names.items()}
|
|
283
|
+
local_names = {ws.title: {name.casefold(): value for name, value in ws.defined_names.items()}
|
|
284
|
+
for ws in workbook.worksheets}
|
|
285
|
+
by_sheet = {}
|
|
286
|
+
for key, cell in nodes.items():
|
|
287
|
+
by_sheet.setdefault(key[0], []).append((key, cell.row, cell.column))
|
|
288
|
+
ref_re = re.compile(r"^(?:\$?[A-Z]{1,3}\$?\d+(?::\$?[A-Z]{1,3}\$?\d+)?|\$?[A-Z]{1,3}:\$?[A-Z]{1,3}|\$?\d+:\$?\d+)$", re.I)
|
|
289
|
+
|
|
290
|
+
def dependencies(formula, context_sheet, name_stack=()):
|
|
291
|
+
try:
|
|
292
|
+
tokens = Tokenizer(formula if formula.startswith('=') else '=' + formula).items
|
|
293
|
+
except Exception as exc:
|
|
294
|
+
raise FormulaValueError(f"Unable to parse formula dependencies in {context_sheet}") from exc
|
|
295
|
+
result = set()
|
|
296
|
+
for token in tokens:
|
|
297
|
+
if token.type != "OPERAND" or token.subtype != "RANGE":
|
|
298
|
+
continue
|
|
299
|
+
raw = token.value.strip()
|
|
300
|
+
if allow_errors and "#REF!" in raw.upper():
|
|
301
|
+
# A deleted reference (including Sheet!#REF!) has no target
|
|
302
|
+
# node. Excel still computes and records its typed error;
|
|
303
|
+
# other dependencies and cycles remain fully checked.
|
|
304
|
+
continue
|
|
305
|
+
if "!" in raw:
|
|
306
|
+
sheet_text, raw_ref = raw.rsplit("!", 1)
|
|
307
|
+
if ":" in sheet_text:
|
|
308
|
+
raise FormulaValueError(f"Unsupported 3-D formula reference: {raw}")
|
|
309
|
+
if sheet_text.startswith("'") and sheet_text.endswith("'"):
|
|
310
|
+
sheet_text = sheet_text[1:-1].replace("''", "'")
|
|
311
|
+
target_sheet = sheet_names.get(sheet_text.casefold())
|
|
312
|
+
if target_sheet is None:
|
|
313
|
+
raise FormulaValueError(f"Unknown formula reference sheet: {raw}")
|
|
314
|
+
else:
|
|
315
|
+
raw_ref, target_sheet = raw, context_sheet
|
|
316
|
+
if ref_re.fullmatch(raw_ref):
|
|
317
|
+
# Relative references in defined names are anchored to Excel's
|
|
318
|
+
# active-cell context, not necessarily the formula cell. Do
|
|
319
|
+
# not guess that context when checking dependencies.
|
|
320
|
+
if name_stack and not re.fullmatch(
|
|
321
|
+
r'(?:\$[A-Z]{1,3}\$\d+(?::\$[A-Z]{1,3}\$\d+)?|\$[A-Z]{1,3}:\$[A-Z]{1,3}|\$\d+:\$\d+)',
|
|
322
|
+
raw_ref, re.I,
|
|
323
|
+
):
|
|
324
|
+
raise FormulaValueError(f"Relative defined-name reference is unsupported: {raw}")
|
|
325
|
+
min_col, min_row, max_col, max_row = range_boundaries(raw_ref.upper())
|
|
326
|
+
min_col, min_row = min_col or 1, min_row or 1
|
|
327
|
+
max_col, max_row = max_col or 16384, max_row or 1048576
|
|
328
|
+
if not (1 <= min_col <= max_col <= 16384 and 1 <= min_row <= max_row <= 1048576):
|
|
329
|
+
raise FormulaValueError(f"Invalid formula range: {raw}")
|
|
330
|
+
result.update(key for key, row, col in by_sheet.get(target_sheet, [])
|
|
331
|
+
if min_col <= col <= max_col and min_row <= row <= max_row)
|
|
332
|
+
else:
|
|
333
|
+
name_key = raw_ref.casefold()
|
|
334
|
+
defined = local_names.get(target_sheet, {}).get(name_key) or global_names.get(name_key)
|
|
335
|
+
if defined is None:
|
|
336
|
+
raise FormulaValueError(f"Unsupported or unknown formula reference: {raw}")
|
|
337
|
+
identity = (target_sheet, name_key)
|
|
338
|
+
if identity in name_stack or len(name_stack) >= 100:
|
|
339
|
+
raise FormulaValueError(f"Circular or excessively nested defined name: {raw}")
|
|
340
|
+
result.update(dependencies(str(defined.attr_text or ''), target_sheet, (*name_stack, identity)))
|
|
341
|
+
return result
|
|
342
|
+
|
|
343
|
+
for node, cell in nodes.items():
|
|
344
|
+
formula = cell.value.text if hasattr(cell.value, "text") else str(cell.value)
|
|
345
|
+
graph[node] = dependencies(formula, node[0])
|
|
346
|
+
# Iterative topological elimination supports long formula chains.
|
|
347
|
+
reverse = {node: set() for node in graph}
|
|
348
|
+
counts = {node: len(edges) for node, edges in graph.items()}
|
|
349
|
+
for node, edges in graph.items():
|
|
350
|
+
for edge in edges:
|
|
351
|
+
reverse[edge].add(node)
|
|
352
|
+
ready = [node for node, count in counts.items() if count == 0]
|
|
353
|
+
visited = 0
|
|
354
|
+
while ready:
|
|
355
|
+
node = ready.pop()
|
|
356
|
+
visited += 1
|
|
357
|
+
for dependent in reverse[node]:
|
|
358
|
+
counts[dependent] -= 1
|
|
359
|
+
if counts[dependent] == 0:
|
|
360
|
+
ready.append(dependent)
|
|
361
|
+
if visited != len(graph):
|
|
362
|
+
node = next(node for node, count in counts.items() if count)
|
|
363
|
+
raise FormulaValueError(f"Circular formula dependency detected at {node[0]}!{node[1]}")
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _scan_defined_names(workbook: Any, *, allow_errors: bool = False) -> None:
|
|
367
|
+
"""Reject names which smuggle an external link or executable reference."""
|
|
368
|
+
all_names = [*workbook.defined_names.values(),
|
|
369
|
+
*(name for ws in workbook.worksheets for name in ws.defined_names.values())]
|
|
370
|
+
for defined_name in all_names:
|
|
371
|
+
text = str(getattr(defined_name, "attr_text", "") or "")
|
|
372
|
+
scan_text = _without_string_literals(text)
|
|
373
|
+
scan_upper = scan_text.upper()
|
|
374
|
+
if _EXTERNAL_REF_RE.search(scan_text) or re.search(r"\bDDE\s*\(", scan_text, re.IGNORECASE) or ("#REF!" in scan_upper and not allow_errors):
|
|
375
|
+
name = getattr(defined_name, "name", "<unnamed>")
|
|
376
|
+
raise FormulaValueError(f"Defined name contains an unsafe reference: {name}")
|
|
377
|
+
name = getattr(defined_name, "name", "<unnamed>")
|
|
378
|
+
if any(re.search(rf"(?<![A-Z0-9_.]){re.escape(function)}\s*\(", scan_upper)
|
|
379
|
+
for function in _VOLATILE_FUNCTIONS):
|
|
380
|
+
raise FormulaValueError(f"Defined name contains a volatile formula: {name}")
|
|
381
|
+
if _DYNAMIC_SPILL_RE.search(scan_text) or "@" in scan_text:
|
|
382
|
+
raise FormulaValueError(f"Defined name contains a dynamic-array formula: {name}")
|
|
383
|
+
for match in _FUNCTION_CALL_RE.finditer(scan_text):
|
|
384
|
+
function = match.group(1).upper()
|
|
385
|
+
if function in _DYNAMIC_FUNCTIONS or function.startswith(("_XLFN.", "_XLWS.")) \
|
|
386
|
+
or function not in _BUILTIN_FUNCTIONS:
|
|
387
|
+
raise FormulaValueError(f"Defined name contains unknown/UDF function {function}: {name}")
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _read_state(path: Path) -> dict[str, Any]:
|
|
391
|
+
try:
|
|
392
|
+
text = path.read_text(encoding="utf-8-sig")
|
|
393
|
+
value = json.loads(text)
|
|
394
|
+
return value if isinstance(value, dict) else {}
|
|
395
|
+
except (OSError, ValueError, json.JSONDecodeError):
|
|
396
|
+
return {}
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _cleanup_excel_process(state_path: Path) -> None:
|
|
400
|
+
"""Stop only the Excel PID and start time recorded by our PS harness."""
|
|
401
|
+
state = _read_state(state_path)
|
|
402
|
+
pid = state.get("pid")
|
|
403
|
+
start_filetime = state.get("startFileTimeUtc")
|
|
404
|
+
if not isinstance(pid, int) or not isinstance(start_filetime, int):
|
|
405
|
+
return
|
|
406
|
+
helper = Path(__file__).with_name("recalculate_workbook.ps1")
|
|
407
|
+
try:
|
|
408
|
+
# Pass ownership data as JSON, never as PowerShell source/arguments.
|
|
409
|
+
# The helper validates both PID and process creation time before stop.
|
|
410
|
+
subprocess.run(
|
|
411
|
+
[_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
|
412
|
+
"-File", str(helper), "-CleanupParametersPath", str(state_path)],
|
|
413
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=False,
|
|
414
|
+
)
|
|
415
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
416
|
+
pass
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _run_excel_recalculate(source: Path, result: Path, timeout_seconds: int) -> None:
|
|
420
|
+
from prepare_progress import checkpoint, cancellable, PreparationCancelled
|
|
421
|
+
script = Path(__file__).with_name("recalculate_workbook.ps1")
|
|
422
|
+
if not script.is_file():
|
|
423
|
+
raise FormulaValueError(f"Excel recalculation helper is missing: {script}")
|
|
424
|
+
with tempfile.TemporaryDirectory(prefix="xlmerge-excel-state-") as state_dir:
|
|
425
|
+
state_path = Path(state_dir) / "state.json"
|
|
426
|
+
params_path = Path(state_dir) / "params.json"
|
|
427
|
+
params_path.write_text(json.dumps({
|
|
428
|
+
"source": str(source), "result": str(result), "state": str(state_path),
|
|
429
|
+
"timeoutSeconds": int(timeout_seconds),
|
|
430
|
+
}), encoding="utf-8")
|
|
431
|
+
command = [_powershell_executable(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
|
432
|
+
"-File", str(script), "-ParametersPath", str(params_path)]
|
|
433
|
+
try:
|
|
434
|
+
if not cancellable():
|
|
435
|
+
completed = subprocess.run(command, capture_output=True, text=True,
|
|
436
|
+
encoding="utf-8", errors="replace",
|
|
437
|
+
timeout=max(1, int(timeout_seconds) + 20), check=False)
|
|
438
|
+
else:
|
|
439
|
+
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
440
|
+
text=True, encoding='utf-8', errors='replace')
|
|
441
|
+
deadline = time.monotonic() + max(1, int(timeout_seconds) + 20)
|
|
442
|
+
try:
|
|
443
|
+
while True:
|
|
444
|
+
checkpoint()
|
|
445
|
+
remaining = deadline - time.monotonic()
|
|
446
|
+
if remaining <= 0:
|
|
447
|
+
raise subprocess.TimeoutExpired(command, timeout_seconds)
|
|
448
|
+
try:
|
|
449
|
+
stdout, stderr = process.communicate(timeout=min(.25, remaining))
|
|
450
|
+
completed = subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
|
|
451
|
+
break
|
|
452
|
+
except subprocess.TimeoutExpired:
|
|
453
|
+
continue
|
|
454
|
+
except BaseException:
|
|
455
|
+
# Cleanup before terminating the harness; its PID/start time
|
|
456
|
+
# ownership record excludes Excel instances opened by users.
|
|
457
|
+
_cleanup_excel_process(state_path)
|
|
458
|
+
process.terminate()
|
|
459
|
+
try:
|
|
460
|
+
process.communicate(timeout=10)
|
|
461
|
+
except subprocess.TimeoutExpired:
|
|
462
|
+
process.kill()
|
|
463
|
+
process.communicate()
|
|
464
|
+
raise
|
|
465
|
+
except subprocess.TimeoutExpired as exc:
|
|
466
|
+
_cleanup_excel_process(state_path)
|
|
467
|
+
raise FormulaValueError("Excel recalculation exceeded the hard timeout") from exc
|
|
468
|
+
finally:
|
|
469
|
+
# The helper normally closes Excel itself. This also handles a
|
|
470
|
+
# PowerShell error path without ever scanning/killing user Excel.
|
|
471
|
+
if state_path.exists() and _read_state(state_path).get("status") not in {"completed", "failed"}:
|
|
472
|
+
_cleanup_excel_process(state_path)
|
|
473
|
+
if completed.returncode != 0:
|
|
474
|
+
state = _read_state(state_path)
|
|
475
|
+
detail = state.get("error") or completed.stderr.strip() or completed.stdout.strip()
|
|
476
|
+
raise FormulaValueError(f"Excel could not safely calculate workbook: {detail or 'unknown error'}")
|
|
477
|
+
if not result.is_file() or result.stat().st_size == 0:
|
|
478
|
+
raise FormulaValueError("Excel reported success but did not export a result workbook")
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _set_literal(cell: Any, value: Any) -> None:
|
|
482
|
+
"""Assign a value without allowing a string beginning '=' to become a formula."""
|
|
483
|
+
if isinstance(value, str):
|
|
484
|
+
# openpyxl classifies strings such as '#N/A' as error cells and strings
|
|
485
|
+
# beginning '=' as formulas. Cached formula text is data, regardless
|
|
486
|
+
# of its spelling, so force the serialized string type for all text.
|
|
487
|
+
cell.data_type = "s"
|
|
488
|
+
if value == "":
|
|
489
|
+
# openpyxl normally omits an empty string cell entirely on write,
|
|
490
|
+
# which reloads as None. A zero-length rich-text run is a valid
|
|
491
|
+
# inline string and round-trips as a typed empty string.
|
|
492
|
+
cell._value = CellRichText(TextBlock(InlineFont(), ""))
|
|
493
|
+
else:
|
|
494
|
+
cell._value = value
|
|
495
|
+
else:
|
|
496
|
+
cell.value = value
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _validate_result(value: Any, worksheet: str, coordinate: str, data_type: str | None = None) -> None:
|
|
500
|
+
# Use the serialized cell type, not the displayed text: a formula may
|
|
501
|
+
# intentionally return the literal string "#N/A", while an Excel error is
|
|
502
|
+
# represented by cell type ``e``.
|
|
503
|
+
if data_type == "e":
|
|
504
|
+
raise FormulaValueError(f"Excel returned an error value at {worksheet}!{coordinate}: {value}")
|
|
505
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
506
|
+
raise FormulaValueError(f"Excel returned non-finite numeric value at {worksheet}!{coordinate}")
|
|
507
|
+
if value is not None and not isinstance(value, (str, bool, int, float, date, datetime, datetime_time)):
|
|
508
|
+
raise FormulaValueError(f"Unsupported Excel result type at {worksheet}!{coordinate}: {type(value).__name__}")
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _formula_cache_presence(path: Path) -> dict[tuple[str, str], bool]:
|
|
512
|
+
"""Return whether each formula cell has a serialized ``<v>`` cache.
|
|
513
|
+
|
|
514
|
+
Excel writes an empty-string result as ``<v></v>`` (which openpyxl exposes
|
|
515
|
+
as ``None`` on some Excel versions), whereas an uncalculated formula has no
|
|
516
|
+
``<v>`` element. Keeping this distinction is essential for fail-closed
|
|
517
|
+
calculation verification.
|
|
518
|
+
"""
|
|
519
|
+
result: dict[tuple[str, str], bool] = {}
|
|
520
|
+
try:
|
|
521
|
+
with zipfile.ZipFile(path) as archive:
|
|
522
|
+
wb_root = ElementTree.fromstring(archive.read("xl/workbook.xml"))
|
|
523
|
+
rel_root = ElementTree.fromstring(archive.read("xl/_rels/workbook.xml.rels"))
|
|
524
|
+
rel_targets = {}
|
|
525
|
+
for rel in rel_root:
|
|
526
|
+
rid = rel.attrib.get("Id") or rel.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
|
|
527
|
+
target = rel.attrib.get("Target")
|
|
528
|
+
if rid and target:
|
|
529
|
+
rel_targets[rid] = "xl/" + target.lstrip("/") if not target.startswith("/") else target.lstrip("/")
|
|
530
|
+
for sheet in wb_root.iter():
|
|
531
|
+
if sheet.tag.rsplit("}", 1)[-1] != "sheet":
|
|
532
|
+
continue
|
|
533
|
+
sheet_name = sheet.attrib.get("name")
|
|
534
|
+
rid = sheet.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
|
|
535
|
+
target = rel_targets.get(rid)
|
|
536
|
+
if not sheet_name or not target or target not in archive.namelist():
|
|
537
|
+
continue
|
|
538
|
+
root = ElementTree.fromstring(archive.read(target))
|
|
539
|
+
for cell in root.iter():
|
|
540
|
+
if cell.tag.rsplit("}", 1)[-1] != "c" or not cell.attrib.get("r"):
|
|
541
|
+
continue
|
|
542
|
+
if any(child.tag.rsplit("}", 1)[-1] == "f" for child in cell):
|
|
543
|
+
result[(sheet_name, cell.attrib["r"])] = any(
|
|
544
|
+
child.tag.rsplit("}", 1)[-1] == "v" for child in cell
|
|
545
|
+
)
|
|
546
|
+
except (OSError, KeyError, zipfile.BadZipFile, ElementTree.ParseError):
|
|
547
|
+
return {}
|
|
548
|
+
return result
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _materialize_from_result(source: Path, calculated: Path, output: Path,
|
|
552
|
+
formula_keys: set[tuple[str, str]], *, allow_errors: bool = False) -> Path:
|
|
553
|
+
keep_vba = source.suffix.lower() == ".xlsm"
|
|
554
|
+
try:
|
|
555
|
+
original = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
|
|
556
|
+
values = load_workbook(calculated, data_only=True, keep_links=False, keep_vba=keep_vba)
|
|
557
|
+
except Exception as exc:
|
|
558
|
+
raise FormulaValueError(f"Cannot read Excel calculation result: {exc}") from exc
|
|
559
|
+
try:
|
|
560
|
+
cache_presence = _formula_cache_presence(calculated)
|
|
561
|
+
for sheet_name, coordinate in formula_keys:
|
|
562
|
+
if sheet_name not in values.sheetnames or sheet_name not in original.sheetnames:
|
|
563
|
+
raise FormulaValueError(f"Calculation result lost sheet {sheet_name!r}")
|
|
564
|
+
source_cell = original[sheet_name][coordinate]
|
|
565
|
+
result_cell = values[sheet_name][coordinate]
|
|
566
|
+
value = result_cell.value
|
|
567
|
+
if result_cell.data_type == "e" and allow_errors:
|
|
568
|
+
if not isinstance(value, str) or value not in _ERROR_VALUES:
|
|
569
|
+
raise FormulaValueError(f"Invalid Excel error result at {sheet_name}!{coordinate}: {value}")
|
|
570
|
+
# Historical errors are typed unknowns, never literal text or
|
|
571
|
+
# invented numeric results. Consumers must require review.
|
|
572
|
+
source_cell.value = value
|
|
573
|
+
source_cell.data_type = "e"
|
|
574
|
+
continue
|
|
575
|
+
_validate_result(value, sheet_name, coordinate, result_cell.data_type)
|
|
576
|
+
# A missing <v> cache is different from a legitimate cached empty
|
|
577
|
+
# string. Excel's data_only reader returns None in both unusual
|
|
578
|
+
# cases; normal formulas returning "" have a string cache and are
|
|
579
|
+
# accepted. Missing cache is rejected by the post-save formula
|
|
580
|
+
# count/result verification below if Excel did not calculate it.
|
|
581
|
+
if value is None:
|
|
582
|
+
if result_cell.data_type not in ('s', 'str') or not cache_presence.get((sheet_name, coordinate), False):
|
|
583
|
+
raise FormulaValueError(f"Excel did not provide a cached result at {sheet_name}!{coordinate}")
|
|
584
|
+
# Excel's empty-string cache can be represented by an empty
|
|
585
|
+
# <v> with no Python value. It is still a string result.
|
|
586
|
+
value = ""
|
|
587
|
+
_set_literal(source_cell, value)
|
|
588
|
+
|
|
589
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
590
|
+
temp_output = output.with_name(f".{output.stem}.tmp-{os.getpid()}-{time.time_ns()}{output.suffix}")
|
|
591
|
+
try:
|
|
592
|
+
original.save(temp_output)
|
|
593
|
+
check = load_workbook(temp_output, data_only=False, keep_links=False, keep_vba=keep_vba)
|
|
594
|
+
try:
|
|
595
|
+
leftovers = [(ws.title, c.coordinate) for ws, c in _iter_formula_cells(check)]
|
|
596
|
+
finally:
|
|
597
|
+
check.close()
|
|
598
|
+
if leftovers:
|
|
599
|
+
raise FormulaValueError(f"Formula values remained after materialization: {leftovers[:3]}")
|
|
600
|
+
os.replace(temp_output, output)
|
|
601
|
+
finally:
|
|
602
|
+
if temp_output.exists():
|
|
603
|
+
temp_output.unlink(missing_ok=True)
|
|
604
|
+
finally:
|
|
605
|
+
original.close()
|
|
606
|
+
values.close()
|
|
607
|
+
return output
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def materialize_values(source_path: Path | str, output_path: Path | str, *, allow_errors: bool = False) -> Path:
|
|
611
|
+
"""Calculate every formula in ``source_path`` and emit a literal snapshot.
|
|
612
|
+
|
|
613
|
+
The source is never opened by Excel and is never modified. A workbook with
|
|
614
|
+
no formulas is returned unchanged, as an explicit fast path.
|
|
615
|
+
"""
|
|
616
|
+
source = Path(source_path).resolve()
|
|
617
|
+
output = Path(output_path).resolve()
|
|
618
|
+
if not source.is_file():
|
|
619
|
+
raise FormulaValueError(f"Workbook does not exist: {source}")
|
|
620
|
+
if source == output:
|
|
621
|
+
raise FormulaValueError("Output path must differ from source path")
|
|
622
|
+
if source.suffix.lower() not in {".xlsx", ".xlsm"}:
|
|
623
|
+
raise FormulaValueError(f"Unsupported workbook extension: {source.suffix}")
|
|
624
|
+
if output.suffix.lower() != source.suffix.lower():
|
|
625
|
+
raise FormulaValueError("Output extension must match source extension")
|
|
626
|
+
|
|
627
|
+
_scan_package(source)
|
|
628
|
+
keep_vba = source.suffix.lower() == ".xlsm"
|
|
629
|
+
try:
|
|
630
|
+
workbook = load_workbook(source, data_only=False, keep_links=True, keep_vba=keep_vba)
|
|
631
|
+
except Exception as exc:
|
|
632
|
+
raise FormulaValueError(f"Cannot open workbook for formula inspection: {exc}") from exc
|
|
633
|
+
try:
|
|
634
|
+
formulas = _scan_formulas(workbook, allow_errors=allow_errors)
|
|
635
|
+
_scan_defined_names(workbook, allow_errors=allow_errors)
|
|
636
|
+
finally:
|
|
637
|
+
workbook.close()
|
|
638
|
+
if not formulas:
|
|
639
|
+
return source
|
|
640
|
+
|
|
641
|
+
timeout_setting = os.environ.get("XLMERGE_EXCEL_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))
|
|
642
|
+
try:
|
|
643
|
+
timeout_seconds = max(1, int(timeout_setting))
|
|
644
|
+
except ValueError as exc:
|
|
645
|
+
raise FormulaValueError("XLMERGE_EXCEL_TIMEOUT_SECONDS must be an integer") from exc
|
|
646
|
+
|
|
647
|
+
temp_dir = Path(tempfile.mkdtemp(prefix="xlmerge-formula-values-"))
|
|
648
|
+
temporary_source = temp_dir / source.name
|
|
649
|
+
temporary_result = temp_dir / ("calculated" + source.suffix.lower())
|
|
650
|
+
try:
|
|
651
|
+
shutil.copy2(source, temporary_source)
|
|
652
|
+
_run_excel_recalculate(temporary_source, temporary_result, timeout_seconds)
|
|
653
|
+
return _materialize_from_result(source, temporary_result, output,
|
|
654
|
+
{(sheet, coordinate) for sheet, coordinate, _ in formulas},
|
|
655
|
+
allow_errors=allow_errors)
|
|
656
|
+
finally:
|
|
657
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
__all__ = ["FormulaValueError", "materialize_values"]
|