dsh-zju-lecture-note 2.2.6 → 2.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,6 +18,7 @@ Security boundaries:
18
18
 
19
19
  import hashlib
20
20
  import os
21
+ import tempfile
21
22
  import urllib.error
22
23
  import urllib.parse
23
24
  import zlib
@@ -79,6 +80,7 @@ def _download_one(
79
80
  target_dir: str,
80
81
  timeout: int,
81
82
  token: Optional[str],
83
+ diagnostics: Optional[List[str]] = None,
82
84
  ) -> Tuple[int, Optional[Dict[str, Any]], Optional[str]]:
83
85
  url = str(slide.get("image_url") or "").strip()
84
86
  if not url:
@@ -104,6 +106,7 @@ def _download_one(
104
106
  attempts=3,
105
107
  max_bytes=MAX_IMAGE_BYTES,
106
108
  allow_cross_host_redirect=True,
109
+ diagnostics=diagnostics,
107
110
  )
108
111
  except urllib.error.HTTPError as error:
109
112
  if error.code in (401, 403) and token:
@@ -116,6 +119,7 @@ def _download_one(
116
119
  attempts=2,
117
120
  max_bytes=MAX_IMAGE_BYTES,
118
121
  allow_cross_host_redirect=True,
122
+ diagnostics=diagnostics,
119
123
  )
120
124
  else:
121
125
  raise
@@ -129,16 +133,21 @@ def _download_one(
129
133
  return index, None, "响应不是 JPEG/PNG(可能是登录失效页或错误页)"
130
134
 
131
135
  path = os.path.join(target_dir, f"slide-{index + 1:04d}{extension}")
132
- part = path + ".part"
136
+ part = ""
133
137
  try:
134
- with open(part, "wb") as handle:
138
+ descriptor, part = tempfile.mkstemp(
139
+ dir=target_dir, prefix=".slide-", suffix=".part"
140
+ )
141
+ with os.fdopen(descriptor, "wb") as handle:
135
142
  handle.write(data)
136
143
  os.replace(part, path)
144
+ part = ""
137
145
  except OSError as error:
138
- try:
139
- os.unlink(part)
140
- except OSError:
141
- pass
146
+ if part:
147
+ try:
148
+ os.unlink(part)
149
+ except OSError:
150
+ pass
142
151
  return index, None, f"截图写入失败: {error}"
143
152
 
144
153
  return index, {
@@ -188,22 +197,42 @@ def download_slides(
188
197
 
189
198
  results: List[Tuple[int, Optional[Dict[str, Any]], Optional[str]]] = []
190
199
  pool_size = max(1, min(int(workers or DOWNLOAD_WORKERS), 8))
200
+ downloaded_bytes = 0
201
+ stopped = False
191
202
  with ThreadPoolExecutor(max_workers=pool_size) as pool:
192
- futures = [
193
- pool.submit(
194
- _download_one,
195
- index,
196
- slide,
197
- cookie,
198
- sub_id,
199
- target_dir,
200
- timeout,
201
- token,
202
- )
203
- for index, slide in indexed
204
- ]
205
- for future in as_completed(futures):
206
- results.append(future.result())
203
+ # Batch instead of submitting everything at once: the total-size cap
204
+ # has to stop **future** downloads, not just print a warning after the
205
+ # disk is already full.
206
+ for start in range(0, len(indexed), pool_size):
207
+ batch = indexed[start:start + pool_size]
208
+ futures = [
209
+ pool.submit(
210
+ _download_one,
211
+ index,
212
+ slide,
213
+ cookie,
214
+ sub_id,
215
+ target_dir,
216
+ timeout,
217
+ token,
218
+ diagnostics,
219
+ )
220
+ for index, slide in batch
221
+ ]
222
+ for future in as_completed(futures):
223
+ result = future.result()
224
+ results.append(result)
225
+ if result[1] is not None:
226
+ downloaded_bytes += int(result[1]["bytes"])
227
+ if downloaded_bytes >= MAX_TOTAL_BYTES:
228
+ remaining = len(indexed) - len(results)
229
+ _record(
230
+ diagnostics,
231
+ f"截图总大小达到 {MAX_TOTAL_BYTES // (1024 * 1024)} MiB 上限,"
232
+ f"已下载 {len(results)} 张,停止后续 {remaining} 张",
233
+ )
234
+ stopped = True
235
+ break
207
236
  results.sort(key=lambda item: item[0])
208
237
 
209
238
  failures: List[str] = []
@@ -235,12 +264,8 @@ def download_slides(
235
264
  _record(diagnostics, f"另有 {len(failures) - 5} 张截图下载失败")
236
265
  if skipped:
237
266
  _record(diagnostics, f"{skipped} 页没有截图地址(只有文字或空记录)")
238
- if total_bytes > MAX_TOTAL_BYTES:
239
- _record(
240
- diagnostics,
241
- f"截图总大小 {total_bytes // (1024 * 1024)} MiB 超过 "
242
- f"{MAX_TOTAL_BYTES // (1024 * 1024)} MiB,PDF 可能很大",
243
- )
267
+ if stopped and downloaded == 0:
268
+ _record(diagnostics, "截图总大小达到上限且没有成功落盘任何截图")
244
269
 
245
270
  kept: List[Dict[str, Any]] = []
246
271
  duplicates = 0
@@ -392,12 +417,18 @@ def _png_decode(data: bytes) -> Tuple[int, int, bytes, int]:
392
417
  if width * height * channels > 256 * 1024 * 1024:
393
418
  raise SlideError("PNG 解码后超过 256 MiB 上限")
394
419
 
420
+ stride = width * channels
421
+ expected = height * (stride + 1)
422
+ # zlib.decompress() 会先把整个输出物化到内存;用 decompressobj 限制输出,
423
+ # 超出像素数据应有长度就立刻停,而不是等 OOM。
424
+ decompressor = zlib.decompressobj()
395
425
  try:
396
- raw = zlib.decompress(bytes(idat))
426
+ raw = decompressor.decompress(bytes(idat), expected + 1)
397
427
  except zlib.error as error:
398
428
  raise SlideError(f"PNG 解压失败: {error}") from error
429
+ if len(raw) > expected or decompressor.unconsumed_tail:
430
+ raise SlideError("PNG 解压后超过声明的像素数据长度")
399
431
 
400
- stride = width * channels
401
432
  previous = bytearray(stride)
402
433
  samples = bytearray()
403
434
  cursor = 0
@@ -461,9 +492,14 @@ def _page_size(width: int, height: int) -> Tuple[float, float]:
461
492
 
462
493
 
463
494
  def _write_pdf(images: List[Dict[str, Any]], path: str) -> None:
464
- part = path + ".part"
495
+ part = ""
465
496
  try:
466
- with open(part, "wb") as handle:
497
+ descriptor, part = tempfile.mkstemp(
498
+ dir=os.path.dirname(os.path.abspath(path)) or ".",
499
+ prefix=".pdf-",
500
+ suffix=".part",
501
+ )
502
+ with os.fdopen(descriptor, "wb") as handle:
467
503
  count = len(images)
468
504
  total_objects = 2 + count * 3
469
505
  offsets = [0] * (total_objects + 1)
@@ -545,11 +581,13 @@ def _write_pdf(images: List[Dict[str, Any]], path: str) -> None:
545
581
  ).encode("ascii")
546
582
  )
547
583
  os.replace(part, path)
584
+ part = ""
548
585
  except Exception:
549
- try:
550
- os.unlink(part)
551
- except OSError:
552
- pass
586
+ if part:
587
+ try:
588
+ os.unlink(part)
589
+ except OSError:
590
+ pass
553
591
  raise
554
592
 
555
593
 
@@ -1219,7 +1219,7 @@ def typst_string(value: str) -> str:
1219
1219
  return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"'
1220
1220
 
1221
1221
 
1222
- def escape_typst(text: str, known_labels: Optional[Sequence[str]] = None) -> str:
1222
+ def escape_typst_text(text: str, known_labels: Optional[Sequence[str]] = None) -> str:
1223
1223
  """把一段 Markdown 文本转成可以直接放进 Typst markup 的形式。
1224
1224
 
1225
1225
  Typst 的 markup 有几个会咬人的地方,这里逐个处理:
@@ -1281,6 +1281,35 @@ def escape_typst(text: str, known_labels: Optional[Sequence[str]] = None) -> str
1281
1281
  return restored
1282
1282
 
1283
1283
 
1284
+ #: 正文转义的旧名字;保留给调用方与既有测试。
1285
+ escape_typst = escape_typst_text
1286
+
1287
+
1288
+ def escape_typst_string(value: str) -> str:
1289
+ """Typst 字符串字面量内部转义(不含外层引号)。
1290
+
1291
+ ``escape_typst_text`` 是给 markup 用的,不转义双引号——把它塞进
1292
+ ``"…"`` 里,一个 ``"`` 就能闭合字符串并注入 ``#read(...)``。
1293
+ 这里按 Typst 的字符串语法转义 ``\\``、``"``、换行与制表符,并剥掉
1294
+ 裸 ``\\r``(Typst 只认 ``\\n``)。
1295
+ """
1296
+ text = str(value if value is not None else "").replace("\r\n", "\n")
1297
+ text = text.replace("\r", "")
1298
+ out: List[str] = []
1299
+ for char in text:
1300
+ if char == "\\":
1301
+ out.append("\\\\")
1302
+ elif char == '"':
1303
+ out.append('\\"')
1304
+ elif char == "\n":
1305
+ out.append("\\n")
1306
+ elif char == "\t":
1307
+ out.append("\\t")
1308
+ else:
1309
+ out.append(char)
1310
+ return "".join(out)
1311
+
1312
+
1284
1313
  # --------------------------------------------------------------------- 结果类型
1285
1314
 
1286
1315
 
@@ -2237,12 +2266,15 @@ def _render_cover(draft_text: str) -> str:
2237
2266
  if not fields:
2238
2267
  return ""
2239
2268
 
2240
- arguments = [f'"{escape_typst(fields["title"])}"'] if fields.get("title") else ['""']
2269
+ # 全部走命名参数,值按字符串字面量转义;位置参数一旦被注入的引号打断,
2270
+ # 后面的 `#read(...)` 就会变成可执行代码。
2271
+ title = fields.get("title") or ""
2272
+ arguments = [f'title: "{escape_typst_string(title)}"']
2241
2273
  for key in ("subtitle", "author", "date", "kicker"):
2242
2274
  value = fields.get(key)
2243
2275
  if value:
2244
- arguments.append(f'{key}: "{escape_typst(value)}"')
2245
- return "#cover(" + ", ".join(arguments) + ")"
2276
+ arguments.append(f'{key}: "{escape_typst_string(value)}"')
2277
+ return "#scholia-cover(" + ", ".join(arguments) + ")"
2246
2278
 
2247
2279
 
2248
2280
  # ----------------------------------------------------------------- 讲义 → 底稿