dsh-git-push 2.13.2 → 2.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/git/clone-download.js +82 -6
- package/lib/self/index.js +1 -1
- package/package.json +1 -1
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* 以 sha 命名——sha 变了自然换文件,不会把新旧内容拼成损坏文件。
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
|
-
import { createWriteStream, existsSync, statSync } from 'node:fs';
|
|
25
|
+
import { createWriteStream, existsSync, statSync, readFileSync } from 'node:fs';
|
|
26
26
|
import { mkdir, rename, unlink, chmod, symlink, rm, readdir, rmdir, access } from 'node:fs/promises';
|
|
27
27
|
import { dirname, join } from 'node:path';
|
|
28
28
|
import { pipeline } from 'node:stream/promises';
|
|
@@ -130,7 +130,7 @@ async function downloadRegularEntry(entry, { owner, repo, branch, token, out, pa
|
|
|
130
130
|
const size = Number(entry.size) || 0;
|
|
131
131
|
try {
|
|
132
132
|
const mode = posixModeOf(entry.mode);
|
|
133
|
-
const r = await fetchToFile({ owner, repo, branch, relPath: rel, token, out, partPath, size, signal, onBytes, mode });
|
|
133
|
+
const r = await fetchToFile({ owner, repo, branch, relPath: rel, token, out, partPath, size, signal, onBytes, mode, expectSha: entry.sha });
|
|
134
134
|
if (!r.ok) return { failed: { path: rel, status: r.status, reason: r.reason } };
|
|
135
135
|
let modePreserved = true;
|
|
136
136
|
// 普通文件也必须设:原实现只处理 100755,普通文件保留「写入时的权限」,
|
|
@@ -312,7 +312,19 @@ export async function removeDirForce(dir, attempts = 3) {
|
|
|
312
312
|
try { await access(dir); return false; } catch { return true; } // 不存在即视为成功
|
|
313
313
|
}
|
|
314
314
|
|
|
315
|
-
|
|
315
|
+
/** 从响应头提取「该源声明的文件总长」:206 用 content-range 的 /total,200 用 content-length。0=无从判断。 */
|
|
316
|
+
function totalFromResponse(resp) {
|
|
317
|
+
try {
|
|
318
|
+
if (resp.status === 206) {
|
|
319
|
+
const m = /bytes \d+-\d+\/(\d+)/.exec(String(resp.headers?.get?.('content-range') || ''));
|
|
320
|
+
return m ? Number(m[1]) : 0;
|
|
321
|
+
}
|
|
322
|
+
const cl = Number(resp.headers?.get?.('content-length'));
|
|
323
|
+
return Number.isFinite(cl) && cl > 0 ? cl : 0;
|
|
324
|
+
} catch { return 0; }
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function fetchToFile({ owner, repo, branch, relPath, token, out, partPath, size, onBytes, signal = null, mode = 0o644, expectSha = '' }) {
|
|
316
328
|
if (signal?.aborted) return { ok: false, status: 0, reason: '已中止' };
|
|
317
329
|
// 复用**上次已下完的最终文件**:clone 失败后目录会被保留(见 clone.js 的 kept 语义),
|
|
318
330
|
// 那些已 rename 到位、长度与远端一致的文件无需重下。
|
|
@@ -358,8 +370,10 @@ async function fetchToFile({ owner, repo, branch, relPath, token, out, partPath,
|
|
|
358
370
|
|
|
359
371
|
// 依次尝试候选通道,首个成功即用;全失败则回报最后一条的成因
|
|
360
372
|
let resp = null;
|
|
373
|
+
let respIdx = -1;
|
|
361
374
|
let lastErr = { status: 0, reason: '无可用通道' };
|
|
362
|
-
for (
|
|
375
|
+
for (let ci = 0; ci < urls.length; ci++) {
|
|
376
|
+
const [candidate, withRange] = urls[ci];
|
|
363
377
|
let r;
|
|
364
378
|
try {
|
|
365
379
|
r = await fetch(candidate, { headers: mkHeaders(withRange, withRange), ...(signal ? { signal } : {}) });
|
|
@@ -367,19 +381,71 @@ async function fetchToFile({ owner, repo, branch, relPath, token, out, partPath,
|
|
|
367
381
|
lastErr = { status: 0, reason: `请求失败: ${e?.message || e}` };
|
|
368
382
|
continue;
|
|
369
383
|
}
|
|
370
|
-
if (r.status === 200 || r.status === 206) { resp = r; break; }
|
|
384
|
+
if (r.status === 200 || r.status === 206) { resp = r; respIdx = ci; break; }
|
|
371
385
|
let detail = '';
|
|
372
386
|
try { detail = (await r.text()).slice(0, HTTP_DETAIL_MAX_LEN); } catch { /* 忽略 */ }
|
|
373
387
|
lastErr = { status: r.status, reason: `HTTP ${r.status}${detail ? ': ' + detail : ''}` };
|
|
374
388
|
}
|
|
375
389
|
if (!resp) return { ok: false, status: lastErr.status, reason: lastErr.reason };
|
|
390
|
+
// 镜像坏缓存防御(实测事故):某镜像对一个 8MB 文件返回过 total=590454 的**截断资源**
|
|
391
|
+
// (content-range 尾段),带 Range 续传时每轮「已到该源 total、无法增长」⇒ 长度校验
|
|
392
|
+
// 永远失败、分片停在半截、clone 每轮死在同一文件。对策:校验响应声明的总量
|
|
393
|
+
// (206 取 content-range 的 /total,200 取 content-length),与仓库记录的 size
|
|
394
|
+
// 对不上 ⇒ 该通道数据异常:cancel 响应体、**绝不写入分片**(防脏字节拼接),
|
|
395
|
+
// 换下一条候选通道重试;全部不符则本轮判失败(分片保留,下轮再试)。
|
|
396
|
+
if (size > 0) {
|
|
397
|
+
const declaredTotal = totalFromResponse(resp);
|
|
398
|
+
if (declaredTotal > 0 && declaredTotal !== size) {
|
|
399
|
+
try { await resp.body?.cancel?.(); } catch { /* 无所谓 */ }
|
|
400
|
+
lastErr = { status: resp.status, reason: `通道数据异常(声明总长 ${declaredTotal} ≠ 仓库记录 ${size},疑似代理截断缓存),弃用该源` };
|
|
401
|
+
let rescued = false;
|
|
402
|
+
for (let ci = 0; ci < urls.length; ci++) {
|
|
403
|
+
if (ci === respIdx) continue;
|
|
404
|
+
const [candidate, withRange] = urls[ci];
|
|
405
|
+
let r2;
|
|
406
|
+
try {
|
|
407
|
+
r2 = await fetch(candidate, { headers: mkHeaders(withRange, withRange), ...(signal ? { signal } : {}) });
|
|
408
|
+
} catch (e) {
|
|
409
|
+
lastErr = { status: 0, reason: `备用通道请求失败: ${e?.message || e}` };
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
if (r2.status !== 200 && r2.status !== 206) {
|
|
413
|
+
try { await r2.text?.(); } catch { /* 无所谓 */ }
|
|
414
|
+
lastErr = { status: r2.status, reason: `备用通道 HTTP ${r2.status}` };
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
const t2 = totalFromResponse(r2);
|
|
418
|
+
if (t2 > 0 && t2 !== size) {
|
|
419
|
+
try { await r2.body?.cancel?.(); } catch { /* 无所谓 */ }
|
|
420
|
+
lastErr = { status: r2.status, reason: `备用通道同样数据异常(声明总长 ${t2} ≠ ${size})` };
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
resp = r2;
|
|
424
|
+
rescued = true;
|
|
425
|
+
break;
|
|
426
|
+
}
|
|
427
|
+
if (!rescued) return { ok: false, status: lastErr.status, reason: lastErr.reason };
|
|
428
|
+
}
|
|
429
|
+
}
|
|
376
430
|
// 请求了 Range 却回 200:服务端忽略续传(或文件已变)→ 清空重下,避免拼接错位
|
|
377
431
|
const appending = have > 0 && resp.status === 206;
|
|
378
432
|
if (!appending && have > 0) { try { await unlink(partPath); } catch { /* 忽略 */ } }
|
|
379
433
|
try {
|
|
380
434
|
const ws = createWriteStream(partPath, { flags: appending ? 'a' : 'w', mode }); // mode 必须显式:否则权限随 umask/CIFS 落到 777
|
|
381
435
|
const src = Readable.fromWeb(resp.body);
|
|
382
|
-
|
|
436
|
+
// 增量校验 git blob sha(与 tree 记录比对):长度校验只能挡「截断」,
|
|
437
|
+
// 挡不住「total 相符但内容不同」的镜像陈旧缓存;脏前缀若留着,续传永远拼不出
|
|
438
|
+
// 正确文件(实测事故里分片就是混合了截断内容的脏数据)。流式 update 成本≈0。
|
|
439
|
+
let hasher = null;
|
|
440
|
+
if (expectSha && size > 0) {
|
|
441
|
+
const { createHash } = await import('node:crypto');
|
|
442
|
+
hasher = createHash('sha1');
|
|
443
|
+
hasher.update(`blob ${size}\u0000`, 'latin1'); // git object header( 按 latin1 保持单字节)
|
|
444
|
+
if (appending && have > 0) {
|
|
445
|
+
try { hasher.update(readFileSync(partPath)); } catch { hasher = null; }
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
src.on('data', (c) => { if (onBytes) onBytes(c.length); if (hasher) hasher.update(c); });
|
|
383
449
|
await pipeline(src, ws);
|
|
384
450
|
// 长度校验:能校验则校验,避免「悄悄写了个残文件」
|
|
385
451
|
if (size > 0) {
|
|
@@ -389,6 +455,16 @@ async function fetchToFile({ owner, repo, branch, relPath, token, out, partPath,
|
|
|
389
455
|
return { ok: false, status: resp.status, reason: `长度不符(期望 ${size},实得 ${finalSize})` };
|
|
390
456
|
}
|
|
391
457
|
}
|
|
458
|
+
// sha 校验不符 ⇒ 分片内容被污染(脏前缀/镜像错内容):**废弃分片**让下轮从 0 干净重下
|
|
459
|
+
// —— 留着它续传只会永远凑不出正确文件(这正是「每轮从 0 重来还没救」的死循环形态)。
|
|
460
|
+
// 从 0 起步时 useRawFirst=false,api 直连优先,天然绕开坏镜像通道。
|
|
461
|
+
if (hasher) {
|
|
462
|
+
const got = hasher.digest('hex');
|
|
463
|
+
if (got !== String(expectSha)) {
|
|
464
|
+
try { await unlink(partPath); } catch { /* 忽略 */ }
|
|
465
|
+
return { ok: false, status: resp.status, reason: `内容校验不符(blob sha 期望 ${String(expectSha).slice(0, 10)}…,实得 ${got.slice(0, 10)}…),已废弃分片以便下轮干净重下` };
|
|
466
|
+
}
|
|
467
|
+
}
|
|
392
468
|
await rename(partPath, out);
|
|
393
469
|
return { ok: true };
|
|
394
470
|
} catch (e) {
|
package/lib/self/index.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
export const VERSION = '2.13.
|
|
12
|
+
export const VERSION = '2.13.3'; // 2.13.3:修大仓库克隆「死在同一文件每轮从 0 重来」(实测根因:gh-proxy.com 对某 8MB 文件返回total=590454 的**截断缓存**,带 Range 续传每轮「已到该源 total 无法增长」→ 长度校验永远失败、脏分片留着越拼越歪)。clone-download 三重防御:①响应声明总量校验(206 取 content-range 的 /total、200 取 content-length,与仓库记录 size 不符 ⇒ cancel 响应、不写分片、换候选通道 rescue);②git blob sha 增量校验(流式 update,header 按 latin1 单字节 \0;长度对但内容错——镜像旧缓存——也当场废弃分片,下轮从 0 走 api 直连自愈);③失败保留分片语义不变。新增 test-clone-mirror-truncation 4 例(含真实 git hash 自洽桩);顺带修既有测试两处桩缺陷(两个文件同 sha 撞分片名、预置分片没落在实际 sha 上)。历史:2.13.2 动态权重维度分母。2.13.1 流式 fetch 停摆判据。
|
|
13
13
|
// 实测后果:该目录有 package.json ⇒ npm 去打包整棵 DSH 树、输出 2.1MB > spawnSync 默认 maxBuffer 1MB ⇒ ENOBUFS(npm_publish 的体检与真发全部失败);
|
|
14
14
|
// 该目录没有 package.json 时报 ENOENT。现 publish 的两个调用点显式传 cwd=目标包目录,并给 runNpm 加 16MB maxBuffer。
|
|
15
15
|
// 2.8.3:文案修正——「零改动 / 无新提交」明确为**正常成功路径**(注入用法、工具描述、README、skill 同批改),并修 job 输出自相矛盾(cleanSkipCommit 路径补 committed:false,不再显示「已提交,未推送」)
|