dsh-lan-peers 0.7.2 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HANDOVER.md +5 -1
- package/host/server.mjs +308 -29
- package/host/test/group-cluster.mjs +60 -1
- package/host/test/group-election.test.mjs +64 -1
- package/host/test/group-key-rotation.test.mjs +219 -0
- package/host/test/group-takeover.test.mjs +122 -0
- package/host/test/group.test.mjs +2 -1
- package/lib/admin-group.test.mjs +89 -0
- package/lib/admin.mjs +28 -4
- package/package.json +1 -1
package/HANDOVER.md
CHANGED
|
@@ -180,7 +180,11 @@ ssh ai@192.168.5.51 "sudo systemctl restart dsh-web"
|
|
|
180
180
|
4. 重启桌面版。
|
|
181
181
|
|
|
182
182
|
改完文件不必重启:**改 profile 的 `cordis.patch.yml` 会触发热重载**(改包内文件不会,需要动一下 profile 文件)。
|
|
183
|
-
>
|
|
183
|
+
> ✅ **2026-10-27 更正(重要)**:**改 profile `cordis.patch.yml` 的内容会触发插件热重载**——实测桌面版 14:58:06 因新增一行 `hubPriority: 10` 而重载,日志出现新的 `module imported` 与 `listening`,`/v1/health` 从 0.7.0 直接变成 0.7.2(说明它重新 import 了 `link:` 指向的最新代码)。
|
|
184
|
+
> 但**单纯 `touch`(不改内容)不会**触发——早先据此写下的「热重载不可靠」是**错误结论**,已更正。
|
|
185
|
+
> 实用含义:想在不重启应用的前提下加载新的服务端代码,**改一下 profile 配置内容(哪怕加一行注释)** 即可;客户端组件(面板 UI)是否随之刷新仍未验证,界面上看不到变化时仍需重启。
|
|
186
|
+
|
|
187
|
+
> ⚠️ 旧的错误结论(保留作教训):曾记录「touch profile 不触发热重载」——错在只 `touch` 没改内容。
|
|
184
188
|
|
|
185
189
|
## 8. 回归检查清单(发版/部署后跑)
|
|
186
190
|
|
package/host/server.mjs
CHANGED
|
@@ -96,7 +96,7 @@ import { parseArgs } from 'node:util';
|
|
|
96
96
|
// ---------------------------------------------------------------------------
|
|
97
97
|
|
|
98
98
|
/** 插件版本,用于 /v1/health.version */
|
|
99
|
-
export const PLUGIN_VERSION = '0.7.
|
|
99
|
+
export const PLUGIN_VERSION = '0.7.3';
|
|
100
100
|
|
|
101
101
|
/** §4.1 允许的时钟偏移 */
|
|
102
102
|
export const CLOCK_SKEW_MS = 60_000;
|
|
@@ -986,6 +986,9 @@ export class GroupStore {
|
|
|
986
986
|
if (!Array.isArray(data.messages)) data.messages = [];
|
|
987
987
|
// S3 §5:未提交区。与 messages 分开存,读写都不进 history/pull/events。
|
|
988
988
|
if (!Array.isArray(data.pending)) data.pending = [];
|
|
989
|
+
// 被移除的成员:记住它,防止它用「自助 join」把自己加回来(踢人要真的踢掉)。
|
|
990
|
+
// 只有运维显式 addMember 会把这条记录清掉(等价于重新邀请)。
|
|
991
|
+
if (typeof data.blocked !== 'object' || data.blocked === null || Array.isArray(data.blocked)) data.blocked = {};
|
|
989
992
|
data.pending = data.pending.filter((entry) => entry && typeof entry === 'object'
|
|
990
993
|
&& Number.isFinite(Number(entry.seq)) && Number(entry.seq) > 0);
|
|
991
994
|
if (typeof data.members !== 'object' || data.members === null || Array.isArray(data.members)) data.members = {};
|
|
@@ -1044,6 +1047,11 @@ export class GroupStore {
|
|
|
1044
1047
|
const id = String(peerId ?? '').trim();
|
|
1045
1048
|
if (id === '') throw new Error('member peerId must not be empty');
|
|
1046
1049
|
const data = this.#load();
|
|
1050
|
+
// 运维显式加人 = 重新邀请:解除「被移除」记录
|
|
1051
|
+
if (Object.prototype.hasOwnProperty.call(data.blocked ?? {}, id)) {
|
|
1052
|
+
delete data.blocked[id];
|
|
1053
|
+
this.#save(data);
|
|
1054
|
+
}
|
|
1047
1055
|
const existing = data.members[id];
|
|
1048
1056
|
if (!existing) {
|
|
1049
1057
|
data.members[id] = { name: String(name ?? ''), addedAt: nowIso(this.#now()) };
|
|
@@ -1055,11 +1063,24 @@ export class GroupStore {
|
|
|
1055
1063
|
return data.members[id];
|
|
1056
1064
|
}
|
|
1057
1065
|
|
|
1066
|
+
/** 这个 peerId 是否被运维移除过(自助 join 必须拒绝) */
|
|
1067
|
+
isBlocked(peerId) {
|
|
1068
|
+
const data = this.#load();
|
|
1069
|
+
return Object.prototype.hasOwnProperty.call(data.blocked ?? {}, String(peerId ?? ''));
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1058
1072
|
removeMember(peerId) {
|
|
1059
1073
|
const data = this.#load();
|
|
1060
1074
|
const id = String(peerId ?? '');
|
|
1061
1075
|
if (!Object.prototype.hasOwnProperty.call(data.members, id)) return false;
|
|
1062
1076
|
delete data.members[id];
|
|
1077
|
+
// 记住「被移除」:否则它还能用 per-peer 签名走 /v1/group/join(autoJoin)把自己加回来
|
|
1078
|
+
data.blocked[id] = nowIso(this.#now());
|
|
1079
|
+
// 移除成员 = 它可能还握着旧 groupKey:必须立刻轮换密钥,否则「踢人」形同虚设
|
|
1080
|
+
// (被移除者只要还能用旧钥签名,就能把自己的心跳送进来)。
|
|
1081
|
+
data.key = randomBytes(32).toString('base64url');
|
|
1082
|
+
const version = Number(data.keyVersion);
|
|
1083
|
+
data.keyVersion = (Number.isInteger(version) && version >= 1 ? version : 1) + 1;
|
|
1063
1084
|
this.#save(data);
|
|
1064
1085
|
return true;
|
|
1065
1086
|
}
|
|
@@ -1809,7 +1830,7 @@ export function createPeerServer(overrides = {}) {
|
|
|
1809
1830
|
* groupId 一致、senderPeerId 在成员表、ts ±60s、nonce 120s 去重。
|
|
1810
1831
|
* 返回 {ok:true, senderId, ctx} 或 {ok:false, status, error}。
|
|
1811
1832
|
*/
|
|
1812
|
-
function verifyGroupAuth(req, rawBody) {
|
|
1833
|
+
function verifyGroupAuth(req, rawBody, { allowUnknownSender = false } = {}) {
|
|
1813
1834
|
const senderId = headerValue(req, 'x-peer-id');
|
|
1814
1835
|
const groupId = headerValue(req, GROUP_ID_HEADER.toLowerCase());
|
|
1815
1836
|
const tsRaw = headerValue(req, 'x-peer-ts');
|
|
@@ -1824,9 +1845,8 @@ export function createPeerServer(overrides = {}) {
|
|
|
1824
1845
|
const ctx = localGroupContext();
|
|
1825
1846
|
if (!ctx || !ctx.groupKey) return { ok: false, status: 401, error: 'group-auth-unavailable' };
|
|
1826
1847
|
if (String(groupId) !== String(ctx.groupId)) return { ok: false, status: 401, error: 'group-mismatch' };
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
}
|
|
1848
|
+
// 先验签、再查成员表:被移除的成员拿着**旧群密钥**打进来应当是 bad-signature(密钥世代不对),
|
|
1849
|
+
// 而不是 not-a-member —— 这样「移除成员即轮换密钥」才有可验证的信号。
|
|
1830
1850
|
const canonical = buildGroupCanonical({
|
|
1831
1851
|
method: req.method,
|
|
1832
1852
|
path: req.url,
|
|
@@ -1839,6 +1859,15 @@ export function createPeerServer(overrides = {}) {
|
|
|
1839
1859
|
const { signKey } = deriveGroupKeys(ctx.groupKey);
|
|
1840
1860
|
const expected = signGroupCanonical({ signKey, canonical });
|
|
1841
1861
|
if (!timingSafeStringEqual(expected, signature)) return { ok: false, status: 401, error: 'bad-signature' };
|
|
1862
|
+
if (senderId !== store.instanceId && !Object.prototype.hasOwnProperty.call(ctx.members, senderId)) {
|
|
1863
|
+
// hub 是权威成员表:不认识就是不属于群,一律拒绝(自愈只给成员侧的副本用)
|
|
1864
|
+
if (!allowUnknownSender || groupStore) return { ok: false, status: 401, error: 'not-a-member' };
|
|
1865
|
+
// 自愈(真机 not-a-member 回归):群密钥签名已经证明对端握有群密钥,只是本机成员表
|
|
1866
|
+
// 还停在一份不完整的快照上(成员在 bootstrap 之后才加入)。选举/心跳是领导权协议,
|
|
1867
|
+
// 先把它补进成员表,否则对端永远投不进来、整个群选不出 leader。
|
|
1868
|
+
upsertGroupMember(senderId);
|
|
1869
|
+
if (config.logger) config.logger(`[group] member healed peer=${shortPeerId(senderId)}`);
|
|
1870
|
+
}
|
|
1842
1871
|
// 只有签名通过的 nonce 才登记(同 per-peer 路径)
|
|
1843
1872
|
rememberGroupNonce(nonce, nowMs);
|
|
1844
1873
|
return { ok: true, senderId, ctx };
|
|
@@ -2495,6 +2524,11 @@ export function createPeerServer(overrides = {}) {
|
|
|
2495
2524
|
const existing = groupMemberOf(auth.peerId);
|
|
2496
2525
|
if (!existing) {
|
|
2497
2526
|
if (!config.group.autoJoin) return { status: 403, body: { error: 'not-a-member' }, peerId: auth.peerId };
|
|
2527
|
+
// 被移除过的成员:自助 join 不生效(必须有运维显式 addMember 重新邀请)
|
|
2528
|
+
if (groupStore.isBlocked(auth.peerId)) {
|
|
2529
|
+
if (config.logger) config.logger(`[group] join refused peer=${shortPeerId(auth.peerId)} reason=removed`);
|
|
2530
|
+
return { status: 403, body: { error: 'not-a-member' }, peerId: auth.peerId };
|
|
2531
|
+
}
|
|
2498
2532
|
groupStore.addMember(auth.peerId, requestedName ?? auth.peer?.name ?? '');
|
|
2499
2533
|
if (config.logger) config.logger(`[group] join ${auth.peerId}`);
|
|
2500
2534
|
} else if (requestedName && requestedName !== existing.name) {
|
|
@@ -2676,6 +2710,97 @@ export function createPeerServer(overrides = {}) {
|
|
|
2676
2710
|
memberProgress.set(id, { lastSeq: Math.floor(seq), at: config.now() });
|
|
2677
2711
|
}
|
|
2678
2712
|
|
|
2713
|
+
/** 把对端补进本机成员表(自愈用):hub 写 group.json,成员写 group-key.json */
|
|
2714
|
+
/** 群密钥世代(hub 读 group.json 的 keyVersion,成员读 group-key.json 的 epoch) */
|
|
2715
|
+
function groupKeyVersion() {
|
|
2716
|
+
const ctx = localGroupContext();
|
|
2717
|
+
return Number(ctx?.epoch) || 0;
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
/**
|
|
2721
|
+
* 成员侧换钥:hub 轮换 groupKey(例如移除成员)后,老密钥的请求会变成 401 bad-signature。
|
|
2722
|
+
* 用 per-peer 签名重取 GET /v1/group/state(响应加密、含 groupKey/keyVersion),
|
|
2723
|
+
* 更新本地 group-key.json 后由调用方重试一次。
|
|
2724
|
+
*/
|
|
2725
|
+
let lastKeyRefreshAt = 0;
|
|
2726
|
+
|
|
2727
|
+
async function refreshGroupKeyFromHub() {
|
|
2728
|
+
if (groupRole !== 'member') return { ok: false, reason: 'not-member' };
|
|
2729
|
+
// 节流:同一秒内的连续坏签名不重复拉 state(脏请求打不爆)
|
|
2730
|
+
if (config.now() - lastKeyRefreshAt < 1_000) return { ok: false, reason: 'throttled' };
|
|
2731
|
+
lastKeyRefreshAt = config.now();
|
|
2732
|
+
const hub = groupHubPeer();
|
|
2733
|
+
if (!hub) return { ok: false, reason: 'hub-not-paired' };
|
|
2734
|
+
try {
|
|
2735
|
+
const state = await fetchGroupState({
|
|
2736
|
+
baseUrl: hub.baseUrl,
|
|
2737
|
+
secret: hub.secret,
|
|
2738
|
+
senderId: store.instanceId,
|
|
2739
|
+
timeoutMs: 5_000,
|
|
2740
|
+
});
|
|
2741
|
+
if (!state?.groupId || !state?.groupKey) return { ok: false, reason: 'group-state-incomplete' };
|
|
2742
|
+
const existing = readGroupKeyFile(config.stateDir) ?? { members: {} };
|
|
2743
|
+
const keyVersion = Number(state.keyVersion) || Number(state.epoch) || 0;
|
|
2744
|
+
const members = { ...(existing.members ?? {}) };
|
|
2745
|
+
if (!members[store.instanceId]) members[store.instanceId] = { name: config.name, baseUrl: '' };
|
|
2746
|
+
writeGroupKeyFile(config.stateDir, {
|
|
2747
|
+
...existing,
|
|
2748
|
+
groupId: String(state.groupId),
|
|
2749
|
+
key: String(state.groupKey),
|
|
2750
|
+
// epoch 只增不减:轮换后这里跟着 keyVersion 走,角色初始化再用 max() 兜一次
|
|
2751
|
+
epoch: Math.max(Number(existing.epoch) || 0, keyVersion),
|
|
2752
|
+
leaderPeerId: String(existing.leaderPeerId ?? ''),
|
|
2753
|
+
name: config.name,
|
|
2754
|
+
members,
|
|
2755
|
+
});
|
|
2756
|
+
replicaStore?.ensureHeader?.({ groupId: String(state.groupId) });
|
|
2757
|
+
if (config.logger) config.logger(`[group] key refreshed keyVersion=${keyVersion}`);
|
|
2758
|
+
return { ok: true, keyVersion };
|
|
2759
|
+
} catch (error) {
|
|
2760
|
+
if (config.logger) {
|
|
2761
|
+
config.logger(`[group] key refresh failed reason=${String(error?.message ?? error).slice(0, 60)}`);
|
|
2762
|
+
}
|
|
2763
|
+
return { ok: false, reason: 'refresh-failed' };
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
/**
|
|
2768
|
+
* 群级请求的统一入口:正常发一次;成员侧遇到 401 bad-signature 就换钥再发**一次**。
|
|
2769
|
+
* callFn 拿到的 ctx(groupKey/groupId)在重试时是重新读盘得到的,保证用的是新钥。
|
|
2770
|
+
*/
|
|
2771
|
+
async function withGroupKey(callFn) {
|
|
2772
|
+
const ctx = localGroupContext();
|
|
2773
|
+
if (!ctx?.groupKey) throw new Error('group-key-missing');
|
|
2774
|
+
try {
|
|
2775
|
+
return await callFn(ctx);
|
|
2776
|
+
} catch (error) {
|
|
2777
|
+
const code = String(error?.code ?? '');
|
|
2778
|
+
const status = Number(error?.status);
|
|
2779
|
+
const retryable = groupRole === 'member'
|
|
2780
|
+
&& ((status === 401 && code === 'bad-signature')
|
|
2781
|
+
|| (status === 400 && code === 'bad-encryption'));
|
|
2782
|
+
if (!retryable) throw error;
|
|
2783
|
+
const refreshed = await refreshGroupKeyFromHub();
|
|
2784
|
+
if (refreshed.ok !== true) throw error;
|
|
2785
|
+
const fresh = localGroupContext();
|
|
2786
|
+
if (!fresh?.groupKey) throw error;
|
|
2787
|
+
return callFn(fresh);
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
function upsertGroupMember(peerId) {
|
|
2792
|
+
const id = String(peerId ?? '');
|
|
2793
|
+
if (id === '') return;
|
|
2794
|
+
if (groupStore) {
|
|
2795
|
+
groupStore.addMember(id, '');
|
|
2796
|
+
return;
|
|
2797
|
+
}
|
|
2798
|
+
const existing = readGroupKeyFile(config.stateDir);
|
|
2799
|
+
if (!existing || existing.members?.[id]) return;
|
|
2800
|
+
const members = { ...existing.members, [id]: { name: '', baseUrl: '' } };
|
|
2801
|
+
writeGroupKeyFile(config.stateDir, { ...existing, members });
|
|
2802
|
+
}
|
|
2803
|
+
|
|
2679
2804
|
function groupMemberName(peerId) {
|
|
2680
2805
|
const ctx = localGroupContext();
|
|
2681
2806
|
return String(ctx?.members?.[peerId]?.name ?? '') || config.name;
|
|
@@ -2686,9 +2811,13 @@ export function createPeerServer(overrides = {}) {
|
|
|
2686
2811
|
* 1) X-Peer-Auth: group —— 群级签名(成员之间、成员→hub 的复制/补差);
|
|
2687
2812
|
* 2) 现有 per-peer 签名 —— CLI 排障用(仍要过成员门控,与 state/history 同口径)。
|
|
2688
2813
|
*/
|
|
2689
|
-
|
|
2814
|
+
/** 领导权协议:允许「成员表未同步」的对端自愈进来(其余路径仍严格按成员表 401) */
|
|
2815
|
+
const GROUP_SELF_HEAL_PATHS = new Set(['/v1/group/election', '/v1/group/leader']);
|
|
2816
|
+
|
|
2817
|
+
function authorizeGroupRequest(req, rawBody, url = null) {
|
|
2690
2818
|
if (headerValue(req, GROUP_AUTH_HEADER.toLowerCase()) === GROUP_AUTH_VALUE) {
|
|
2691
|
-
const
|
|
2819
|
+
const allowUnknownSender = Boolean(url && GROUP_SELF_HEAL_PATHS.has(url.pathname));
|
|
2820
|
+
const checked = verifyGroupAuth(req, rawBody, { allowUnknownSender });
|
|
2692
2821
|
if (!checked.ok) return checked;
|
|
2693
2822
|
return { ok: true, mode: 'group', peerId: checked.senderId, ctx: checked.ctx };
|
|
2694
2823
|
}
|
|
@@ -2849,6 +2978,12 @@ export function createPeerServer(overrides = {}) {
|
|
|
2849
2978
|
let roleTimer = null;
|
|
2850
2979
|
let heartbeatTimer = null;
|
|
2851
2980
|
let roleSeeded = false;
|
|
2981
|
+
/** 本进程启动时刻(受限启动竞选的「稳定期」起点) */
|
|
2982
|
+
const processStartedAtMs = config.now();
|
|
2983
|
+
/** 契约补充:每个进程生命周期最多尝试一次按优先级接管 */
|
|
2984
|
+
let takeoverAttempted = false;
|
|
2985
|
+
let takeoverAttempts = 0;
|
|
2986
|
+
const takeoverStableMs = heartbeatMs * 2;
|
|
2852
2987
|
/** 提交串行化:一次只让一条消息走 propose→commit,seq 才连续、pending 区才不撞号 */
|
|
2853
2988
|
let postChain = Promise.resolve();
|
|
2854
2989
|
|
|
@@ -2885,7 +3020,12 @@ export function createPeerServer(overrides = {}) {
|
|
|
2885
3020
|
groupId: String(ctx?.groupId ?? st.groupId ?? ''),
|
|
2886
3021
|
});
|
|
2887
3022
|
}
|
|
2888
|
-
if (st.groupId === '' && ctx?.groupId)
|
|
3023
|
+
if (st.groupId === '' && ctx?.groupId) {
|
|
3024
|
+
return roleStore.update({ groupId: String(ctx.groupId) });
|
|
3025
|
+
}
|
|
3026
|
+
// 密钥轮换(keyVersion 变大)后角色 epoch 至少跟上:epoch 只升不降,
|
|
3027
|
+
// 否则重启后会以更低的 epoch 参与裁决(契约 §4 里 epoch 低就是服从)。
|
|
3028
|
+
if (st.epoch < baseEpoch) return roleStore.update({ epoch: baseEpoch });
|
|
2889
3029
|
return st;
|
|
2890
3030
|
}
|
|
2891
3031
|
|
|
@@ -3152,10 +3292,12 @@ export function createPeerServer(overrides = {}) {
|
|
|
3152
3292
|
selfBaseUrl: localAdvertiseBaseUrl(),
|
|
3153
3293
|
timeoutMs: peerTimeoutMs,
|
|
3154
3294
|
};
|
|
3155
|
-
const results = await Promise.allSettled(targets.map((peerId) => sendGroupHeartbeat({
|
|
3295
|
+
const results = await Promise.allSettled(targets.map((peerId) => withGroupKey((keyCtx) => sendGroupHeartbeat({
|
|
3156
3296
|
...base,
|
|
3297
|
+
groupKey: keyCtx.groupKey,
|
|
3298
|
+
groupId: String(keyCtx.groupId),
|
|
3157
3299
|
baseUrl: baseUrlOf(peerId),
|
|
3158
|
-
})));
|
|
3300
|
+
}))));
|
|
3159
3301
|
let conflictWith = '';
|
|
3160
3302
|
let ahead = null;
|
|
3161
3303
|
for (const result of results) {
|
|
@@ -3206,10 +3348,12 @@ export function createPeerServer(overrides = {}) {
|
|
|
3206
3348
|
timeoutMs: peerTimeoutMs,
|
|
3207
3349
|
};
|
|
3208
3350
|
const targets = peerTargets();
|
|
3209
|
-
const results = await Promise.allSettled(targets.map((peerId) => commitGroupMessage({
|
|
3351
|
+
const results = await Promise.allSettled(targets.map((peerId) => withGroupKey((keyCtx) => commitGroupMessage({
|
|
3210
3352
|
...base,
|
|
3353
|
+
groupKey: keyCtx.groupKey,
|
|
3354
|
+
groupId: String(keyCtx.groupId),
|
|
3211
3355
|
baseUrl: baseUrlOf(peerId),
|
|
3212
|
-
})));
|
|
3356
|
+
}))));
|
|
3213
3357
|
return { ok: true, delivered: results.filter((entry) => entry.status === 'fulfilled').length, total: targets.length };
|
|
3214
3358
|
}
|
|
3215
3359
|
|
|
@@ -3267,17 +3411,17 @@ export function createPeerServer(overrides = {}) {
|
|
|
3267
3411
|
let unreachable = unreachableBase;
|
|
3268
3412
|
const tasks = targets.map(async (peerId) => {
|
|
3269
3413
|
try {
|
|
3270
|
-
const reply = await requestGroupVote({
|
|
3414
|
+
const reply = await withGroupKey((keyCtx) => requestGroupVote({
|
|
3271
3415
|
baseUrl: baseUrlOf(peerId),
|
|
3272
|
-
groupKey:
|
|
3273
|
-
groupId: String(
|
|
3416
|
+
groupKey: keyCtx.groupKey,
|
|
3417
|
+
groupId: String(keyCtx.groupId),
|
|
3274
3418
|
senderId: store.instanceId,
|
|
3275
3419
|
epoch,
|
|
3276
3420
|
candidate: store.instanceId,
|
|
3277
3421
|
priority: hubPriority,
|
|
3278
3422
|
selfBaseUrl: localAdvertiseBaseUrl(),
|
|
3279
3423
|
timeoutMs: peerTimeoutMs,
|
|
3280
|
-
});
|
|
3424
|
+
}));
|
|
3281
3425
|
if (Number(reply?.epoch) > epoch && String(reply?.leaderPeerId ?? '') !== '') ahead = reply;
|
|
3282
3426
|
if (typeof reply?.advertiseBaseUrl === 'string') rememberPeerBaseUrl(peerId, reply.advertiseBaseUrl);
|
|
3283
3427
|
return reply?.vote === true;
|
|
@@ -3336,11 +3480,61 @@ export function createPeerServer(overrides = {}) {
|
|
|
3336
3480
|
}
|
|
3337
3481
|
}
|
|
3338
3482
|
|
|
3483
|
+
/**
|
|
3484
|
+
* 受限的「启动后按优先级接管」(每进程最多一次)。
|
|
3485
|
+
*
|
|
3486
|
+
* 背景:契约 §4 是「epoch 高者胜,同 epoch 才比 priority」,所以优先级**不能跨 epoch** 夺权。
|
|
3487
|
+
* 真机事故形态:优先节点(NAS,hubPriority=0)重启期间备份节点抢到 epoch=4 当了 leader,
|
|
3488
|
+
* NAS 回来只能一直当 follower —— priority 永远等不到生效的机会。
|
|
3489
|
+
*
|
|
3490
|
+
* 因此给「配置上没有 group.hub 的本机」(即配置上的 hub)一次**有边界**的重试权:
|
|
3491
|
+
* 进程启动稳定 2×heartbeatMs、当前有一个租约有效的 leader、且本机优先级严格更高时,
|
|
3492
|
+
* 发起一次 epoch+1 的选举。赢则收回 leadership,输则本进程内永久放弃(不循环重试)。
|
|
3493
|
+
* 成员节点(配了 group.hub)不参与:运维的接线意图是「以 hub 为准」。
|
|
3494
|
+
*/
|
|
3495
|
+
async function maybePriorityTakeover() {
|
|
3496
|
+
if (!roleStore || takeoverAttempted) return { attempted: false, reason: 'already-done-or-off' };
|
|
3497
|
+
if (String(config.group.hub ?? '') !== '') return { attempted: false, reason: 'member-role' };
|
|
3498
|
+
const st = ensureRole();
|
|
3499
|
+
if (!st || st.role !== 'follower') return { attempted: false, reason: 'not-follower' };
|
|
3500
|
+
// 只在「跟得好好的」时候动手:没有 leader 或租约已过期,交给正常的 lease-expired 选举
|
|
3501
|
+
if (st.leaderPeerId === '' || config.now() > st.leaseUntil) {
|
|
3502
|
+
return { attempted: false, reason: 'no-stable-leader' };
|
|
3503
|
+
}
|
|
3504
|
+
if (config.now() - processStartedAtMs < takeoverStableMs) {
|
|
3505
|
+
return { attempted: false, reason: 'not-stable-yet' };
|
|
3506
|
+
}
|
|
3507
|
+
const leaderPriority = priorityOf(st.leaderPeerId);
|
|
3508
|
+
// 只有**严格**更高(数值更小)才值得动一次;相等一律不动(避免同优先级互抢)
|
|
3509
|
+
if (!(hubPriority < leaderPriority)) return { attempted: false, reason: 'not-higher-priority' };
|
|
3510
|
+
takeoverAttempted = true;
|
|
3511
|
+
takeoverAttempts += 1;
|
|
3512
|
+
const epoch = st.epoch + 1;
|
|
3513
|
+
if (config.logger) {
|
|
3514
|
+
config.logger(`[group] takeover attempt epoch=${epoch} priority=${hubPriority} leaderPriority=${leaderPriority}`);
|
|
3515
|
+
}
|
|
3516
|
+
const out = await runElection({ reason: 'priority-takeover', forcedEpoch: epoch });
|
|
3517
|
+
if (config.logger) {
|
|
3518
|
+
if (out?.ok === true) {
|
|
3519
|
+
config.logger(`[group] takeover won epoch=${out.epoch} votes=${out.votes}/${out.needed}`);
|
|
3520
|
+
} else {
|
|
3521
|
+
config.logger(
|
|
3522
|
+
`[group] takeover lost epoch=${out?.epoch ?? epoch} votes=${out?.votes ?? 1}/${out?.needed ?? quorumNeeded()}`
|
|
3523
|
+
+ ` reason=${out?.reason ?? 'unknown'}`,
|
|
3524
|
+
);
|
|
3525
|
+
}
|
|
3526
|
+
}
|
|
3527
|
+
return { attempted: true, result: out };
|
|
3528
|
+
}
|
|
3339
3529
|
/** 租约检查:非 leader 且租约过期 → 进入候选并发起选举(抖动在 runElection 里) */
|
|
3340
3530
|
async function roleTick() {
|
|
3341
3531
|
const st = ensureRole();
|
|
3342
3532
|
if (!st || st.role === 'leader') return { ok: false, reason: 'leader-or-off' };
|
|
3343
3533
|
if (electionInFlight) return { ok: false, reason: 'election-in-flight' };
|
|
3534
|
+
// ① 受限的启动竞选(每进程最多一次;不满足条件时什么也不做)
|
|
3535
|
+
const takeover = await maybePriorityTakeover();
|
|
3536
|
+
if (takeover.attempted) return takeover.result ?? { ok: false, reason: 'takeover-attempted' };
|
|
3537
|
+
// ② 租约过期 → 正常选举
|
|
3344
3538
|
if (config.now() <= st.leaseUntil) return { ok: false, reason: 'lease-valid' };
|
|
3345
3539
|
if (peerTargets().length === 0) return { ok: false, reason: 'no-peers-with-address' };
|
|
3346
3540
|
return runElection({ reason: 'lease-expired' });
|
|
@@ -3437,6 +3631,8 @@ export function createPeerServer(overrides = {}) {
|
|
|
3437
3631
|
// 分母与门槛:成员表全集(含没有地址的成员),地址只影响投递
|
|
3438
3632
|
memberCount: quorumMembers().length,
|
|
3439
3633
|
quorumNeeded: quorumNeeded(),
|
|
3634
|
+
// 群密钥世代(只给世代,不给密钥):轮换后可从这里看出来
|
|
3635
|
+
keyVersion: groupKeyVersion(),
|
|
3440
3636
|
degraded: isDegraded(),
|
|
3441
3637
|
advertiseBaseUrl: localAdvertiseBaseUrl(),
|
|
3442
3638
|
},
|
|
@@ -3694,7 +3890,12 @@ export function createPeerServer(overrides = {}) {
|
|
|
3694
3890
|
};
|
|
3695
3891
|
const tasks = targets.map(async (peerId) => {
|
|
3696
3892
|
try {
|
|
3697
|
-
const reply = await
|
|
3893
|
+
const reply = await withGroupKey((keyCtx) => proposeGroupMessage({
|
|
3894
|
+
...base,
|
|
3895
|
+
groupKey: keyCtx.groupKey,
|
|
3896
|
+
groupId: String(keyCtx.groupId),
|
|
3897
|
+
baseUrl: baseUrlOf(peerId),
|
|
3898
|
+
}));
|
|
3698
3899
|
return reply?.ok === true;
|
|
3699
3900
|
} catch {
|
|
3700
3901
|
return false;
|
|
@@ -3737,7 +3938,13 @@ export function createPeerServer(overrides = {}) {
|
|
|
3737
3938
|
};
|
|
3738
3939
|
const handler = handlers[`${method} ${url.pathname}`];
|
|
3739
3940
|
if (!handler) return { status: 405, body: { error: 'method-not-allowed' } };
|
|
3740
|
-
|
|
3941
|
+
let auth = authorizeGroupRequest(req, rawBody, url);
|
|
3942
|
+
if (!auth.ok && groupRole === 'member' && String(auth.error) === 'bad-signature') {
|
|
3943
|
+
// hub 轮换密钥后,成员手里的旧钥会让**入站**请求也变成 bad-signature;
|
|
3944
|
+
// 这里先按 per-peer 换钥,再验一次(只重试一次),避免「谁也没法先动」的死锁。
|
|
3945
|
+
const refreshed = await refreshGroupKeyFromHub();
|
|
3946
|
+
if (refreshed.ok === true) auth = authorizeGroupRequest(req, rawBody, url);
|
|
3947
|
+
}
|
|
3741
3948
|
if (!auth.ok) return { status: auth.status, body: { error: auth.error }, peerId: null };
|
|
3742
3949
|
return handler(req, rawBody, url, auth);
|
|
3743
3950
|
}
|
|
@@ -3894,10 +4101,15 @@ export function createPeerServer(overrides = {}) {
|
|
|
3894
4101
|
return { peerId: String(hub.peerId ?? ''), name: String(hub.name ?? ''), baseUrl, secret: String(hub.secret) };
|
|
3895
4102
|
}
|
|
3896
4103
|
|
|
4104
|
+
/**
|
|
4105
|
+
* 刷新成员侧成员表。**只增不减**:一份过早/不完整的快照(成员在 bootstrap 之后才加入)
|
|
4106
|
+
* 不该把仍在群的成员从本机表里抹掉 —— 真机上这会让该成员的心跳/投票被 401 not-a-member 挡在门外。
|
|
4107
|
+
* 移除成员的权威在 hub 侧 group.json(本机表里多出来的条目只会让多数派更保守)。
|
|
4108
|
+
*/
|
|
3897
4109
|
function saveGroupMembers(members, names = {}) {
|
|
3898
4110
|
const existing = readGroupKeyFile(config.stateDir);
|
|
3899
4111
|
if (!existing) return;
|
|
3900
|
-
const next = {};
|
|
4112
|
+
const next = { ...existing.members };
|
|
3901
4113
|
for (const peerId of members) {
|
|
3902
4114
|
const id = String(peerId ?? '');
|
|
3903
4115
|
if (id === '') continue;
|
|
@@ -3988,10 +4200,10 @@ export function createPeerServer(overrides = {}) {
|
|
|
3988
4200
|
}
|
|
3989
4201
|
if (!ctx?.groupKey) return { ok: false, reason: 'no-group-key' };
|
|
3990
4202
|
const sinceSeq = replicaStore.state().lastSeq;
|
|
3991
|
-
const page = await pullGroupMessages({
|
|
4203
|
+
const page = await withGroupKey((keyCtx) => pullGroupMessages({
|
|
3992
4204
|
baseUrl: hub.baseUrl,
|
|
3993
|
-
groupKey:
|
|
3994
|
-
groupId:
|
|
4205
|
+
groupKey: keyCtx.groupKey,
|
|
4206
|
+
groupId: keyCtx.groupId,
|
|
3995
4207
|
senderId: store.instanceId,
|
|
3996
4208
|
sinceSeq,
|
|
3997
4209
|
limit: GROUP_HISTORY_MAX_LIMIT,
|
|
@@ -3999,7 +4211,7 @@ export function createPeerServer(overrides = {}) {
|
|
|
3999
4211
|
// 自报复制进度:hub 的面板据此显示「落后 leader 多少条」
|
|
4000
4212
|
lastSeq: sinceSeq,
|
|
4001
4213
|
timeoutMs: Math.max(1000, replicaPullMs),
|
|
4002
|
-
});
|
|
4214
|
+
}));
|
|
4003
4215
|
if (String(page?.groupId ?? '') !== String(ctx.groupId)) {
|
|
4004
4216
|
throw new Error('group-mismatch');
|
|
4005
4217
|
}
|
|
@@ -4091,14 +4303,38 @@ export function createPeerServer(overrides = {}) {
|
|
|
4091
4303
|
if (groupAuthRequest) {
|
|
4092
4304
|
const groupCtx = localGroupContext();
|
|
4093
4305
|
if (groupCtx?.groupKey) {
|
|
4094
|
-
|
|
4306
|
+
let bodyKey = deriveGroupKeys(groupCtx.groupKey).bodyKey;
|
|
4095
4307
|
groupResponseKey = bodyKey;
|
|
4096
4308
|
const encrypted = String(req.headers[ENC_HEADER] ?? '') === ENC_VERSION;
|
|
4097
4309
|
if (encrypted && rawBody.length > 0) {
|
|
4310
|
+
let decrypted = false;
|
|
4098
4311
|
try {
|
|
4099
4312
|
rawBody = decryptGroupBody(bodyKey, rawBody.toString('utf8'));
|
|
4313
|
+
decrypted = true;
|
|
4100
4314
|
} catch {
|
|
4315
|
+
decrypted = false;
|
|
4316
|
+
}
|
|
4317
|
+
if (!decrypted) {
|
|
4318
|
+
// hub 轮换了群密钥:成员侧先按 per-peer 换钥再解一次(只一次)
|
|
4319
|
+
const healed = groupRole === 'member' ? await refreshGroupKeyFromHub() : { ok: false };
|
|
4320
|
+
if (healed.ok === true) {
|
|
4321
|
+
const freshCtx = localGroupContext();
|
|
4322
|
+
if (freshCtx?.groupKey) {
|
|
4323
|
+
bodyKey = deriveGroupKeys(freshCtx.groupKey).bodyKey;
|
|
4324
|
+
groupResponseKey = bodyKey;
|
|
4325
|
+
try {
|
|
4326
|
+
rawBody = decryptGroupBody(bodyKey, rawBody.toString('utf8'));
|
|
4327
|
+
decrypted = true;
|
|
4328
|
+
} catch {
|
|
4329
|
+
decrypted = false;
|
|
4330
|
+
}
|
|
4331
|
+
}
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
if (!decrypted) {
|
|
4335
|
+
// 对端拿的可能仍是旧钥:错误体走**明文**,否则它连信封都解不开
|
|
4101
4336
|
result = { status: 400, body: { error: 'bad-encryption' } };
|
|
4337
|
+
groupResponseKey = null;
|
|
4102
4338
|
}
|
|
4103
4339
|
}
|
|
4104
4340
|
}
|
|
@@ -4138,7 +4374,9 @@ export function createPeerServer(overrides = {}) {
|
|
|
4138
4374
|
|
|
4139
4375
|
if (!res.headersSent) {
|
|
4140
4376
|
try {
|
|
4141
|
-
|
|
4377
|
+
// 错误体一律明文:群级请求失败时对端可能拿着旧钥(密钥刚轮换),加密过去它连错误码都读不到,
|
|
4378
|
+
// 也就无法据此换钥重试。错误体只有 {error, epoch, ...} 之类的码,不含任何密钥。
|
|
4379
|
+
const body = (groupResponseKey && result.status < 400)
|
|
4142
4380
|
? {
|
|
4143
4381
|
enc: ENC_VERSION,
|
|
4144
4382
|
data: encryptGroupBody(groupResponseKey, Buffer.from(JSON.stringify(result.body), 'utf8')),
|
|
@@ -4151,10 +4389,13 @@ export function createPeerServer(overrides = {}) {
|
|
|
4151
4389
|
}
|
|
4152
4390
|
if (config.logger) {
|
|
4153
4391
|
const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1e6;
|
|
4154
|
-
// 只记录方法、路径、状态、耗时、peerId
|
|
4392
|
+
// 只记录方法、路径、状态、耗时、peerId、错误码(4xx/5xx 时才带,便于排障);
|
|
4393
|
+
// 绝不记录 header / secret / 签名原文
|
|
4394
|
+
const errorCode = result.status >= 400 ? String(result.body?.error ?? '') : '';
|
|
4155
4395
|
config.logger(
|
|
4156
4396
|
`${nowIso(Date.now())} ${String(req.method).toUpperCase()} ${url.pathname} `
|
|
4157
|
-
+ `${result.status} ${elapsedMs.toFixed(1)}ms peer=${result.peerId ?? '-'}
|
|
4397
|
+
+ `${result.status} ${elapsedMs.toFixed(1)}ms peer=${result.peerId ?? '-'}`
|
|
4398
|
+
+ `${errorCode === '' ? '' : ` err=${errorCode}`}`,
|
|
4158
4399
|
);
|
|
4159
4400
|
}
|
|
4160
4401
|
});
|
|
@@ -4178,6 +4419,7 @@ export function createPeerServer(overrides = {}) {
|
|
|
4178
4419
|
groupId: st.groupId,
|
|
4179
4420
|
degraded: isDegraded(),
|
|
4180
4421
|
quorum: quorumMode,
|
|
4422
|
+
keyVersion: groupKeyVersion(),
|
|
4181
4423
|
};
|
|
4182
4424
|
},
|
|
4183
4425
|
/** 契约 §7:POST /api/lan-peers/group/leader {action:claim} 的落点 */
|
|
@@ -4202,6 +4444,15 @@ export function createPeerServer(overrides = {}) {
|
|
|
4202
4444
|
heartbeat: () => broadcastHeartbeat(),
|
|
4203
4445
|
quorumMode: () => quorumMode,
|
|
4204
4446
|
pendingCount: () => pendingCount(),
|
|
4447
|
+
/** 受限启动竞选的状态:attempted 只在本进程尝试过后为 true,attempts 恒 ≤ 1 */
|
|
4448
|
+
takeover: () => ({
|
|
4449
|
+
attempted: takeoverAttempted,
|
|
4450
|
+
attempts: takeoverAttempts,
|
|
4451
|
+
stableMs: takeoverStableMs,
|
|
4452
|
+
sinceStartMs: config.now() - processStartedAtMs,
|
|
4453
|
+
priority: hubPriority,
|
|
4454
|
+
leaderPriority: roleState()?.leaderPeerId ? priorityOf(roleState().leaderPeerId) : null,
|
|
4455
|
+
}),
|
|
4205
4456
|
degraded: () => isDegraded(),
|
|
4206
4457
|
roleFile: roleStore ? roleStore.rolePath : null,
|
|
4207
4458
|
};
|
|
@@ -4226,7 +4477,18 @@ export function createPeerServer(overrides = {}) {
|
|
|
4226
4477
|
state: (options = {}) => groupStore.state(options),
|
|
4227
4478
|
members: () => groupMembersView(),
|
|
4228
4479
|
addMember: (peerId, name) => groupStore.addMember(peerId, name),
|
|
4229
|
-
removeMember: (peerId) =>
|
|
4480
|
+
removeMember: (peerId) => {
|
|
4481
|
+
const removed = groupStore.removeMember(peerId);
|
|
4482
|
+
if (!removed) return removed;
|
|
4483
|
+
const id = String(peerId ?? '');
|
|
4484
|
+
memberProgress.delete(id);
|
|
4485
|
+
peerBaseUrls.delete(id);
|
|
4486
|
+
peerPriorities.delete(id);
|
|
4487
|
+
if (config.logger) {
|
|
4488
|
+
config.logger(`[group] key rotated keyVersion=${groupStore.state().keyVersion} reason=member-removed`);
|
|
4489
|
+
}
|
|
4490
|
+
return removed;
|
|
4491
|
+
},
|
|
4230
4492
|
rename: (nextName) => groupStore.renameGroup(nextName),
|
|
4231
4493
|
post: (input) => groupStore.append(input),
|
|
4232
4494
|
// 与 /v1/group/history 同一个渲染口径:同名成员在这里就已经带上短 id 后缀
|
|
@@ -4350,7 +4612,10 @@ Options:
|
|
|
4350
4612
|
--group-auto-join 允许已配对成员自助加入(默认关闭)
|
|
4351
4613
|
--group-retention-days <n> 历史保留天数(默认 7,0 = 读到时立即清旧)
|
|
4352
4614
|
--group-advertise-base-url <url> 本机对外可达地址(默认自动探测私有 IPv4)
|
|
4353
|
-
--group-quorum <mode> majority|none(S3
|
|
4615
|
+
--group-quorum <mode> majority|none(S3:majority = propose/commit 两阶段)
|
|
4616
|
+
--group-heartbeat-ms <ms> leader 心跳间隔(默认 3000;租约 = 3×)
|
|
4617
|
+
--group-commit-timeout-ms <ms> propose→commit 超时(默认 3000)
|
|
4618
|
+
--group-hub-priority <n> 同 epoch 裁决优先级(数值小者优先,默认 0)
|
|
4354
4619
|
--group-replica-pull-ms <ms> 成员向 leader 补差间隔(默认 10000)
|
|
4355
4620
|
--code 只生成并打印一次性配对码,然后退出
|
|
4356
4621
|
-h, --help 显示帮助
|
|
@@ -4380,6 +4645,9 @@ async function main(argv) {
|
|
|
4380
4645
|
'group-retention-days': { type: 'string' },
|
|
4381
4646
|
'group-advertise-base-url': { type: 'string' },
|
|
4382
4647
|
'group-quorum': { type: 'string' },
|
|
4648
|
+
'group-heartbeat-ms': { type: 'string' },
|
|
4649
|
+
'group-commit-timeout-ms': { type: 'string' },
|
|
4650
|
+
'group-hub-priority': { type: 'string' },
|
|
4383
4651
|
'group-replica-pull-ms': { type: 'string' },
|
|
4384
4652
|
code: { type: 'boolean', default: false },
|
|
4385
4653
|
help: { type: 'boolean', short: 'h', default: false },
|
|
@@ -4407,6 +4675,8 @@ async function main(argv) {
|
|
|
4407
4675
|
group: (values.group || values['group-hub'] != null || values['group-name'] != null
|
|
4408
4676
|
|| values['group-auto-join'] || values['group-retention-days'] != null
|
|
4409
4677
|
|| values['group-advertise-base-url'] != null || values['group-quorum'] != null
|
|
4678
|
+
|| values['group-heartbeat-ms'] != null || values['group-commit-timeout-ms'] != null
|
|
4679
|
+
|| values['group-hub-priority'] != null
|
|
4410
4680
|
|| values['group-replica-pull-ms'] != null)
|
|
4411
4681
|
? {
|
|
4412
4682
|
enabled: values.group === true,
|
|
@@ -4418,6 +4688,15 @@ async function main(argv) {
|
|
|
4418
4688
|
: Number(values['group-retention-days']),
|
|
4419
4689
|
advertiseBaseUrl: values['group-advertise-base-url'],
|
|
4420
4690
|
quorum: values['group-quorum'],
|
|
4691
|
+
heartbeatMs: values['group-heartbeat-ms'] == null
|
|
4692
|
+
? undefined
|
|
4693
|
+
: Number(values['group-heartbeat-ms']),
|
|
4694
|
+
commitTimeoutMs: values['group-commit-timeout-ms'] == null
|
|
4695
|
+
? undefined
|
|
4696
|
+
: Number(values['group-commit-timeout-ms']),
|
|
4697
|
+
hubPriority: values['group-hub-priority'] == null
|
|
4698
|
+
? undefined
|
|
4699
|
+
: Number(values['group-hub-priority']),
|
|
4421
4700
|
replicaPullMs: values['group-replica-pull-ms'] == null
|
|
4422
4701
|
? undefined
|
|
4423
4702
|
: Number(values['group-replica-pull-ms']),
|
|
@@ -89,6 +89,12 @@ export async function startCluster(t, options = {}) {
|
|
|
89
89
|
commitTimeoutMs: commitOverride = commitTimeoutMs,
|
|
90
90
|
tickMs: tickOverride = tickMs,
|
|
91
91
|
jitterMs = 0,
|
|
92
|
+
// S3 补充:哪些下标是「hub 角色」(配置上没有 group.hub,可参与启动接管)。默认只有 0。
|
|
93
|
+
hubRoles = [0],
|
|
94
|
+
// 预置角色文件:{ 下标: leader 下标 } → 该节点以 follower 起步(leader 指向另一台)
|
|
95
|
+
followerOf = {},
|
|
96
|
+
// 假时钟(「推进 30s 验证不再重试」这类断言用)
|
|
97
|
+
now: nowOverride = null,
|
|
92
98
|
} = options;
|
|
93
99
|
|
|
94
100
|
const ids = customIds ?? Array.from({ length: count }, (_, index) => `peer-s3-${String.fromCharCode(97 + index)}`);
|
|
@@ -106,9 +112,46 @@ export async function startCluster(t, options = {}) {
|
|
|
106
112
|
|
|
107
113
|
const nodes = [];
|
|
108
114
|
const urls = [];
|
|
115
|
+
const hubRoleIndexes = new Set(hubRoles);
|
|
116
|
+
hubRoleIndexes.add(0);
|
|
117
|
+
/** node0 的群快照:后面的 hub 角色节点要复用同一 groupId/groupKey */
|
|
118
|
+
let hubSnapshot = null;
|
|
109
119
|
for (let index = 0; index < count; index += 1) {
|
|
110
120
|
// 先写 peers.json 再 listen:成员启动那一轮就能解析到 hub
|
|
111
121
|
writePeersJson(dirs[index], ids, names, urls, index);
|
|
122
|
+
const isHubRole = hubRoleIndexes.has(index);
|
|
123
|
+
if (index > 0 && isHubRole && hubSnapshot) {
|
|
124
|
+
// 第二个 hub 角色节点:先把 group.json 写成与 node0 同一把群密钥/groupId,
|
|
125
|
+
// 否则 GroupStore 首次 load 会自己生成一份新的(那就不是一个群了)
|
|
126
|
+
const members = {};
|
|
127
|
+
for (let other = 0; other < index; other += 1) {
|
|
128
|
+
members[ids[other]] = { name: names[other], baseUrl: urls[other], addedAt: new Date().toISOString() };
|
|
129
|
+
}
|
|
130
|
+
members[ids[index]] = { name: names[index], addedAt: new Date().toISOString() };
|
|
131
|
+
writeJson0600(path.join(dirs[index], 'group.json'), {
|
|
132
|
+
groupId: hubSnapshot.groupId,
|
|
133
|
+
key: hubSnapshot.groupKey,
|
|
134
|
+
keyVersion: hubSnapshot.epoch,
|
|
135
|
+
name: hubSnapshot.name,
|
|
136
|
+
createdAt: new Date().toISOString(),
|
|
137
|
+
members,
|
|
138
|
+
messages: [],
|
|
139
|
+
pending: [],
|
|
140
|
+
lastSeq: 0,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
if (index in followerOf) {
|
|
144
|
+
// 预置角色文件:以 follower 起步(leader 指向 followerOf[index] 那台)
|
|
145
|
+
writeJson0600(path.join(dirs[index], 'group-role.json'), {
|
|
146
|
+
role: 'follower',
|
|
147
|
+
epoch: hubSnapshot ? hubSnapshot.epoch : 1,
|
|
148
|
+
leaderPeerId: ids[followerOf[index]],
|
|
149
|
+
leaseUntil: Date.now() + 60_000,
|
|
150
|
+
commitSeq: 0,
|
|
151
|
+
votedFor: null,
|
|
152
|
+
groupId: hubSnapshot ? hubSnapshot.groupId : '',
|
|
153
|
+
});
|
|
154
|
+
}
|
|
112
155
|
const group = {
|
|
113
156
|
enabled: true,
|
|
114
157
|
name: 's3',
|
|
@@ -119,11 +162,12 @@ export async function startCluster(t, options = {}) {
|
|
|
119
162
|
commitTimeoutMs: commitOverride,
|
|
120
163
|
hubPriority: priorities[index] ?? 0,
|
|
121
164
|
};
|
|
122
|
-
if (
|
|
165
|
+
if (!isHubRole) group.hub = names[0];
|
|
123
166
|
const server = createPeerServer({
|
|
124
167
|
listenHost: '127.0.0.1',
|
|
125
168
|
stateDir: dirs[index],
|
|
126
169
|
requireEncryption,
|
|
170
|
+
...(nowOverride ? { now: nowOverride } : {}),
|
|
127
171
|
allowRemotePrompts: true,
|
|
128
172
|
logger,
|
|
129
173
|
group,
|
|
@@ -140,6 +184,11 @@ export async function startCluster(t, options = {}) {
|
|
|
140
184
|
const info = await server.listen(0, '127.0.0.1');
|
|
141
185
|
server.config.group.advertiseBaseUrl = info.url;
|
|
142
186
|
urls[index] = info.url;
|
|
187
|
+
if (isHubRole) {
|
|
188
|
+
const self = server.group.state().members[ids[index]];
|
|
189
|
+
if (self) self.baseUrl = info.url;
|
|
190
|
+
if (!hubSnapshot) hubSnapshot = server.group.state({ includeKey: true });
|
|
191
|
+
}
|
|
143
192
|
nodes.push({
|
|
144
193
|
index,
|
|
145
194
|
peerId: ids[index],
|
|
@@ -160,6 +209,16 @@ export async function startCluster(t, options = {}) {
|
|
|
160
209
|
hub.server.group.state().members[ids[index]].baseUrl = urls[index];
|
|
161
210
|
}
|
|
162
211
|
|
|
212
|
+
// 第二个及以后的 hub 角色节点:成员表补齐(自己那条地址在 listen 后已写)
|
|
213
|
+
for (const node of nodes.slice(1)) {
|
|
214
|
+
if (!node.server.group.isHub) continue;
|
|
215
|
+
for (let index = 0; index < count; index += 1) {
|
|
216
|
+
if (index === node.index) continue;
|
|
217
|
+
node.server.group.addMember(ids[index], names[index]);
|
|
218
|
+
node.server.group.state().members[ids[index]].baseUrl = urls[index];
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
163
222
|
const snapshot = hub.server.group.state({ includeKey: true });
|
|
164
223
|
for (let index = 1; index < count; index += 1) {
|
|
165
224
|
const members = {};
|
|
@@ -11,8 +11,11 @@ import fs from 'node:fs';
|
|
|
11
11
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
12
12
|
import {
|
|
13
13
|
fetchGroupLease,
|
|
14
|
+
pullGroupMessages,
|
|
15
|
+
readGroupKeyFile,
|
|
14
16
|
requestGroupVote,
|
|
15
17
|
sendGroupHeartbeat,
|
|
18
|
+
writeGroupKeyFile,
|
|
16
19
|
} from '../../lib/group-client.mjs';
|
|
17
20
|
import { fileMode, makeTempStateDir } from './helpers.mjs';
|
|
18
21
|
import { postAs, readRoleFile, roleOf, startCluster, waitFor } from './group-cluster.mjs';
|
|
@@ -362,7 +365,26 @@ test('S3 quorum 分母 = 成员表全集:没有任何地址的成员也计入
|
|
|
362
365
|
for (const id of ['peer-ghost-1', 'peer-ghost-2', 'peer-ghost-3']) a.server.group.removeMember(id);
|
|
363
366
|
const ok = await postAs(a.baseUrl, b.peerId, { kind: 'chat', text: 'ghost-gone' });
|
|
364
367
|
assert.equal(ok.status, 201, ok.text);
|
|
365
|
-
|
|
368
|
+
// 移除成员会轮换群密钥(keyVersion+1):旧钥必须被拒,新钥照常
|
|
369
|
+
const rotated = a.server.group.state({ includeKey: true });
|
|
370
|
+
assert.ok(rotated.keyVersion > 1, 'removeMember 必须轮换 keyVersion');
|
|
371
|
+
assert.notEqual(rotated.groupKey, cluster.groupKey, '轮换后 groupKey 必须变化');
|
|
372
|
+
await assert.rejects(
|
|
373
|
+
() => fetchGroupLease({ baseUrl: a.baseUrl, groupKey: cluster.groupKey, groupId: cluster.groupId, senderId: b.peerId }),
|
|
374
|
+
(error) => {
|
|
375
|
+
assert.equal(error.status, 401);
|
|
376
|
+
assert.equal(error.code, 'bad-signature');
|
|
377
|
+
return true;
|
|
378
|
+
},
|
|
379
|
+
);
|
|
380
|
+
const after = await fetchGroupLease({
|
|
381
|
+
baseUrl: a.baseUrl,
|
|
382
|
+
groupKey: rotated.groupKey,
|
|
383
|
+
groupId: cluster.groupId,
|
|
384
|
+
senderId: b.peerId,
|
|
385
|
+
});
|
|
386
|
+
assert.equal(after.quorumNeeded, 2);
|
|
387
|
+
assert.equal(after.keyVersion, rotated.keyVersion, 'lease 必须暴露 keyVersion');
|
|
366
388
|
});
|
|
367
389
|
|
|
368
390
|
test('S3 单节点自立为 leader 必败:N=2 且另一台不可达 → quorum-unavailable', async (t) => {
|
|
@@ -399,6 +421,47 @@ test('S3 N=3:两台可达能当选(2 ≥ 2),只剩一台不能(1 < 2
|
|
|
399
421
|
assert.equal(lonely.needed, 2);
|
|
400
422
|
assert.notEqual(roleOf(c).role, 'leader');
|
|
401
423
|
});
|
|
424
|
+
test('S3 成员表未同步时:选举/心跳自愈入表(真机 not-a-member 的回归)', async (t) => {
|
|
425
|
+
const cluster = await startCluster(t, { count: 3, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
426
|
+
const [a, b, c] = cluster.nodes;
|
|
427
|
+
// 造「c 的成员表来自一份不完整的快照」:把 b 从 c 的 group-key.json 里删掉
|
|
428
|
+
const keyFile = readGroupKeyFile(c.dir);
|
|
429
|
+
delete keyFile.members[b.peerId];
|
|
430
|
+
writeGroupKeyFile(c.dir, keyFile);
|
|
431
|
+
const stale = readGroupKeyFile(c.dir);
|
|
432
|
+
assert.equal(Boolean(stale.members[b.peerId]), false, '前提:c 不认识 b');
|
|
433
|
+
|
|
434
|
+
// b 拉票:原来会 401 not-a-member(对端永远投不进来 → 群选不出 leader)
|
|
435
|
+
const vote = await requestGroupVote({
|
|
436
|
+
baseUrl: c.baseUrl,
|
|
437
|
+
...groupArgs(cluster, b.peerId),
|
|
438
|
+
epoch: 3,
|
|
439
|
+
candidate: b.peerId,
|
|
440
|
+
});
|
|
441
|
+
assert.equal(vote.vote, true, JSON.stringify(vote));
|
|
442
|
+
assert.equal(vote.epoch, 3);
|
|
443
|
+
// 自愈:c 把 b 补回成员表(并落盘)
|
|
444
|
+
const healed = readGroupKeyFile(c.dir);
|
|
445
|
+
assert.ok(healed.members[b.peerId], 'c 必须把 b 补回成员表并落盘');
|
|
446
|
+
// 第二次直接投,不再走自愈分支
|
|
447
|
+
const again = await requestGroupVote({
|
|
448
|
+
baseUrl: c.baseUrl,
|
|
449
|
+
...groupArgs(cluster, b.peerId),
|
|
450
|
+
epoch: 4,
|
|
451
|
+
candidate: b.peerId,
|
|
452
|
+
});
|
|
453
|
+
assert.equal(again.vote, true);
|
|
454
|
+
|
|
455
|
+
// 非领导权路径仍然严格:成员表没有的发送方打 pull 仍 401 not-a-member(不自愈)
|
|
456
|
+
await assert.rejects(
|
|
457
|
+
() => pullGroupMessages({ baseUrl: c.baseUrl, ...groupArgs(cluster, 'peer-not-in-group') }),
|
|
458
|
+
(error) => {
|
|
459
|
+
assert.equal(error.status, 401);
|
|
460
|
+
assert.equal(error.code, 'not-a-member');
|
|
461
|
+
return true;
|
|
462
|
+
},
|
|
463
|
+
);
|
|
464
|
+
});
|
|
402
465
|
// 说明:post 走 per-peer 签名,这里顺带确认 leader 真的能收(详情见 group-commit.test.mjs)
|
|
403
466
|
test('S3 leader 就绪:hub leader 的 post 通道可用(202/201 由 commit 测试覆盖)', async (t) => {
|
|
404
467
|
const cluster = await startCluster(t, { count: 2, quorum: 'none' });
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.7.3 安全硬化 · 移除成员即轮换群密钥 + 成员侧换钥自愈(契约补充)。
|
|
3
|
+
*
|
|
4
|
+
* 两个洞的回归:
|
|
5
|
+
* ① hub 收到「不在成员表但握着有效 groupKey」的心跳时必须拒绝,且**不得**把它写回成员表
|
|
6
|
+
* ② removeMember 必须轮换 keyVersion;被移除者的旧钥一律 401 bad-signature
|
|
7
|
+
* 另外两条保护:③成员侧换钥后自动重试一次 ④成员侧(副本)自愈仍然有效
|
|
8
|
+
*/
|
|
9
|
+
import test from 'node:test';
|
|
10
|
+
import assert from 'node:assert/strict';
|
|
11
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
12
|
+
import {
|
|
13
|
+
fetchGroupLease,
|
|
14
|
+
readGroupKeyFile,
|
|
15
|
+
requestGroupVote,
|
|
16
|
+
sendGroupHeartbeat,
|
|
17
|
+
writeGroupKeyFile,
|
|
18
|
+
} from '../../lib/group-client.mjs';
|
|
19
|
+
import { postAs, roleOf, startCluster, waitFor } from './group-cluster.mjs';
|
|
20
|
+
|
|
21
|
+
const groupArgs = (cluster, senderId) => ({
|
|
22
|
+
groupKey: cluster.groupKey,
|
|
23
|
+
groupId: cluster.groupId,
|
|
24
|
+
senderId,
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('① hub 永不自愈:持有有效 groupKey 但不在成员表 → 401 not-a-member,且不写回成员表', async (t) => {
|
|
28
|
+
const cluster = await startCluster(t, { count: 2, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
29
|
+
const [hub] = cluster.nodes;
|
|
30
|
+
const before = Object.keys(hub.server.group.state().members).sort();
|
|
31
|
+
const outsider = 'peer-not-invited-9f3a';
|
|
32
|
+
|
|
33
|
+
const heartbeat = await assert.rejects(
|
|
34
|
+
() => sendGroupHeartbeat({
|
|
35
|
+
baseUrl: hub.baseUrl,
|
|
36
|
+
...groupArgs(cluster, outsider),
|
|
37
|
+
epoch: 1,
|
|
38
|
+
leaderPeerId: outsider,
|
|
39
|
+
}),
|
|
40
|
+
(error) => {
|
|
41
|
+
assert.equal(error.status, 401);
|
|
42
|
+
assert.equal(error.code, 'not-a-member');
|
|
43
|
+
return true;
|
|
44
|
+
},
|
|
45
|
+
);
|
|
46
|
+
assert.equal(heartbeat, undefined);
|
|
47
|
+
// 投票端点同样是领导权协议,但 hub 侧也不得自愈
|
|
48
|
+
await assert.rejects(
|
|
49
|
+
() => requestGroupVote({
|
|
50
|
+
baseUrl: hub.baseUrl,
|
|
51
|
+
...groupArgs(cluster, outsider),
|
|
52
|
+
epoch: 2,
|
|
53
|
+
candidate: outsider,
|
|
54
|
+
}),
|
|
55
|
+
(error) => {
|
|
56
|
+
assert.equal(error.status, 401);
|
|
57
|
+
assert.equal(error.code, 'not-a-member');
|
|
58
|
+
return true;
|
|
59
|
+
},
|
|
60
|
+
);
|
|
61
|
+
assert.deepEqual(Object.keys(hub.server.group.state().members).sort(), before, 'group.json 的成员表不得被自愈写入');
|
|
62
|
+
assert.equal(readGroupKeyFile(hub.dir), null, 'hub 不写 group-key.json');
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test('② 移除成员即轮换密钥:keyVersion+1,被移除者用旧钥打 election/leader → 401 bad-signature', async (t) => {
|
|
66
|
+
const lines = [];
|
|
67
|
+
const cluster = await startCluster(t, { count: 3, heartbeatMs: 5_000, tickMs: 60_000, logger: (line) => lines.push(String(line)) });
|
|
68
|
+
const [hub, , removed] = cluster.nodes;
|
|
69
|
+
const before = hub.server.group.state({ includeKey: true });
|
|
70
|
+
assert.equal(before.keyVersion, 1);
|
|
71
|
+
|
|
72
|
+
assert.equal(hub.server.group.removeMember(removed.peerId), true);
|
|
73
|
+
const after = hub.server.group.state({ includeKey: true });
|
|
74
|
+
assert.equal(after.keyVersion, before.keyVersion + 1, '移除成员必须抬 keyVersion');
|
|
75
|
+
assert.notEqual(after.groupKey, before.groupKey, '必须换一把新 groupKey');
|
|
76
|
+
assert.equal(after.members[removed.peerId], undefined, '成员表里不能再有它');
|
|
77
|
+
// 日志:一行摘要,且绝不打印密钥本身
|
|
78
|
+
const rotatedLine = lines.find((line) => line.startsWith('[group] key rotated '));
|
|
79
|
+
assert.ok(rotatedLine, '必须有 [group] key rotated 日志');
|
|
80
|
+
assert.match(rotatedLine, new RegExp(`keyVersion=${after.keyVersion} reason=member-removed`));
|
|
81
|
+
assert.equal(rotatedLine.includes(after.groupKey), false, '日志不得打印密钥');
|
|
82
|
+
assert.equal(rotatedLine.includes(before.groupKey), false, '日志不得打印旧密钥');
|
|
83
|
+
if (process.platform !== 'win32') {
|
|
84
|
+
const mode = (await import('node:fs')).statSync(hub.server.group.state ? hub.dir + '/group.json' : '').mode & 0o777;
|
|
85
|
+
assert.equal(mode, 0o600, 'group.json 仍必须 0600');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// 被移除者仍握着旧钥(它的 group-key.json 没变)→ 一律拒绝,且**不是** not-a-member。
|
|
89
|
+
// 有 body 的请求(心跳/投票)在信封层就先炸:400 bad-encryption;
|
|
90
|
+
// 无 body 的请求(GET /v1/group/lease)能走到验签:401 bad-signature。两者都证明旧钥已失效。
|
|
91
|
+
const rejectedWithOldKey = (error) => {
|
|
92
|
+
assert.equal(error.status === 401 || error.status === 400, true, '状态必须是 401/400,实际 ' + error.status);
|
|
93
|
+
assert.equal(['bad-signature', 'bad-encryption'].includes(String(error.code)), true, '错误码 ' + error.code);
|
|
94
|
+
assert.notEqual(String(error.code), 'not-a-member', '不能只是 not-a-member —— 说明密钥没轮换');
|
|
95
|
+
return true;
|
|
96
|
+
};
|
|
97
|
+
await assert.rejects(
|
|
98
|
+
() => sendGroupHeartbeat({
|
|
99
|
+
baseUrl: hub.baseUrl,
|
|
100
|
+
...groupArgs(cluster, removed.peerId),
|
|
101
|
+
epoch: 2,
|
|
102
|
+
leaderPeerId: removed.peerId,
|
|
103
|
+
}),
|
|
104
|
+
rejectedWithOldKey,
|
|
105
|
+
);
|
|
106
|
+
await assert.rejects(
|
|
107
|
+
() => requestGroupVote({
|
|
108
|
+
baseUrl: hub.baseUrl,
|
|
109
|
+
...groupArgs(cluster, removed.peerId),
|
|
110
|
+
epoch: 2,
|
|
111
|
+
candidate: removed.peerId,
|
|
112
|
+
}),
|
|
113
|
+
rejectedWithOldKey,
|
|
114
|
+
);
|
|
115
|
+
// 无 body 的那条给出精确的 401 bad-signature(验签先于成员表检查)
|
|
116
|
+
await assert.rejects(
|
|
117
|
+
() => fetchGroupLease({
|
|
118
|
+
baseUrl: hub.baseUrl,
|
|
119
|
+
...groupArgs(cluster, removed.peerId),
|
|
120
|
+
}),
|
|
121
|
+
(error) => {
|
|
122
|
+
assert.equal(error.status, 401);
|
|
123
|
+
assert.equal(error.code, 'bad-signature');
|
|
124
|
+
return true;
|
|
125
|
+
},
|
|
126
|
+
);
|
|
127
|
+
// hub 侧不再把它算进多数派分母
|
|
128
|
+
const lease = await fetchGroupLease({
|
|
129
|
+
baseUrl: hub.baseUrl,
|
|
130
|
+
groupKey: after.groupKey,
|
|
131
|
+
groupId: cluster.groupId,
|
|
132
|
+
senderId: cluster.nodes[1].peerId,
|
|
133
|
+
});
|
|
134
|
+
assert.equal(lease.memberCount, 2);
|
|
135
|
+
assert.equal(lease.keyVersion, after.keyVersion);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test('②b 被移除的成员不能靠自助 join 回来(同一个洞的另一扇门)', async (t) => {
|
|
139
|
+
const cluster = await startCluster(t, { count: 2, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
140
|
+
const [hub, member] = cluster.nodes;
|
|
141
|
+
assert.equal(hub.server.group.state().members[member.peerId] !== undefined, true);
|
|
142
|
+
hub.server.group.removeMember(member.peerId);
|
|
143
|
+
// 成员用 per-peer 签名打 /v1/group/join(autoJoin 是开的)——原来会被加回成员表
|
|
144
|
+
const res = await postAs(hub.baseUrl, member.peerId, { name: 'sneak-back' }, { path: '/v1/group/join' });
|
|
145
|
+
assert.equal(res.status, 403, res.text);
|
|
146
|
+
assert.equal(res.json.error, 'not-a-member');
|
|
147
|
+
assert.equal(hub.server.group.state().members[member.peerId], undefined, '成员表不得被 join 写回');
|
|
148
|
+
// 运维显式重新邀请(addMember)会解除 blocked,之后 join 也恢复正常语义
|
|
149
|
+
hub.server.group.addMember(member.peerId, 'node-b');
|
|
150
|
+
assert.equal(hub.server.group.state().members[member.peerId] !== undefined, true);
|
|
151
|
+
});
|
|
152
|
+
test('③ 成员换钥:hub 轮换后首次请求被拒 → 自动 refetch state → 重试一次成功', async (t) => {
|
|
153
|
+
const cluster = await startCluster(t, { count: 2, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
154
|
+
const [hub, member] = cluster.nodes;
|
|
155
|
+
// 让 hub 轮换密钥,但 member 仍在成员表里(模拟「踢掉第三台」后的换钥)
|
|
156
|
+
hub.server.group.addMember('peer-ghost-1', 'ghost');
|
|
157
|
+
hub.server.group.removeMember('peer-ghost-1');
|
|
158
|
+
const rotated = hub.server.group.state({ includeKey: true });
|
|
159
|
+
assert.ok(rotated.keyVersion > 1);
|
|
160
|
+
assert.notEqual(readGroupKeyFile(member.dir)?.key, rotated.groupKey, '前提:成员手里还是旧钥');
|
|
161
|
+
assert.equal(hub.server.group.members().some((entry) => entry.peerId === member.peerId), true);
|
|
162
|
+
|
|
163
|
+
// 成员的下一次出站群请求(拉补差)会先撞 401 bad-signature → 换钥 → 重试一次
|
|
164
|
+
let out = await member.server.group.pull();
|
|
165
|
+
for (let attempt = 0; attempt < 40 && out?.reason === 'skipped'; attempt += 1) {
|
|
166
|
+
await sleep(25);
|
|
167
|
+
out = await member.server.group.pull();
|
|
168
|
+
}
|
|
169
|
+
assert.equal(out.ok, true, JSON.stringify(out));
|
|
170
|
+
const refreshed = readGroupKeyFile(member.dir);
|
|
171
|
+
assert.equal(refreshed.key, rotated.groupKey, '成员必须把新钥落进 group-key.json');
|
|
172
|
+
assert.equal(roleOf(member).keyVersion, rotated.keyVersion, 'role() 必须暴露新 keyVersion');
|
|
173
|
+
// 只重试一次:再拉一次应当直接成功
|
|
174
|
+
const second = await member.server.group.pull();
|
|
175
|
+
assert.equal(second.ok, true);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test('④ 成员侧(副本)自愈仍然有效:表未同步但未移除的成员,election 仍被接受并补表', async (t) => {
|
|
179
|
+
const cluster = await startCluster(t, { count: 3, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
180
|
+
const [, b, c] = cluster.nodes;
|
|
181
|
+
// 造「c 的成员表来自不完整快照」:把 b 从 c 的 group-key.json 里删掉
|
|
182
|
+
const keyFile = readGroupKeyFile(c.dir);
|
|
183
|
+
delete keyFile.members[b.peerId];
|
|
184
|
+
writeGroupKeyFile(c.dir, keyFile);
|
|
185
|
+
assert.equal(Boolean(readGroupKeyFile(c.dir).members[b.peerId]), false);
|
|
186
|
+
|
|
187
|
+
const vote = await requestGroupVote({
|
|
188
|
+
baseUrl: c.baseUrl,
|
|
189
|
+
...groupArgs(cluster, b.peerId),
|
|
190
|
+
epoch: 3,
|
|
191
|
+
candidate: b.peerId,
|
|
192
|
+
});
|
|
193
|
+
assert.equal(vote.vote, true, JSON.stringify(vote));
|
|
194
|
+
assert.ok(readGroupKeyFile(c.dir).members[b.peerId], '成员侧必须把对端补回成员表');
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test('⑥ 轮换后 epoch 不回退:重启(新的 server 实例读同一份状态)时角色 epoch 不小于 keyVersion', async (t) => {
|
|
198
|
+
const cluster = await startCluster(t, { count: 2, heartbeatMs: 5_000, tickMs: 60_000 });
|
|
199
|
+
const [hub] = cluster.nodes;
|
|
200
|
+
hub.server.group.addMember('peer-ghost-2', 'ghost');
|
|
201
|
+
hub.server.group.removeMember('peer-ghost-2');
|
|
202
|
+
const keyVersion = hub.server.group.state().keyVersion;
|
|
203
|
+
assert.ok(keyVersion >= 2);
|
|
204
|
+
// 同一份 stateDir 再起一个实例(等价于重启):角色 epoch 必须 ≥ keyVersion
|
|
205
|
+
const { createPeerServer } = await import('../server.mjs');
|
|
206
|
+
const restarted = createPeerServer({
|
|
207
|
+
listenHost: '127.0.0.1',
|
|
208
|
+
stateDir: hub.dir,
|
|
209
|
+
requireEncryption: false,
|
|
210
|
+
group: { enabled: true, name: 's3', quorum: 'none', heartbeatMs: 5_000 },
|
|
211
|
+
groupElection: { heartbeatMs: 5_000, tickMs: 60_000, peerTimeoutMs: 200 },
|
|
212
|
+
});
|
|
213
|
+
t.after(async () => { await restarted.close(); });
|
|
214
|
+
const info = await restarted.listen(0, '127.0.0.1');
|
|
215
|
+
assert.equal(info.port > 0, true);
|
|
216
|
+
const role = restarted.group.role();
|
|
217
|
+
assert.ok(role.epoch >= keyVersion, '角色 epoch 不得小于 keyVersion,实际 ' + role.epoch + ' vs ' + keyVersion);
|
|
218
|
+
});
|
|
219
|
+
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.7.0 S3 补充 · 受限的启动竞选(priority-takeover)。
|
|
3
|
+
*
|
|
4
|
+
* 契约 §4 是「epoch 高者胜,同 epoch 才比 priority」——优先级不能跨 epoch 夺权,
|
|
5
|
+
* 于是优先节点重启期间被别人抢到更高 epoch 后会永久当 follower。这里给它一次有边界的重试权:
|
|
6
|
+
* 配置上没有 group.hub 的本机,启动稳定 2×heartbeatMs 后、优先级**严格更高**时尝试一次。
|
|
7
|
+
*
|
|
8
|
+
* 覆盖:①一次成功接管(epoch+1,原 leader 降级)②priority 相等不触发
|
|
9
|
+
* ③接管失败后推进 30s 也不重试 ④配了 group.hub 的成员不触发
|
|
10
|
+
*/
|
|
11
|
+
import test from 'node:test';
|
|
12
|
+
import assert from 'node:assert/strict';
|
|
13
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
14
|
+
import { roleOf, startCluster, waitFor } from './group-cluster.mjs';
|
|
15
|
+
|
|
16
|
+
const takeoverOf = (node) => node.server.group.takeover();
|
|
17
|
+
|
|
18
|
+
const electionLines = (lines, kind) => lines
|
|
19
|
+
.filter((line) => line.includes('[group] election ' + kind) && line.includes('reason=priority-takeover'));
|
|
20
|
+
|
|
21
|
+
test('S3 启动接管:高优先节点一次接管成功(epoch+1),原 leader 降级', async (t) => {
|
|
22
|
+
const lines = [];
|
|
23
|
+
const cluster = await startCluster(t, {
|
|
24
|
+
count: 2,
|
|
25
|
+
// 两台都是 hub 角色(配置上没有 group.hub),才有接管资格
|
|
26
|
+
hubRoles: [0, 1],
|
|
27
|
+
// 低优先(10)先当 leader;高优先(0)以 follower 起步
|
|
28
|
+
priorities: [10, 0],
|
|
29
|
+
followerOf: { 1: 0 },
|
|
30
|
+
logger: (line) => lines.push(String(line)),
|
|
31
|
+
});
|
|
32
|
+
const [low, high] = cluster.nodes;
|
|
33
|
+
assert.equal(roleOf(low).role, 'leader');
|
|
34
|
+
assert.equal(roleOf(high).role, 'follower');
|
|
35
|
+
assert.equal(roleOf(high).leaderPeerId, low.peerId);
|
|
36
|
+
assert.equal(takeoverOf(high).attempts, 0, '刚开始还不该动手');
|
|
37
|
+
|
|
38
|
+
await waitFor(() => takeoverOf(high).attempts === 1, { timeoutMs: 5_000 });
|
|
39
|
+
await waitFor(() => roleOf(high).role === 'leader', { timeoutMs: 5_000 });
|
|
40
|
+
await waitFor(() => roleOf(low).leaderPeerId === high.peerId, { timeoutMs: 5_000 });
|
|
41
|
+
|
|
42
|
+
assert.equal(roleOf(high).epoch, 2, '接管必须抬 epoch');
|
|
43
|
+
assert.equal(roleOf(low).role, 'follower');
|
|
44
|
+
assert.equal(roleOf(low).epoch, 2);
|
|
45
|
+
assert.equal(takeoverOf(high).attempts, 1);
|
|
46
|
+
assert.equal(takeoverOf(high).attempted, true);
|
|
47
|
+
|
|
48
|
+
const text = lines.join('\n');
|
|
49
|
+
assert.match(text, /\[group\] takeover attempt epoch=2 priority=0 leaderPriority=10/);
|
|
50
|
+
assert.equal(electionLines(lines, 'won').length, 1, '一次成功接管要留一行带票数的日志');
|
|
51
|
+
assert.match(text, /\[group\] takeover won epoch=2 votes=2\/2/);
|
|
52
|
+
|
|
53
|
+
// 稳定后不再重试:推进到远超过稳定期,attempts 仍然是 1
|
|
54
|
+
await sleep(400);
|
|
55
|
+
assert.equal(takeoverOf(high).attempts, 1);
|
|
56
|
+
assert.equal(electionLines(lines, 'won').length, 1);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('S3 启动接管:priority 相等时一律不触发', async (t) => {
|
|
60
|
+
const cluster = await startCluster(t, {
|
|
61
|
+
count: 2,
|
|
62
|
+
hubRoles: [0, 1],
|
|
63
|
+
priorities: [0, 0],
|
|
64
|
+
followerOf: { 1: 0 },
|
|
65
|
+
});
|
|
66
|
+
const [low, high] = cluster.nodes;
|
|
67
|
+
// 等过稳定期(2×heartbeat=240ms)再多等几轮 tick
|
|
68
|
+
await sleep(700);
|
|
69
|
+
assert.equal(takeoverOf(high).attempts, 0, '优先级相等不得夺权');
|
|
70
|
+
assert.equal(takeoverOf(high).attempted, false);
|
|
71
|
+
assert.equal(roleOf(high).role, 'follower');
|
|
72
|
+
assert.equal(roleOf(high).leaderPeerId, low.peerId);
|
|
73
|
+
assert.equal(roleOf(low).role, 'leader');
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('S3 启动接管:失败一次后不再重试(假时钟推进 30s,接管仍只有 1 次)', async (t) => {
|
|
77
|
+
let fake = Date.now();
|
|
78
|
+
const clock = () => fake;
|
|
79
|
+
const lines = [];
|
|
80
|
+
const cluster = await startCluster(t, {
|
|
81
|
+
count: 2,
|
|
82
|
+
hubRoles: [0, 1],
|
|
83
|
+
priorities: [10, 0],
|
|
84
|
+
followerOf: { 1: 0 },
|
|
85
|
+
// 心跳放慢,好让「稳定期」在假时钟里可控(租约 = 3×400 = 1200ms)
|
|
86
|
+
heartbeatMs: 400,
|
|
87
|
+
tickMs: 25,
|
|
88
|
+
now: clock,
|
|
89
|
+
logger: (line) => lines.push(String(line)),
|
|
90
|
+
});
|
|
91
|
+
const [low, high] = cluster.nodes;
|
|
92
|
+
// 让 low 的心跳把「leader 优先级=10」教给 high(假时钟下真实时间照样走几轮)
|
|
93
|
+
await sleep(600);
|
|
94
|
+
assert.equal(takeoverOf(high).attempts, 0, '还没到稳定期(2×400=800ms)');
|
|
95
|
+
|
|
96
|
+
// 把 leader 打死,再跨过稳定期:这一枪注定落空
|
|
97
|
+
await low.server.close();
|
|
98
|
+
fake += 900;
|
|
99
|
+
await waitFor(() => takeoverOf(high).attempts === 1, { timeoutMs: 3_000 });
|
|
100
|
+
assert.equal(roleOf(high).role, 'follower', '失败后不得成为 leader');
|
|
101
|
+
assert.equal(electionLines(lines, 'lost').length, 1);
|
|
102
|
+
assert.match(lines.join('\n'), /\[group\] takeover lost epoch=2 votes=1\/2/);
|
|
103
|
+
|
|
104
|
+
// 推进 30s:普通租约过期选举可以重试,但**接管**必须一次就停
|
|
105
|
+
fake += 30_000;
|
|
106
|
+
await sleep(400);
|
|
107
|
+
assert.equal(takeoverOf(high).attempts, 1, '接管每进程只允许一次');
|
|
108
|
+
assert.equal(electionLines(lines, 'lost').length, 1);
|
|
109
|
+
assert.equal(electionLines(lines, 'won').length, 0);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('S3 启动接管:配了 group.hub 的成员节点不触发(即使优先级更高)', async (t) => {
|
|
113
|
+
const cluster = await startCluster(t, { count: 2, priorities: [10, 0] });
|
|
114
|
+
const [low, member] = cluster.nodes;
|
|
115
|
+
assert.equal(low.server.group.isHub, true);
|
|
116
|
+
assert.equal(member.server.group.isHub, false);
|
|
117
|
+
await sleep(700);
|
|
118
|
+
assert.equal(takeoverOf(member).attempts, 0, '成员节点(配置了 group.hub)不得夺权');
|
|
119
|
+
assert.equal(roleOf(member).role, 'follower');
|
|
120
|
+
assert.equal(roleOf(low).role, 'leader');
|
|
121
|
+
});
|
|
122
|
+
|
package/host/test/group.test.mjs
CHANGED
|
@@ -278,7 +278,8 @@ test('§4 hub 首次初始化:group.json 0600、hub 自己是成员、管理
|
|
|
278
278
|
const raw = JSON.parse(fs.readFileSync(groupFile, 'utf8'));
|
|
279
279
|
// S3 §5:group.json 追加 pending(未提交区,与 messages 分开存)
|
|
280
280
|
assert.deepEqual(Object.keys(raw).sort(), [
|
|
281
|
-
|
|
281
|
+
// blocked:被移除的成员(自助 join 不得把 ta 加回来)
|
|
282
|
+
'blocked', 'createdAt', 'groupId', 'key', 'keyVersion', 'lastSeq', 'members', 'messages', 'name', 'pending',
|
|
282
283
|
]);
|
|
283
284
|
assert.equal(Buffer.from(raw.key, 'base64url').length, 32);
|
|
284
285
|
assert.equal(raw.name, 'team');
|
package/lib/admin-group.test.mjs
CHANGED
|
@@ -863,6 +863,95 @@ test('S3 · leader.epoch 以角色门面为准:角色 3 / 群元数据 1 → 3
|
|
|
863
863
|
assert.equal(fallback.json.leader.epoch, 1, '没有角色门面时回退到群元数据的 epoch');
|
|
864
864
|
});
|
|
865
865
|
|
|
866
|
+
test('S3 · 成员侧 leader.peerId 同样以角色门面为准(配置的 group.hub 只是引导入口)', async () => {
|
|
867
|
+
// 动态 hub 的语义:leader 可能是任何成员,group.hub 只是引导入口,不代表当前 leader。
|
|
868
|
+
const members = [
|
|
869
|
+
{ peerId: 'peer-hub', name: 'hub-box', online: true, lastSeen: null, joinedAt: 1, baseUrl: 'http://192.168.5.51:7788', lastSeq: 8 },
|
|
870
|
+
{ peerId: 'peer-other', name: 'dsh-desktop', online: true, lastSeen: null, joinedAt: 2, baseUrl: 'http://192.168.5.20:7788', lastSeq: 8 },
|
|
871
|
+
{ peerId: 'member-inst', name: 'me', online: true, lastSeen: null, joinedAt: 3, baseUrl: '', lastSeq: null },
|
|
872
|
+
];
|
|
873
|
+
const leadsAs = (leaderPeerId) => stubAdmin({
|
|
874
|
+
group: { enabled: true, hub: 'hub-box' },
|
|
875
|
+
api: apiWith({ isHub: false, lastSeq: 8, members }),
|
|
876
|
+
peers: HUB_PEERS,
|
|
877
|
+
facade: { enabled: false, isHub: false, role: () => ({ role: 'follower', epoch: 3, leaderPeerId, leaseUntil: Date.now() + 9000 }) },
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
const out = await hit(leadsAs('peer-other').admin, 'GET', '/api/lan-peers/group');
|
|
881
|
+
assert.equal(out.json.selfRole, 'member');
|
|
882
|
+
assert.equal(out.json.role, 'follower');
|
|
883
|
+
assert.equal(out.json.leader.peerId, 'peer-other', '门面的 leaderPeerId 压过配置的 group.hub');
|
|
884
|
+
assert.notEqual(out.json.leader.peerId, 'peer-hub');
|
|
885
|
+
assert.equal(out.json.leader.name, 'dsh-desktop', '名字从群成员表查');
|
|
886
|
+
assert.equal(out.json.leader.epoch, 3);
|
|
887
|
+
|
|
888
|
+
// 本机自己当 leader:成员表里有名字就用表里的,没有就用本机名(不是 UUID)
|
|
889
|
+
const named = await hit(leadsAs('member-inst').admin, 'GET', '/api/lan-peers/group');
|
|
890
|
+
assert.equal(named.json.leader.peerId, 'member-inst');
|
|
891
|
+
assert.equal(named.json.leader.name, 'me');
|
|
892
|
+
// 成员表里没有本机那条(还没进群状态):本机自己当 leader 时用自己的名字,不是 UUID
|
|
893
|
+
const unnamed = await hit(stubAdmin({
|
|
894
|
+
group: { enabled: true, hub: 'hub-box' },
|
|
895
|
+
api: apiWith({ isHub: false, lastSeq: 8 }),
|
|
896
|
+
peers: HUB_PEERS,
|
|
897
|
+
facade: { enabled: false, isHub: false, role: () => ({ role: 'leader', epoch: 3, leaderPeerId: 'inst-self' }) },
|
|
898
|
+
}).admin, 'GET', '/api/lan-peers/group');
|
|
899
|
+
assert.equal(unnamed.json.leader.peerId, 'inst-self', '本机(store.instanceId)当选');
|
|
900
|
+
assert.equal(unnamed.json.leader.name, 'self', '查不到时本机用自己的名字');
|
|
901
|
+
|
|
902
|
+
// 门面缺席 / leaderPeerId 为空:回退到配置的 group.hub(旧行为不破)
|
|
903
|
+
const legacy = await hit(stubAdmin({
|
|
904
|
+
group: { enabled: true, hub: 'hub-box' },
|
|
905
|
+
api: apiWith({ isHub: false, lastSeq: 8, members }),
|
|
906
|
+
peers: HUB_PEERS,
|
|
907
|
+
}).admin, 'GET', '/api/lan-peers/group');
|
|
908
|
+
assert.equal(legacy.json.leader.peerId, 'peer-hub', '没有角色门面时仍认配置的 hub');
|
|
909
|
+
assert.equal(legacy.json.leader.name, 'hub-box');
|
|
910
|
+
|
|
911
|
+
const emptyLeader = await hit(leadsAs('').admin, 'GET', '/api/lan-peers/group');
|
|
912
|
+
assert.equal(emptyLeader.json.leader.peerId, 'peer-hub', 'leaderPeerId 为空字符串等于没接线');
|
|
913
|
+
});
|
|
914
|
+
|
|
915
|
+
test('S3 · hub 且选举为 follower:leader.peerId 是真 leader(不是本机),selfRole 仍是 hub', async () => {
|
|
916
|
+
// 真机(0.7.2 NAS):配置上是 hub,但选举输给了 dsh-desktop。
|
|
917
|
+
const members = [
|
|
918
|
+
{ peerId: 'inst-self', name: 'fn-302', online: true, lastSeen: null, joinedAt: 1, baseUrl: 'http://192.168.5.51:7788', lastSeq: 8 },
|
|
919
|
+
{ peerId: 'peer-desktop', name: 'dsh-desktop', online: true, lastSeen: null, joinedAt: 2, baseUrl: 'http://192.168.5.20:7788', lastSeq: 8 },
|
|
920
|
+
];
|
|
921
|
+
const state = () => ({
|
|
922
|
+
groupId: 'g-1', name: 'default', epoch: 4, members, messages: MESSAGES, lastSeq: 8,
|
|
923
|
+
});
|
|
924
|
+
const followerHub = await hit(stubAdmin({
|
|
925
|
+
group: { enabled: true, hub: '' },
|
|
926
|
+
api: { ...makeApi(), isHub: true, state },
|
|
927
|
+
facade: { enabled: true, isHub: true, role: () => ({ role: 'follower', epoch: 4, leaderPeerId: 'peer-desktop', leaseUntil: Date.now() + 9000 }) },
|
|
928
|
+
}).admin, 'GET', '/api/lan-peers/group');
|
|
929
|
+
|
|
930
|
+
assert.equal(followerHub.json.selfRole, 'hub', 'selfRole 是「配置上的 hub」,与选举角色不是一个概念');
|
|
931
|
+
assert.equal(followerHub.json.role, 'follower', '选举角色如实透出');
|
|
932
|
+
assert.equal(followerHub.json.leader.peerId, 'peer-desktop', 'leader 必须是选举赢家,不能写死本机');
|
|
933
|
+
assert.notEqual(followerHub.json.leader.peerId, 'inst-self');
|
|
934
|
+
assert.equal(followerHub.json.leader.name, 'dsh-desktop', '名字从成员表里查');
|
|
935
|
+
assert.equal(followerHub.json.leader.epoch, 4);
|
|
936
|
+
|
|
937
|
+
// 成员表里查不到那个 peerId:名字退回 peerId(不编名字,也不回退本机)
|
|
938
|
+
const ghost = await hit(stubAdmin({
|
|
939
|
+
group: { enabled: true, hub: '' },
|
|
940
|
+
api: { ...makeApi(), isHub: true, state },
|
|
941
|
+
facade: { enabled: true, isHub: true, role: () => ({ role: 'follower', epoch: 4, leaderPeerId: 'peer-ghost' }) },
|
|
942
|
+
}).admin, 'GET', '/api/lan-peers/group');
|
|
943
|
+
assert.equal(ghost.json.leader.peerId, 'peer-ghost');
|
|
944
|
+
assert.equal(ghost.json.leader.name, 'peer-ghost', '查不到名字就用 peerId');
|
|
945
|
+
|
|
946
|
+
// 角色门面缺席(老 host / 未接线):回退到「hub 就是本机」的旧行为
|
|
947
|
+
const legacy = await hit(stubAdmin({
|
|
948
|
+
group: { enabled: true, hub: '' },
|
|
949
|
+
api: { ...makeApi(), isHub: true, state },
|
|
950
|
+
}).admin, 'GET', '/api/lan-peers/group');
|
|
951
|
+
assert.equal(legacy.json.leader.peerId, 'inst-self', '没有角色门面时回退本机');
|
|
952
|
+
assert.equal(legacy.json.leader.name, 'self');
|
|
953
|
+
});
|
|
954
|
+
|
|
866
955
|
test('S3 · 群状态快照里的角色字段是第二来源(门面缺席时也能显示)', async () => {
|
|
867
956
|
const leaseUntil = Date.now() + 9000;
|
|
868
957
|
const api = apiWith({ isHub: true, lastSeq: 8 });
|
package/lib/admin.mjs
CHANGED
|
@@ -339,15 +339,39 @@ export function createAdmin({
|
|
|
339
339
|
return typeof fromConfig === 'string' ? fromConfig : '';
|
|
340
340
|
});
|
|
341
341
|
|
|
342
|
+
/** 群状态成员表里的名字(找不到就是空串):给「真 leader 不是本机」时用。 */
|
|
343
|
+
const memberNameOf = (snapshot, peerId) => {
|
|
344
|
+
if (peerId === '') return '';
|
|
345
|
+
const list = Array.isArray(snapshot?.members) ? snapshot.members : [];
|
|
346
|
+
const found = list.find((member) => String(member?.peerId ?? '') === peerId);
|
|
347
|
+
return String(found?.name ?? '');
|
|
348
|
+
};
|
|
349
|
+
|
|
342
350
|
/**
|
|
343
|
-
* S1:leader —— hub 侧就是本机,成员侧就是 group.hub 指向的那个 peer。
|
|
351
|
+
* S1:leader —— 没有选举信息时,hub 侧就是本机,成员侧就是 group.hub 指向的那个 peer。
|
|
344
352
|
*
|
|
345
|
-
* S3
|
|
346
|
-
*
|
|
347
|
-
*
|
|
353
|
+
* S3:角色门面说了算的两个字段:
|
|
354
|
+
* - `epoch` 取选举世代(群元数据那份是 group.json 的 keyVersion 世代,真机上出现
|
|
355
|
+
* 过「leader.epoch=1 而状态行 epoch=3」的打架);
|
|
356
|
+
* - `leaderPeerId` 取真 leader(配置上是 hub 的机器可能输掉选举,那时 selfRole 仍是
|
|
357
|
+
* hub、role 是 follower,leader 必须是赢家,而不是本机)。
|
|
358
|
+
* 门面缺席(老 host / 未接线)时两者都回退到 S1 的旧口径,行为不变。
|
|
348
359
|
*/
|
|
349
360
|
const leaderOf = (snapshot, resolved, face = null) => {
|
|
350
361
|
const epoch = asSeqOrNull(face?.epoch) ?? asSeqOrNull(snapshot?.epoch);
|
|
362
|
+
const declared = face && typeof face === 'object' ? face.leaderPeerId : undefined;
|
|
363
|
+
const declaredId = declared == null ? '' : String(declared);
|
|
364
|
+
if (declaredId !== '') {
|
|
365
|
+
const fromMembers = memberNameOf(snapshot, declaredId);
|
|
366
|
+
// 成员表里查不到时:若声明的 leader 就是本机,仍用本机名称(别把 peerId 当名字显示);
|
|
367
|
+
// 是别的机器就退到 peerId(面板会原样显示,不会编名字)。
|
|
368
|
+
const name = fromMembers !== ''
|
|
369
|
+
? fromMembers
|
|
370
|
+
: (declaredId === String(store?.instanceId ?? '')
|
|
371
|
+
? String(typeof nameOf === 'function' ? nameOf() : '')
|
|
372
|
+
: declaredId);
|
|
373
|
+
return { peerId: declaredId, name, epoch };
|
|
374
|
+
}
|
|
351
375
|
if (resolved.isHub === true) {
|
|
352
376
|
return {
|
|
353
377
|
peerId: String(store?.instanceId ?? ''),
|