@hile/micro 2.0.8 → 2.1.1

package/dist/application.js

@@ -35,39 +35,122 @@ function withTimeout(promise, ms, label) {
             clearTimeout(timer);
     });
 }
+function createPubSubPayloadSnapshot(topic, payload) {
+    const serialized = JSON.stringify({ topic, payload });
+    const parsed = JSON.parse(serialized);
+    if (Object.prototype.hasOwnProperty.call(parsed, 'payload')) {
+        return parsed.payload;
+    }
+    return undefined;
+}
+function stablePayloadSignature(value) {
+    if (value === undefined)
+        return 'undefined';
+    if (value === null)
+        return 'null';
+    const type = typeof value;
+    if (type === 'string')
+        return `string:${JSON.stringify(value)}`;
+    if (type === 'number')
+        return `number:${String(value)}`;
+    if (type === 'boolean')
+        return `boolean:${String(value)}`;
+    if (Array.isArray(value)) {
+        return `array:[${value.map(stablePayloadSignature).join(',')}]`;
+    }
+    if (type === 'object') {
+        const keys = Object.keys(value).sort();
+        return `object:{${keys.map(key => `${JSON.stringify(key)}:${stablePayloadSignature(value[key])}`).join(',')}}`;
+    }
+    return `${type}:${String(value)}`;
+}
+const DEFAULT_CIRCUIT_BREAKER = {
+    failureThreshold: 3,
+    failureWindowMs: 60_000,
+    successThreshold: 2,
+    cooldownMs: 10_000,
+    maxCooldownMs: 120_000,
+    halfOpenMaxProbes: 1,
+    shouldRecordFailure: () => true,
+    shouldRetry: () => true,
+};
+function positiveIntegerOr(value, fallback) {
+    if (!Number.isFinite(value) || value === undefined || value < 1)
+        return fallback;
+    return Math.trunc(value);
+}
+function resolveCircuitBreakerOptions(options) {
+    const cooldownMs = positiveIntegerOr(options?.cooldownMs, DEFAULT_CIRCUIT_BREAKER.cooldownMs);
+    const configuredMaxCooldownMs = positiveIntegerOr(options?.maxCooldownMs, DEFAULT_CIRCUIT_BREAKER.maxCooldownMs);
+    return {
+        failureThreshold: positiveIntegerOr(options?.failureThreshold, DEFAULT_CIRCUIT_BREAKER.failureThreshold),
+        failureWindowMs: positiveIntegerOr(options?.failureWindowMs, DEFAULT_CIRCUIT_BREAKER.failureWindowMs),
+        successThreshold: positiveIntegerOr(options?.successThreshold, DEFAULT_CIRCUIT_BREAKER.successThreshold),
+        cooldownMs,
+        maxCooldownMs: Math.max(cooldownMs, configuredMaxCooldownMs),
+        halfOpenMaxProbes: positiveIntegerOr(options?.halfOpenMaxProbes, DEFAULT_CIRCUIT_BREAKER.halfOpenMaxProbes),
+        shouldRecordFailure: options?.shouldRecordFailure ?? DEFAULT_CIRCUIT_BREAKER.shouldRecordFailure,
+        shouldRetry: options?.shouldRetry ?? DEFAULT_CIRCUIT_BREAKER.shouldRetry,
+    };
+}
 export class Application extends Server {
     registry;
     reconnectTimeout;
     registryReconnectPromise;
+    registryReconnectGeneration = 0;
     /** 为 true 时不再向 Registry 重连（listen 返回的 teardown 已触发） */
     stopped = false;
+    listenGeneration = 0;
     _registry_address;
     _registryLookupTimeoutMs;
     _requestTimeoutMs;
+    _circuitBreaker;
     namespaces = new Map();
-    static CB_COOLDOWN_MS = 30_000;
     circuitBreakers = new Map();
-    fallbacks = new Set();
     topics = new Map();
+    publishedTopics = new Map();
+    publishedTopicRevisions = new Map();
+    publishedTopicDirty = new Set();
+    publishedTopicVersions = new Map();
+    publishedTopicSignatures = new Map();
+    topicSyncs = new Map();
+    topicUpdateVersions = new Map();
+    publishIntentVersion = 0;
     constructor(props) {
-        const { namespace, registry, registryLookupTimeoutMs = 10_000, requestTimeoutMs = 30_000, ...microAndLoader } = props;
+        const { namespace, registry, registryLookupTimeoutMs = 10_000, requestTimeoutMs = 30_000, circuitBreaker, ...microAndLoader } = props;
         super(namespace, microAndLoader);
         assertValidRegistrySocket('registry address', registry.host, registry.port);
         this._registry_address = registry;
         this._registryLookupTimeoutMs = registryLookupTimeoutMs;
         this._requestTimeoutMs = requestTimeoutMs;
-        this.fallbacks.add(this.register('/-/health', async () => ({
+        this._circuitBreaker = resolveCircuitBreakerOptions(circuitBreaker);
+        this.register('/-/health', async () => ({
             status: 'ok',
             registry: !!this.registry,
             uptime: process.uptime(),
             namespaces: [...this.namespaces.keys()],
-        })));
-        this.fallbacks.add(this.register('/-/topic/update', async ({ data }) => {
-            this.events.emit('topic:' + data.topic, data.payload);
-            return Date.now();
         }));
+        this.register('/-/topic/update', async ({ data }) => {
+            this.dispatchTopicUpdate(data.topic, data.payload);
+            return Date.now();
+        });
+    }
+    dispatchTopicUpdate(topic, payload) {
+        this.topicUpdateVersions.set(topic, (this.topicUpdateVersions.get(topic) ?? 0) + 1);
+        for (const listener of this.events.listeners('topic:' + topic)) {
+            try {
+                const listenerPayload = createPubSubPayloadSnapshot(topic, payload);
+                void Promise.resolve(listener(listenerPayload)).catch(err => {
+                    this.logger.error(err);
+                });
+            }
+            catch (err) {
+                this.logger.error(err);
+            }
+        }
     }
     async listen(port = 0) {
+        const generation = ++this.listenGeneration;
         this.stopped = false;
         const callback = await super.listen(port);
         try {
@@ -86,10 +169,9 @@ export class Application extends Server {
         // 这里也不清理 declare 和 undeclare 由业务方自己清理
         return async () => {
             this.stopped = true;
-            for (const fallback of this.fallbacks) {
-                fallback();
+            if (this.listenGeneration === generation) {
+                this.listenGeneration++;
             }
-            this.fallbacks.clear();
             if (this.reconnectTimeout) {
                 clearTimeout(this.reconnectTimeout);
                 this.reconnectTimeout = undefined;
@@ -99,29 +181,180 @@ export class Application extends Server {
             await callback();
         };
     }
-    scheduleRegistryRetry() {
-        if (this.stopped)
+    scheduleRegistryRetry(generation = this.listenGeneration) {
+        if (this.stopped || this.listenGeneration !== generation)
             return;
         if (this.reconnectTimeout)
             clearTimeout(this.reconnectTimeout);
         this.reconnectTimeout = setTimeout(() => {
             this.reconnectTimeout = undefined;
+            if (this.stopped || this.listenGeneration !== generation)
+                return;
             this.logger.debug('[reconnecting] %s:%d', this._registry_address.host, this._registry_address.port);
             void this.reconnectToRegistry().catch(() => {
-                if (this.stopped)
+                if (this.stopped || this.listenGeneration !== generation)
                     return;
-                this.scheduleRegistryRetry();
+                this.scheduleRegistryRetry(generation);
             });
         }, 3000);
     }
+    canUsePubSub() {
+        return !!this.port && !this.stopped;
+    }
+    assertCanUsePubSub() {
+        if (!this.canUsePubSub()) {
+            throw new Error('Registry not found');
+        }
+    }
+    ensureRegistryReconnectScheduled() {
+        if (!this.canUsePubSub())
+            return;
+        const generation = this.listenGeneration;
+        void this.reconnectToRegistry().catch(() => {
+            if (this.stopped || this.listenGeneration !== generation)
+                return;
+            this.scheduleRegistryRetry(generation);
+        });
+    }
+    handleRegistrySyncFailure(registry, err) {
+        if (this.stopped)
+            return;
+        const generation = this.listenGeneration;
+        this.logger.error(err);
+        if (this.registry === registry) {
+            this.registry = undefined;
+            registry.dispose();
+        }
+        if (!this.stopped && this.listenGeneration === generation) {
+            this.scheduleRegistryRetry(generation);
+        }
+    }
+    registryRequestOptions() {
+        if (!Number.isFinite(this._registryLookupTimeoutMs) || this._registryLookupTimeoutMs <= 0) {
+            return undefined;
+        }
+        return { timeout: this._registryLookupTimeoutMs };
+    }
+    recordPublishedTopic(topic, payload) {
+        const signature = stablePayloadSignature(payload);
+        const previousSignature = this.publishedTopicSignatures.get(topic);
+        const version = this.publishedTopics.has(topic) && previousSignature === signature
+            ? this.publishedTopicVersions.get(topic) ?? ++this.publishIntentVersion
+            : ++this.publishIntentVersion;
+        this.publishedTopics.set(topic, payload);
+        this.publishedTopicDirty.add(topic);
+        this.publishedTopicVersions.set(topic, version);
+        this.publishedTopicSignatures.set(topic, signature);
+        return version;
+    }
+    enqueueTopicSync(topic, operation, options = {}) {
+        const previous = this.topicSyncs.get(topic) ?? Promise.resolve();
+        const run = previous.catch(() => undefined).then(async () => {
+            if (!this.canUsePubSub())
+                return;
+            const registry = this.registry;
+            if (!registry) {
+                this.ensureRegistryReconnectScheduled();
+                if (options.propagateError)
+                    throw new Error('Registry not found');
+                return;
+            }
+            try {
+                await operation(registry);
+            }
+            catch (err) {
+                this.handleRegistrySyncFailure(registry, err);
+                if (options.propagateError)
+                    throw err;
+            }
+        });
+        const tracked = run.finally(() => {
+            if (this.topicSyncs.get(topic) === tracked) {
+                this.topicSyncs.delete(topic);
+            }
+        });
+        this.topicSyncs.set(topic, tracked);
+        return tracked;
+    }
+    syncPublishedTopic(topic, options = {}) {
+        const hasPayload = Object.prototype.hasOwnProperty.call(options, 'payload');
+        return this.enqueueTopicSync(topic, async (registry) => {
+            if (!this.publishedTopics.has(topic))
+                return;
+            const version = options.version ?? this.publishedTopicVersions.get(topic);
+            const request = {
+                topic,
+                payload: hasPayload ? options.payload : this.publishedTopics.get(topic),
+            };
+            const preserveRevision = options.preserveRevision
+                ?? (!this.publishedTopicDirty.has(topic) && this.publishedTopicRevisions.has(topic));
+            if (preserveRevision && this.publishedTopicRevisions.has(topic)) {
+                request.revision = this.publishedTopicRevisions.get(topic);
+            }
+            const revision = await registry.request('/-/declare', request, this.registryRequestOptions());
+            if (this.publishedTopics.has(topic) &&
+                this.publishedTopicVersions.get(topic) === version &&
+                Number.isFinite(revision)) {
+                this.publishedTopicRevisions.set(topic, revision);
+                this.publishedTopicDirty.delete(topic);
+            }
+        }, options);
+    }
+    syncUnpublishedTopic(topic, options = {}) {
+        return this.enqueueTopicSync(topic, async (registry) => {
+            await registry.request('/-/undeclare', { topic }, this.registryRequestOptions());
+        }, options);
+    }
+    syncUnsubscribedTopic(topic, options = {}) {
+        return this.enqueueTopicSync(topic, async (registry) => {
+            await registry.request('/-/unsubscribe', { topic }, this.registryRequestOptions());
+        }, options);
+    }
+    syncRestoredSubscription(topic, callback, options = {}) {
+        return this.enqueueTopicSync(topic, async (registry) => {
+            if (!this.topics.get(topic)?.has(callback))
+                return;
+            await this.restoreSubscription(topic, callback, registry, true);
+        }, options);
+    }
+    async restoreSubscription(topic, callback, registry, isReconnect, requireLocal = true) {
+        const replayBaseVersion = this.topicUpdateVersions.get(topic) ?? 0;
+        const snapshot = await registry.request('/-/subscribe', { topic }, this.registryRequestOptions());
+        if (requireLocal && !this.topics.get(topic)?.has(callback))
+            return;
+        if (!snapshot.hasData)
+            return;
+        if ((this.topicUpdateVersions.get(topic) ?? 0) !== replayBaseVersion)
+            return;
+        try {
+            await Promise.resolve(callback(snapshot.payload));
+        }
+        catch (err) {
+            if (!isReconnect)
+                throw err;
+            this.logger.error(err);
+        }
+    }
+    async rollbackLocalSubscription(topic, callback) {
+        const callbacks = this.topics.get(topic);
+        callbacks?.delete(callback);
+        this.events.off('topic:' + topic, callback);
+        if (callbacks?.size === 0) {
+            this.topics.delete(topic);
+            await this.syncUnsubscribedTopic(topic);
+        }
+    }
     async reconnectToRegistry() {
         if (this.stopped)
             return;
-        if (this.registryReconnectPromise)
+        const generation = this.listenGeneration;
+        if (this.registryReconnectPromise && this.registryReconnectGeneration === generation) {
             return this.registryReconnectPromise;
-        this.registryReconnectPromise = (async () => {
+        }
+        this.registryReconnectGeneration = generation;
+        const run = (async () => {
             const registry = await this.connect(this._registry_address.host, this._registry_address.port);
-            if (this.stopped) {
+            if (this.stopped || this.listenGeneration !== generation) {
                 registry.dispose();
                 return;
             }
@@ -129,58 +362,304 @@ export class Application extends Server {
                 if (this.registry !== registry)
                     return;
                 this.registry = undefined;
-                if (this.stopped)
+                if (this.stopped || this.listenGeneration !== generation)
                     return;
                 void this.reconnectToRegistry().catch(() => {
-                    if (this.stopped)
+                    if (this.stopped || this.listenGeneration !== generation)
                         return;
-                    this.scheduleRegistryRetry();
+                    this.scheduleRegistryRetry(generation);
                 });
             });
             this.registry = registry;
+            // 重新声明所有仍处于发布状态的 topic
+            for (const topic of [...this.publishedTopics.keys()]) {
+                if (this.stopped || this.listenGeneration !== generation) {
+                    registry.dispose();
+                    return;
+                }
+                if (!this.publishedTopics.has(topic))
+                    continue;
+                await this.syncPublishedTopic(topic, { propagateError: true });
+            }
             // 重新订阅所有 topic
-            for (const [topic, callback] of this.topics) {
-                await this.subscribe(topic, callback, true);
+            for (const [topic, callbacks] of [...this.topics]) {
+                for (const callback of [...callbacks]) {
+                    if (this.stopped || this.listenGeneration !== generation) {
+                        registry.dispose();
+                        return;
+                    }
+                    if (!this.topics.get(topic)?.has(callback))
+                        continue;
+                    await this.syncRestoredSubscription(topic, callback, { propagateError: true });
+                }
             }
             this.logger.debug('[reconnected] %s:%d', this._registry_address.host, this._registry_address.port);
-        })().finally(() => {
-            this.registryReconnectPromise = undefined;
+        })();
+        const tracked = run.finally(() => {
+            if (this.registryReconnectPromise === tracked) {
+                this.registryReconnectPromise = undefined;
+                this.registryReconnectGeneration = 0;
+            }
         });
-        return this.registryReconnectPromise;
+        this.registryReconnectPromise = tracked;
+        return tracked;
+    }
+    peerKey(host, port) {
+        return `${host}:${port}`;
     }
-    recordSuccess(ns, host, port) {
-        const excludes = this.circuitBreakers.get(ns);
-        if (excludes) {
-            excludes.delete(`${host}:${port}`);
-            if (excludes.size === 0)
-                this.circuitBreakers.delete(ns);
+    getPeerStates(ns, create = false) {
+        let states = this.circuitBreakers.get(ns);
+        if (!states && create) {
+            states = new Map();
+            this.circuitBreakers.set(ns, states);
         }
+        return states;
     }
-    recordFailure(ns, host, port) {
-        let excludes = this.circuitBreakers.get(ns);
-        if (!excludes) {
-            excludes = new Map();
-            this.circuitBreakers.set(ns, excludes);
+    getOrCreatePeerState(ns, host, port) {
+        const states = this.getPeerStates(ns, true);
+        const key = this.peerKey(host, port);
+        let state = states.get(key);
+        if (!state) {
+            state = {
+                status: 'closed',
+                failures: 0,
+                successes: 0,
+                openedAt: 0,
+                nextAttemptAt: 0,
+                cooldownMs: this._circuitBreaker.cooldownMs,
+                lastFailureAt: 0,
+                halfOpenInFlight: 0,
+            };
+            states.set(key, state);
         }
-        excludes.set(`${host}:${port}`, Date.now());
+        return state;
     }
-    getActiveExcludes(ns) {
-        const excludes = this.circuitBreakers.get(ns);
-        if (!excludes)
-            return [];
+    deletePeerState(ns, key) {
+        const states = this.circuitBreakers.get(ns);
+        if (!states)
+            return;
+        states.delete(key);
+        if (states.size === 0)
+            this.circuitBreakers.delete(ns);
+    }
+    openCircuit(state, cooldownMs) {
+        const now = Date.now();
+        state.status = 'open';
+        state.successes = 0;
+        state.openedAt = now;
+        state.lastFailureAt = now;
+        state.cooldownMs = Math.min(cooldownMs, this._circuitBreaker.maxCooldownMs);
+        state.nextAttemptAt = now + state.cooldownMs;
+        state.halfOpenInFlight = 0;
+    }
+    acquireCircuitProbe(ns, host, port) {
+        const key = this.peerKey(host, port);
+        const state = this.circuitBreakers.get(ns)?.get(key);
+        if (state?.status === 'open') {
+            return {
+                acquired: false,
+                wasHalfOpen: false,
+                cooldownMs: state.cooldownMs,
+                release: () => { },
+            };
+        }
+        if (state?.status !== 'half-open') {
+            return {
+                acquired: true,
+                wasHalfOpen: false,
+                cooldownMs: this._circuitBreaker.cooldownMs,
+                release: () => { },
+            };
+        }
+        if (state.halfOpenInFlight >= this._circuitBreaker.halfOpenMaxProbes) {
+            return {
+                acquired: false,
+                wasHalfOpen: true,
+                cooldownMs: state.cooldownMs,
+                release: () => { },
+            };
+        }
+        state.halfOpenInFlight++;
+        let released = false;
+        return {
+            acquired: true,
+            wasHalfOpen: true,
+            cooldownMs: state.cooldownMs,
+            release: () => {
+                if (released)
+                    return;
+                released = true;
+                const current = this.circuitBreakers.get(ns)?.get(key);
+                if (current?.status === 'half-open' && current.halfOpenInFlight > 0) {
+                    current.halfOpenInFlight--;
+                }
+            },
+        };
+    }
+    shouldRecordCircuitFailure(err) {
+        try {
+            return this._circuitBreaker.shouldRecordFailure(err);
+        }
+        catch (hookErr) {
+            this.logger.error(hookErr);
+            return true;
+        }
+    }
+    shouldRetryCircuitFailure(err) {
+        try {
+            return this._circuitBreaker.shouldRetry(err);
+        }
+        catch (hookErr) {
+            this.logger.error(hookErr);
+            return false;
+        }
+    }
+    recordSuccess(ns, host, port, probe) {
+        const key = this.peerKey(host, port);
+        const state = this.circuitBreakers.get(ns)?.get(key);
+        if (!state)
+            return;
+        if (probe.wasHalfOpen) {
+            if (state.status !== 'half-open')
+                return;
+            state.failures = 0;
+            state.successes++;
+            if (state.successes < this._circuitBreaker.successThreshold)
+                return;
+        }
+        else if (state.status !== 'closed') {
+            return;
+        }
+        this.deletePeerState(ns, key);
+    }
+    recordFailure(ns, host, port, err, probe) {
+        if (!this.shouldRecordCircuitFailure(err))
+            return;
         const now = Date.now();
-        const active = [];
-        for (const [key, openedAt] of excludes) {
-            if (now - openedAt >= Application.CB_COOLDOWN_MS) {
-                excludes.delete(key);
+        const key = this.peerKey(host, port);
+        if (probe.wasHalfOpen) {
+            const state = this.circuitBreakers.get(ns)?.get(key);
+            if (state?.status !== 'half-open')
+                return;
+            state.failures = this._circuitBreaker.failureThreshold;
+            state.successes = 0;
+            this.openCircuit(state, Math.max(state.cooldownMs, probe.cooldownMs) * 2);
+            return;
+        }
+        const state = this.getOrCreatePeerState(ns, host, port);
+        if (state.status !== 'closed')
+            return;
+        state.successes = 0;
+        if (state.lastFailureAt > 0 && now - state.lastFailureAt >= this._circuitBreaker.failureWindowMs) {
+            state.failures = 0;
+        }
+        state.lastFailureAt = now;
+        state.failures++;
+        if (state.failures >= this._circuitBreaker.failureThreshold) {
+            this.openCircuit(state, this._circuitBreaker.cooldownMs);
+        }
+    }
+    getActiveCircuitExcludes(ns) {
+        const states = this.circuitBreakers.get(ns);
+        if (!states)
+            return { keys: [], hasProbeLimitedPeer: false };
+        const now = Date.now();
+        const keys = [];
+        let hasProbeLimitedPeer = false;
+        for (const [key, state] of states) {
+            if (state.status === 'closed' &&
+                state.lastFailureAt > 0 &&
+                now - state.lastFailureAt >= this._circuitBreaker.failureWindowMs) {
+                states.delete(key);
+                continue;
             }
-            else {
-                active.push(key);
+            if (state.status === 'open') {
+                if (now >= state.nextAttemptAt) {
+                    state.status = 'half-open';
+                    state.successes = 0;
+                    state.halfOpenInFlight = 0;
+                }
+                else {
+                    keys.push(key);
+                }
+            }
+            if (state.status === 'half-open' && state.halfOpenInFlight >= this._circuitBreaker.halfOpenMaxProbes) {
+                keys.push(key);
+                hasProbeLimitedPeer = true;
             }
         }
-        if (excludes.size === 0)
+        if (states.size === 0)
             this.circuitBreakers.delete(ns);
-        return active;
+        return { keys, hasProbeLimitedPeer };
+    }
+    getActiveExcludes(ns) {
+        return this.getActiveCircuitExcludes(ns).keys;
+    }
+    trackCircuitStream(ns, host, port, probe, readable) {
+        let settled = false;
+        const settle = (status, err) => {
+            if (settled)
+                return;
+            settled = true;
+            if (status === 'failure') {
+                this.recordFailure(ns, host, port, err, probe);
+            }
+            else if (status === 'success') {
+                this.recordSuccess(ns, host, port, probe);
+            }
+            probe.release();
+        };
+        readable.once('error', err => settle('failure', err));
+        readable.once('end', () => settle('success'));
+        readable.once('close', () => settle('neutral'));
+        return readable;
+    }
+    async selectCircuitClient(namespace, allowExcludedCachedFallback = true) {
+        const { keys: exclude, hasProbeLimitedPeer } = this.getActiveCircuitExcludes(namespace);
+        try {
+            return await this.resolveClient(namespace, exclude, {
+                allowExcludedCachedFallback: allowExcludedCachedFallback && !hasProbeLimitedPeer,
+            });
+        }
+        catch (err) {
+            if (!allowExcludedCachedFallback || hasProbeLimitedPeer)
+                throw err;
+            this.circuitBreakers.delete(namespace);
+            return this.get(namespace);
+        }
+    }
+    async selectCircuitProbe(namespace, allowCircuitReset) {
+        let client = await this.selectCircuitClient(namespace, allowCircuitReset);
+        let probe = this.acquireCircuitProbe(namespace, client.host, client.port);
+        if (probe.acquired)
+            return { client, probe };
+        let blockedByHalfOpenProbe = probe.wasHalfOpen;
+        let resetClient = probe.wasHalfOpen ? undefined : client;
+        try {
+            client = await this.selectCircuitClient(namespace, false);
+            probe = this.acquireCircuitProbe(namespace, client.host, client.port);
+            if (probe.acquired)
+                return { client, probe };
+            blockedByHalfOpenProbe ||= probe.wasHalfOpen;
+            if (!probe.wasHalfOpen)
+                resetClient = client;
+        }
+        catch (err) {
+            if (!allowCircuitReset)
+                throw err;
+        }
+        if (!allowCircuitReset || blockedByHalfOpenProbe) {
+            throw new Error(`Circuit breaker probe unavailable for ${namespace}`);
+        }
+        this.circuitBreakers.delete(namespace);
+        // Preserve a cached client selected before the reset. A registry failure can
+        // clear namespace cache while the peer connection itself is still usable.
+        client = resetClient ?? await this.get(namespace);
+        probe = this.acquireCircuitProbe(namespace, client.host, client.port);
+        if (!probe.acquired) {
+            throw new Error(`Circuit breaker probe unavailable for ${namespace}`);
+        }
+        return { client, probe };
     }
     async findFromRegistry(namespace, exclude) {
         if (!this.registry)
@@ -189,6 +668,9 @@ export class Application extends Server {
         return await withTimeout(promise, this._registryLookupTimeoutMs, 'Registry /-/find');
     }
     get(namespace, exclude) {
+        return this.resolveClient(namespace, exclude);
+    }
+    resolveClient(namespace, exclude, options = {}) {
         if (!this.namespaces.has(namespace)) {
             this.namespaces.set(namespace, {
                 host: '',
@@ -238,7 +720,9 @@ export class Application extends Server {
                 }).catch(e => {
                     // Registry unavailable but previously cached client still valid -> degrade
                     const cachedKey = `${cachedHost}:${cachedPort}`;
-                    if (cachedHost && this.clients.has(cachedKey)) {
+                    const cachedExcluded = exclude?.includes(cachedKey);
+                    const allowCachedFallback = options.allowExcludedCachedFallback ?? true;
+                    if (cachedHost && this.clients.has(cachedKey) && (allowCachedFallback || !cachedExcluded)) {
                         const client = this.clients.get(cachedKey);
                         // Restore cache so subsequent calls hit the fast path
                         stack.host = cachedHost;
@@ -259,96 +743,174 @@ export class Application extends Server {
     }
     async call(namespace, url, data, options) {
         const { timeout = this._requestTimeoutMs, retries = 1, signal } = options || {};
-        const exclude = this.getActiveExcludes(namespace);
-        let client;
-        try {
-            client = await this.get(namespace, exclude);
-        }
-        catch {
-            this.circuitBreakers.delete(namespace);
-            client = await this.get(namespace);
-        }
-        try {
-            const result = await client.request(url, data, {
-                timeout: timeout ?? this._requestTimeoutMs,
-                signal,
-            });
-            this.recordSuccess(namespace, client.host, client.port);
-            return result;
-        }
-        catch (err) {
-            this.recordFailure(namespace, client.host, client.port);
-            if (retries > 0) {
-                return this.call(namespace, url, data, { timeout, retries: retries - 1, signal });
+        let remainingRetries = retries;
+        let retrySourceError;
+        let hasRetrySourceError = false;
+        for (;;) {
+            let selected;
+            try {
+                selected = await this.selectCircuitProbe(namespace, !hasRetrySourceError);
+            }
+            catch (err) {
+                if (hasRetrySourceError)
+                    throw retrySourceError;
+                throw err;
+            }
+            const { client, probe } = selected;
+            try {
+                const result = await client.request(url, data, {
+                    timeout: timeout ?? this._requestTimeoutMs,
+                    signal,
+                });
+                this.recordSuccess(namespace, client.host, client.port, probe);
+                return result;
+            }
+            catch (err) {
+                this.recordFailure(namespace, client.host, client.port, err, probe);
+                if (remainingRetries > 0 && this.shouldRetryCircuitFailure(err)) {
+                    retrySourceError = err;
+                    hasRetrySourceError = true;
+                    remainingRetries--;
+                    continue;
+                }
+                throw err;
+            }
+            finally {
+                probe.release();
             }
-            throw err;
         }
     }
     async stream(namespace, url, data, options) {
         const { signal, retries = 1 } = options || {};
-        const exclude = this.getActiveExcludes(namespace);
-        let client;
-        try {
-            client = await this.get(namespace, exclude);
-        }
-        catch {
-            this.circuitBreakers.delete(namespace);
-            client = await this.get(namespace);
-        }
-        try {
-            const readable = client.stream(url, data, { signal });
-            this.recordSuccess(namespace, client.host, client.port);
-            return readable;
-        }
-        catch (err) {
-            this.recordFailure(namespace, client.host, client.port);
-            if (retries > 0) {
-                return this.stream(namespace, url, data, { signal, retries: retries - 1 });
+        let remainingRetries = retries;
+        let retrySourceError;
+        let hasRetrySourceError = false;
+        for (;;) {
+            let selected;
+            try {
+                selected = await this.selectCircuitProbe(namespace, !hasRetrySourceError);
+            }
+            catch (err) {
+                if (hasRetrySourceError)
+                    throw retrySourceError;
+                throw err;
+            }
+            const { client, probe } = selected;
+            try {
+                const readable = client.stream(url, data, { signal });
+                return this.trackCircuitStream(namespace, client.host, client.port, probe, readable);
+            }
+            catch (err) {
+                this.recordFailure(namespace, client.host, client.port, err, probe);
+                probe.release();
+                if (remainingRetries > 0 && this.shouldRetryCircuitFailure(err)) {
+                    retrySourceError = err;
+                    hasRetrySourceError = true;
+                    remainingRetries--;
+                    continue;
+                }
+                throw err;
             }
-            throw err;
         }
     }
     async publish(topic, data) {
-        if (!this.registry)
-            throw new Error('Registry not found');
-        await this.registry.request('/-/declare', { topic, payload: data });
+        this.assertCanUsePubSub();
+        const snapshot = createPubSubPayloadSnapshot(topic, data);
+        const version = this.recordPublishedTopic(topic, snapshot);
+        await this.syncPublishedTopic(topic, { payload: snapshot, preserveRevision: false, version });
+        let refVersion = version;
         const ref = {
             update: async (payload) => {
-                if (!this.registry)
-                    throw new Error('Registry not found');
-                await this.registry.request('/-/topic/update', { topic, payload });
+                this.assertCanUsePubSub();
+                if (this.publishedTopicVersions.get(topic) !== refVersion)
+                    return ref;
+                const snapshot = createPubSubPayloadSnapshot(topic, payload);
+                const version = this.recordPublishedTopic(topic, snapshot);
+                refVersion = version;
+                await this.syncPublishedTopic(topic, { payload: snapshot, preserveRevision: false, version });
                 return ref;
             },
             unpublish: async () => {
-                if (!this.registry)
-                    throw new Error('Registry not found');
-                await this.registry.request('/-/undeclare', { topic });
+                this.assertCanUsePubSub();
+                if (this.publishedTopicVersions.get(topic) !== refVersion)
+                    return ref;
+                this.publishedTopics.delete(topic);
+                this.publishedTopicRevisions.delete(topic);
+                this.publishedTopicDirty.delete(topic);
+                this.publishedTopicVersions.delete(topic);
+                this.publishedTopicSignatures.delete(topic);
+                await this.syncUnpublishedTopic(topic);
                 return ref;
             }
         };
         return ref;
     }
-    /** 对同一 topic 重复 subscribe 是幂等的：第二次调用只返回 unsubscribe 函数，不会注册第二个 callback */
+    /**
+     * 对同一 topic 可多次 subscribe，各自独立回调。
+     * 传入同一个 callback 引用第二次调用时幂等返回 unsubscribe，不重复注册。
+     */
     async subscribe(topic, callback, isReconnect = false) {
-        if (!this.registry)
-            throw new Error('Registry not found');
+        this.assertCanUsePubSub();
         const fallback = async () => {
-            if (!this.registry)
-                throw new Error('Registry not found');
-            await this.registry.request('/-/unsubscribe', { topic });
             if (this.topics.has(topic)) {
-                const _callback = this.topics.get(topic);
-                this.events.off('topic:' + topic, _callback);
-                this.topics.delete(topic);
+                const callbacks = this.topics.get(topic);
+                callbacks.delete(callback);
+                this.events.off('topic:' + topic, callback);
+                if (callbacks.size === 0) {
+                    this.topics.delete(topic);
+                    await this.syncUnsubscribedTopic(topic);
+                }
             }
         };
-        if (this.topics.has(topic) && !isReconnect)
+        let localRegistered = false;
+        if (isReconnect) {
+            if (!this.registry) {
+                this.ensureRegistryReconnectScheduled();
+                return fallback;
+            }
+            await this.restoreSubscription(topic, callback, this.registry, true, false);
             return fallback;
-        const payload = await this.registry.request('/-/subscribe', { topic });
+        }
         if (!isReconnect) {
+            // 预分配 Set 必须在 await 之前；空 Set 残留无害，会在 unsubscribe / shutdown 时清理
+            if (!this.topics.has(topic)) {
+                this.topics.set(topic, new Set());
+            }
+            // 同一个 callback 引用已注册，幂等返回 unsubscribe，不重复发起订阅
+            if (this.topics.get(topic).has(callback))
+                return fallback;
+            this.topics.get(topic).add(callback);
             this.events.on('topic:' + topic, callback);
-            this.topics.set(topic, callback);
-            callback(payload);
+            localRegistered = true;
+        }
+        const registry = this.registry;
+        if (!registry) {
+            this.ensureRegistryReconnectScheduled();
+            return fallback;
+        }
+        const replayBaseVersion = this.topicUpdateVersions.get(topic) ?? 0;
+        let snapshot;
+        let synced = false;
+        await this.enqueueTopicSync(topic, async (registry) => {
+            if (!this.topics.get(topic)?.has(callback))
+                return;
+            snapshot = await registry.request('/-/subscribe', { topic }, this.registryRequestOptions());
+            synced = true;
+        });
+        if (!synced || !snapshot) {
+            return fallback;
+        }
+        try {
+            if (this.topics.get(topic)?.has(callback) &&
+                snapshot.hasData &&
+                (this.topicUpdateVersions.get(topic) ?? 0) === replayBaseVersion) {
+                await Promise.resolve(callback(snapshot.payload));
+            }
+        }
+        catch (err) {
+            if (localRegistered)
+                await this.rollbackLocalSubscription(topic, callback);
+            throw err;
         }
         return fallback;
     }