@cocorograph/hub-agent 0.8.44 → 0.8.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/codex-stream-bridge.mjs +349 -39
- package/src/main.mjs +52 -6
- package/src/usage.mjs +169 -31
- package/src/ws-client.mjs +136 -37
package/package.json
CHANGED
|
@@ -136,6 +136,14 @@ const STUCK_TURN_TTL_MS = readPositiveIntEnv(
|
|
|
136
136
|
60 * 60 * 1000,
|
|
137
137
|
)
|
|
138
138
|
|
|
139
|
+
// 子 turn の停止 handle 台帳を回収できる最短 age(既定 4h)。通常は保護 TTL(既定 1h)を
|
|
140
|
+
// 先に超えて protectionExpiredAtMs が立ち、その後も長時間 heartbeat が無い record だけを回収する。
|
|
141
|
+
// つまり保護 TTL 直後には停止可能性を残し、台帳だけが永久化することも避ける。設定で保護 TTL を
|
|
142
|
+
// 4h 以上へ延ばした場合も、protectionExpiredAtMs が立つまでは active handle を回収しない。
|
|
143
|
+
const SUBAGENT_INTERRUPT_RECORD_TTL_MS = 4 * 60 * 60 * 1000
|
|
144
|
+
/** 個別停止中に後発 descendant を拾う再走査回数。無制限 spawn に応答を占有されない上限。 */
|
|
145
|
+
const SUBAGENT_INTERRUPT_SCAN_ROUNDS = 3
|
|
146
|
+
|
|
139
147
|
// 案B (T06542): 死端末 GC の TTL。多端末購読で、応答を返さなくなった(クラッシュ / タブ強制終了で
|
|
140
148
|
// codex.detach を送れなかった)購読端末を購読集合から掃除する閾値。claude-stream-bridge.mjs の
|
|
141
149
|
// DEAD_TERMINAL_TTL_MS(既定 1h)と同型・同既定。stuck turn TTL とは別軸(あちらはターン固着、
|
|
@@ -774,6 +782,8 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
774
782
|
subagentInterruptRegistry: new Map(),
|
|
775
783
|
/** dynamicToolCall(spawn_agent) の item.id -> 表示メタデータ。 */
|
|
776
784
|
subagentSpawnMetadata: new Map(),
|
|
785
|
+
/** invalid background の同一 item 再送で warn を重複させない bounded set。 */
|
|
786
|
+
subagentInvalidBackgroundWarned: new Set(),
|
|
777
787
|
/** live subAgentActivity の spawn metadata miss を child thread/read で補完する
|
|
778
788
|
* single-flight 台帳。key = agentThreadId, value = 進行中 Promise。 */
|
|
779
789
|
subagentRoleReads: new Map(),
|
|
@@ -1668,8 +1678,12 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1668
1678
|
description:
|
|
1669
1679
|
metadata?.description || previous?.description || subagentFallbackName(agentPath),
|
|
1670
1680
|
prompt: metadata?.prompt || previous?.prompt || "",
|
|
1671
|
-
//
|
|
1672
|
-
|
|
1681
|
+
// metadata に明示 boolean があるときは後着値を権威とする。metadata miss は spawn の
|
|
1682
|
+
// background 省略既定と同じ true に倒し、明示 false だけが訂正する。
|
|
1683
|
+
background:
|
|
1684
|
+
typeof metadata?.background === "boolean"
|
|
1685
|
+
? metadata.background
|
|
1686
|
+
: previous?.background ?? true,
|
|
1673
1687
|
atMs: transitionAtMs,
|
|
1674
1688
|
startedAtMs:
|
|
1675
1689
|
phase === "started" && terminalSubagentPhase(previous?.phase)
|
|
@@ -1791,12 +1805,35 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1791
1805
|
if (!args) return
|
|
1792
1806
|
const agent = typeof args.agent_type === "string" ? args.agent_type.trim() : ""
|
|
1793
1807
|
const agentExplicit = agent.length > 0
|
|
1808
|
+
// 現行 Codex schema は background を送らず spawn は非同期で返るため、省略・非 boolean は
|
|
1809
|
+
// 背景扱いにする。invalid 値は本文を永続ログへ出さず、型と item id だけを記録する。
|
|
1810
|
+
const hasBackground = Object.prototype.hasOwnProperty.call(args, "background")
|
|
1811
|
+
const background = typeof args.background === "boolean" ? args.background : true
|
|
1812
|
+
if (hasBackground && typeof args.background !== "boolean") {
|
|
1813
|
+
if (!session.subagentInvalidBackgroundWarned.has(item.id)) {
|
|
1814
|
+
session.subagentInvalidBackgroundWarned.add(item.id)
|
|
1815
|
+
while (session.subagentInvalidBackgroundWarned.size > 256) {
|
|
1816
|
+
session.subagentInvalidBackgroundWarned.delete(
|
|
1817
|
+
session.subagentInvalidBackgroundWarned.values().next().value,
|
|
1818
|
+
)
|
|
1819
|
+
}
|
|
1820
|
+
const backgroundType = Array.isArray(args.background)
|
|
1821
|
+
? "array"
|
|
1822
|
+
: args.background === null
|
|
1823
|
+
? "null"
|
|
1824
|
+
: typeof args.background
|
|
1825
|
+
this.logger?.warn?.(
|
|
1826
|
+
{ item_id: item.id, background_type: backgroundType },
|
|
1827
|
+
"codex subagent spawn background must be boolean; defaulting to background",
|
|
1828
|
+
)
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1794
1831
|
const metadata = {
|
|
1795
1832
|
agent: agentExplicit ? agent : null,
|
|
1796
1833
|
agentExplicit,
|
|
1797
1834
|
description: typeof args.task_name === "string" && args.task_name ? args.task_name : null,
|
|
1798
1835
|
prompt: typeof args.message === "string" ? args.message : null,
|
|
1799
|
-
background
|
|
1836
|
+
background,
|
|
1800
1837
|
}
|
|
1801
1838
|
session.subagentSpawnMetadata.set(item.id, metadata)
|
|
1802
1839
|
while (session.subagentSpawnMetadata.size > 256) {
|
|
@@ -1892,32 +1929,52 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1892
1929
|
}
|
|
1893
1930
|
const { parentThreadId, ownerTurnId, ...mutableFields } = fields
|
|
1894
1931
|
Object.assign(record, mutableFields, { lastSeenAtMs: now })
|
|
1932
|
+
delete record.protectionExpiredAtMs
|
|
1895
1933
|
if (!record.parentThreadId && typeof parentThreadId === "string" && parentThreadId) {
|
|
1896
1934
|
record.parentThreadId = parentThreadId
|
|
1897
1935
|
}
|
|
1898
1936
|
if (!record.ownerTurnId && typeof ownerTurnId === "string" && ownerTurnId) {
|
|
1899
1937
|
record.ownerTurnId = ownerTurnId
|
|
1900
1938
|
}
|
|
1901
|
-
|
|
1939
|
+
this._sweepSubagentInterruptRegistry(session, now)
|
|
1940
|
+
return record
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
/** delta hot path。対象 record だけを更新し、全台帳 sweep/sort は行わない。 */
|
|
1944
|
+
_heartbeatSubagentInterruptRecord(session, threadId, now = Date.now()) {
|
|
1945
|
+
const record = session?.subagentInterruptRegistry?.get(threadId)
|
|
1946
|
+
if (!record) return null
|
|
1947
|
+
record.lastSeenAtMs = now
|
|
1948
|
+
delete record.protectionExpiredAtMs
|
|
1949
|
+
return record
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
/** lifecycle 更新時だけ、完了済みまたは保護失効済みの古い停止 record と上限超過を回収する。 */
|
|
1953
|
+
_sweepSubagentInterruptRegistry(session, now = Date.now()) {
|
|
1954
|
+
const registry = session?.subagentInterruptRegistry
|
|
1955
|
+
if (!registry) return
|
|
1956
|
+
const expiredBefore = now - SUBAGENT_INTERRUPT_RECORD_TTL_MS
|
|
1902
1957
|
for (const [id, candidate] of session.subagentInterruptRegistry) {
|
|
1903
|
-
|
|
1958
|
+
const evictable = candidate.turnId === null || candidate.protectionExpiredAtMs !== undefined
|
|
1959
|
+
if (evictable && candidate.lastSeenAtMs < expiredBefore) {
|
|
1904
1960
|
this._evictSubagentInterruptRecord(session, id, candidate)
|
|
1905
1961
|
}
|
|
1906
1962
|
}
|
|
1907
|
-
while (
|
|
1908
|
-
const oldest = [...
|
|
1909
|
-
.filter(([, candidate]) =>
|
|
1963
|
+
while (registry.size > 256) {
|
|
1964
|
+
const oldest = [...registry.entries()]
|
|
1965
|
+
.filter(([, candidate]) => (
|
|
1966
|
+
candidate.turnId === null || candidate.protectionExpiredAtMs !== undefined
|
|
1967
|
+
))
|
|
1910
1968
|
.sort((a, b) => a[1].lastSeenAtMs - b[1].lastSeenAtMs)[0]
|
|
1911
1969
|
if (!oldest) {
|
|
1912
1970
|
this.logger?.warn?.(
|
|
1913
|
-
{ size:
|
|
1971
|
+
{ size: registry.size, limit: 256 },
|
|
1914
1972
|
"codex subagent interrupt registry exceeds limit with all records active",
|
|
1915
1973
|
)
|
|
1916
1974
|
break
|
|
1917
1975
|
}
|
|
1918
1976
|
this._evictSubagentInterruptRecord(session, oldest[0], oldest[1])
|
|
1919
1977
|
}
|
|
1920
|
-
return record
|
|
1921
1978
|
}
|
|
1922
1979
|
|
|
1923
1980
|
_evictSubagentInterruptRecord(session, threadId, record) {
|
|
@@ -1927,25 +1984,50 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1927
1984
|
session.subagentInterruptRegistry.delete(threadId)
|
|
1928
1985
|
}
|
|
1929
1986
|
|
|
1987
|
+
/**
|
|
1988
|
+
* 子 record から root までの所有チェーンを辿る。cycle は呼び出し側が停止対象から除外する。
|
|
1989
|
+
* @returns {{ownerRootTurnId:string|null, hasBackgroundAncestor:boolean, lineage:string[], cycle:boolean}}
|
|
1990
|
+
*/
|
|
1930
1991
|
_subagentOwnership(session, record) {
|
|
1931
1992
|
const visited = new Set()
|
|
1993
|
+
const lineage = []
|
|
1932
1994
|
let current = record
|
|
1933
1995
|
let hasBackgroundAncestor = false
|
|
1934
1996
|
while (current) {
|
|
1935
|
-
if (visited.has(current.threadId))
|
|
1997
|
+
if (visited.has(current.threadId)) {
|
|
1998
|
+
return { ownerRootTurnId: null, hasBackgroundAncestor, lineage, cycle: true }
|
|
1999
|
+
}
|
|
1936
2000
|
visited.add(current.threadId)
|
|
2001
|
+
lineage.push(current.threadId)
|
|
2002
|
+
// background は表示台帳の省略既定を含む。親チェーン上の背景 root を一度検出したら、
|
|
2003
|
+
// capability 対応 browser の root interrupt では部分木全体を残す。
|
|
1937
2004
|
if (session.subagents.get(current.threadId)?.background === true) {
|
|
1938
2005
|
hasBackgroundAncestor = true
|
|
1939
2006
|
}
|
|
1940
2007
|
if (current.parentThreadId === session.threadId) {
|
|
1941
|
-
return {
|
|
2008
|
+
return {
|
|
2009
|
+
ownerRootTurnId: current.ownerTurnId || null,
|
|
2010
|
+
hasBackgroundAncestor,
|
|
2011
|
+
lineage,
|
|
2012
|
+
cycle: false,
|
|
2013
|
+
}
|
|
1942
2014
|
}
|
|
1943
2015
|
current = session.subagentInterruptRegistry.get(current.parentThreadId)
|
|
1944
2016
|
}
|
|
1945
|
-
return { ownerRootTurnId: null, hasBackgroundAncestor }
|
|
2017
|
+
return { ownerRootTurnId: null, hasBackgroundAncestor, lineage, cycle: false }
|
|
1946
2018
|
}
|
|
1947
2019
|
|
|
1948
|
-
async _interruptSubagent(
|
|
2020
|
+
async _interruptSubagent(
|
|
2021
|
+
session,
|
|
2022
|
+
record,
|
|
2023
|
+
targetRootTurnId,
|
|
2024
|
+
{
|
|
2025
|
+
allowBackground = false,
|
|
2026
|
+
requireRootOwnership = true,
|
|
2027
|
+
expectedDelegationId = null,
|
|
2028
|
+
expectedAncestorThreadId = null,
|
|
2029
|
+
} = {},
|
|
2030
|
+
) {
|
|
1949
2031
|
const displayed = session.subagents.get(record.threadId)
|
|
1950
2032
|
const backgroundKnown = displayed ? typeof displayed.background === "boolean" : false
|
|
1951
2033
|
const background = backgroundKnown ? displayed.background : null
|
|
@@ -1957,11 +2039,21 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1957
2039
|
const mismatch = parseTurnMismatch(reason)
|
|
1958
2040
|
const latestRecord = session.subagentInterruptRegistry.get(record.threadId)
|
|
1959
2041
|
const latestOwnership = latestRecord ? this._subagentOwnership(session, latestRecord) : null
|
|
2042
|
+
const latestDisplayed = session.subagents.get(record.threadId)
|
|
2043
|
+
const lineageStillMatches = !expectedAncestorThreadId || (
|
|
2044
|
+
latestOwnership &&
|
|
2045
|
+
!latestOwnership.cycle &&
|
|
2046
|
+
latestOwnership.lineage.includes(expectedAncestorThreadId)
|
|
2047
|
+
)
|
|
1960
2048
|
if (
|
|
1961
2049
|
mismatch &&
|
|
1962
2050
|
latestRecord?.turnId === mismatch.found &&
|
|
1963
|
-
|
|
1964
|
-
|
|
2051
|
+
(!expectedDelegationId || latestDisplayed?.delegationId === expectedDelegationId) &&
|
|
2052
|
+
lineageStillMatches &&
|
|
2053
|
+
(!requireRootOwnership || (
|
|
2054
|
+
latestOwnership?.ownerRootTurnId === targetRootTurnId &&
|
|
2055
|
+
(allowBackground || !latestOwnership.hasBackgroundAncestor)
|
|
2056
|
+
))
|
|
1965
2057
|
) {
|
|
1966
2058
|
try {
|
|
1967
2059
|
await session.client.request("turn/interrupt", {
|
|
@@ -1986,6 +2078,15 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1986
2078
|
}
|
|
1987
2079
|
}
|
|
1988
2080
|
}
|
|
2081
|
+
if (mismatch && latestRecord?.turnId === mismatch.found && !lineageStillMatches) {
|
|
2082
|
+
this.logger?.warn?.(
|
|
2083
|
+
{
|
|
2084
|
+
thread_id: record.threadId,
|
|
2085
|
+
expected_ancestor_thread_id: expectedAncestorThreadId,
|
|
2086
|
+
},
|
|
2087
|
+
"codex subagent interrupt retry skipped after ownership changed",
|
|
2088
|
+
)
|
|
2089
|
+
}
|
|
1989
2090
|
return {
|
|
1990
2091
|
threadId: record.threadId,
|
|
1991
2092
|
turnId: record.turnId,
|
|
@@ -1997,6 +2098,111 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
1997
2098
|
}
|
|
1998
2099
|
}
|
|
1999
2100
|
|
|
2101
|
+
/**
|
|
2102
|
+
* 対象子と active descendants を停止する。対象 RPC 待ちや子停止中に増えた descendant も
|
|
2103
|
+
* 最大3ラウンド再走査する。上限後の未処理 active turn は skipped へ明示し、無限 spawn に
|
|
2104
|
+
* WebSocket 応答を占有されないようにする。
|
|
2105
|
+
* @returns {Promise<object>} targetOutcome/targetReason は対象自身、outcome は部分木全体の結果。
|
|
2106
|
+
*/
|
|
2107
|
+
async interruptSubagent({ stream_id, delegation_id, agent_thread_id }) {
|
|
2108
|
+
const session = this._require(stream_id)
|
|
2109
|
+
this._touch(session, stream_id)
|
|
2110
|
+
const displayed = session.subagents.get(agent_thread_id)
|
|
2111
|
+
if (!displayed || displayed.delegationId !== delegation_id) {
|
|
2112
|
+
return { outcome: "failed", reason: "subagent_not_found", children: [], skipped: [] }
|
|
2113
|
+
}
|
|
2114
|
+
const record = session.subagentInterruptRegistry.get(agent_thread_id)
|
|
2115
|
+
const result = record?.turnId
|
|
2116
|
+
? await this._interruptSubagent(session, { ...record }, null, {
|
|
2117
|
+
allowBackground: true,
|
|
2118
|
+
requireRootOwnership: false,
|
|
2119
|
+
expectedDelegationId: delegation_id,
|
|
2120
|
+
})
|
|
2121
|
+
: { outcome: "no_active_turn", reason: "subagent_not_active" }
|
|
2122
|
+
const children = []
|
|
2123
|
+
const processedTurns = new Set()
|
|
2124
|
+
const warnedCycleThreads = new Set()
|
|
2125
|
+
const turnKey = (candidate) => `${candidate.threadId}\u0000${candidate.turnId}`
|
|
2126
|
+
const activeDescendants = () => [...session.subagentInterruptRegistry.values()]
|
|
2127
|
+
.filter((candidate) => {
|
|
2128
|
+
if (!candidate?.turnId || candidate.threadId === agent_thread_id) return false
|
|
2129
|
+
const ownership = this._subagentOwnership(session, candidate)
|
|
2130
|
+
if (ownership.cycle) {
|
|
2131
|
+
if (!warnedCycleThreads.has(candidate.threadId)) {
|
|
2132
|
+
warnedCycleThreads.add(candidate.threadId)
|
|
2133
|
+
this.logger?.warn?.(
|
|
2134
|
+
{ thread_id: candidate.threadId },
|
|
2135
|
+
"codex subagent descendant skipped because ownership chain is cyclic",
|
|
2136
|
+
)
|
|
2137
|
+
}
|
|
2138
|
+
return false
|
|
2139
|
+
}
|
|
2140
|
+
return ownership.lineage.includes(agent_thread_id)
|
|
2141
|
+
})
|
|
2142
|
+
|
|
2143
|
+
for (let round = 0; round < SUBAGENT_INTERRUPT_SCAN_ROUNDS; round += 1) {
|
|
2144
|
+
const descendants = activeDescendants()
|
|
2145
|
+
.filter((candidate) => !processedTurns.has(turnKey(candidate)))
|
|
2146
|
+
.map((candidate) => ({ ...candidate }))
|
|
2147
|
+
if (descendants.length === 0) break
|
|
2148
|
+
for (const descendant of descendants) {
|
|
2149
|
+
const key = turnKey(descendant)
|
|
2150
|
+
// 同じ round の先行 RPC 待ち中にも lineage/turn は動き得るため、送信直前にも live 台帳で確認する。
|
|
2151
|
+
const current = session.subagentInterruptRegistry.get(descendant.threadId)
|
|
2152
|
+
const ownership = current ? this._subagentOwnership(session, current) : null
|
|
2153
|
+
if (
|
|
2154
|
+
!current?.turnId ||
|
|
2155
|
+
current.turnId !== descendant.turnId ||
|
|
2156
|
+
ownership?.cycle ||
|
|
2157
|
+
!ownership?.lineage.includes(agent_thread_id)
|
|
2158
|
+
) {
|
|
2159
|
+
this.logger?.warn?.(
|
|
2160
|
+
{ thread_id: descendant.threadId, round: round + 1 },
|
|
2161
|
+
"codex subagent descendant skipped after ownership changed before interrupt",
|
|
2162
|
+
)
|
|
2163
|
+
continue
|
|
2164
|
+
}
|
|
2165
|
+
processedTurns.add(key)
|
|
2166
|
+
const childResult = await this._interruptSubagent(session, { ...current }, null, {
|
|
2167
|
+
allowBackground: true,
|
|
2168
|
+
requireRootOwnership: false,
|
|
2169
|
+
expectedAncestorThreadId: agent_thread_id,
|
|
2170
|
+
})
|
|
2171
|
+
children.push(childResult)
|
|
2172
|
+
if (childResult?.threadId && childResult?.turnId) {
|
|
2173
|
+
processedTurns.add(turnKey(childResult))
|
|
2174
|
+
}
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
const targetOutcome = result.outcome
|
|
2178
|
+
const descendantInterrupted = children.some((child) => child.outcome === "interrupted")
|
|
2179
|
+
const outcome =
|
|
2180
|
+
targetOutcome === "no_active_turn" && descendantInterrupted
|
|
2181
|
+
? "interrupted"
|
|
2182
|
+
: targetOutcome
|
|
2183
|
+
const response = {
|
|
2184
|
+
...result,
|
|
2185
|
+
outcome,
|
|
2186
|
+
targetOutcome,
|
|
2187
|
+
delegationId: displayed.delegationId,
|
|
2188
|
+
agentThreadId: displayed.agentThreadId,
|
|
2189
|
+
children,
|
|
2190
|
+
skipped: activeDescendants()
|
|
2191
|
+
.filter((candidate) => !processedTurns.has(turnKey(candidate)))
|
|
2192
|
+
.map((candidate) => ({
|
|
2193
|
+
threadId: candidate.threadId,
|
|
2194
|
+
turnId: candidate.turnId,
|
|
2195
|
+
reason: "spawned_during_interrupt",
|
|
2196
|
+
})),
|
|
2197
|
+
}
|
|
2198
|
+
// 部分木として成功した応答に failure 用 reason を残さない。対象単体の理由は別軸で保持する。
|
|
2199
|
+
if (outcome === "interrupted" && result.reason) {
|
|
2200
|
+
delete response.reason
|
|
2201
|
+
response.targetReason = result.reason
|
|
2202
|
+
}
|
|
2203
|
+
return response
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2000
2206
|
_subagentSnapshots(session, history = session?.historyItems) {
|
|
2001
2207
|
const items = Array.isArray(history) ? history : []
|
|
2002
2208
|
const startedIndexByThread = new Map()
|
|
@@ -2077,21 +2283,47 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
2077
2283
|
{ agentPath: record.agentPath, metadata, atMs: lastTurnAtMs },
|
|
2078
2284
|
{ emit: false },
|
|
2079
2285
|
)
|
|
2080
|
-
} else if (
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2286
|
+
} else if (thread?.status?.type === "active") {
|
|
2287
|
+
if (
|
|
2288
|
+
lastTurn?.status === "inProgress" &&
|
|
2289
|
+
typeof lastTurn.id === "string" &&
|
|
2290
|
+
lastTurn.id
|
|
2291
|
+
) {
|
|
2292
|
+
this._setSubagentPhase(
|
|
2293
|
+
session,
|
|
2294
|
+
record.agentThreadId,
|
|
2295
|
+
"started",
|
|
2296
|
+
{
|
|
2297
|
+
agentPath: record.agentPath,
|
|
2298
|
+
metadata,
|
|
2299
|
+
atMs: record.startedAtMs,
|
|
2300
|
+
},
|
|
2301
|
+
{ emit: false },
|
|
2302
|
+
)
|
|
2303
|
+
this._touchSubagentInterruptRecord(session, record.agentThreadId, {
|
|
2304
|
+
turnId: lastTurn.id,
|
|
2305
|
+
startedAtMs: lastTurnAtMs,
|
|
2306
|
+
})
|
|
2307
|
+
} else {
|
|
2308
|
+
// active 表示なのに停止 RPC に必要な turn ID が無い。running と表示すると停止可能な
|
|
2309
|
+
// capability を偽るため、cold snapshot は正直な terminal へ倒す。
|
|
2310
|
+
this._setSubagentPhase(
|
|
2311
|
+
session,
|
|
2312
|
+
record.agentThreadId,
|
|
2313
|
+
"failed",
|
|
2314
|
+
{ agentPath: record.agentPath, metadata, atMs: lastTurnAtMs },
|
|
2315
|
+
{ emit: false },
|
|
2316
|
+
)
|
|
2317
|
+
this.logger?.warn?.(
|
|
2318
|
+
{
|
|
2319
|
+
thread_id: session.threadId,
|
|
2320
|
+
subagent_thread_id: record.agentThreadId,
|
|
2321
|
+
child_status: thread?.status?.type || null,
|
|
2322
|
+
last_turn_status: lastTurn?.status || null,
|
|
2323
|
+
},
|
|
2324
|
+
"codex active subagent resume lacked an interruptible turn id; marked failed",
|
|
2325
|
+
)
|
|
2326
|
+
}
|
|
2095
2327
|
} else {
|
|
2096
2328
|
session.subagents.delete(record.agentThreadId)
|
|
2097
2329
|
}
|
|
@@ -2168,6 +2400,14 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
2168
2400
|
// ※ hub-agent 再起動 (cold resume) では historyItems が揮発するため青バブルへ戻る (claude の
|
|
2169
2401
|
// jsonl at-rest と同じ許容範囲。source 永続マップは TTL/ID 再利用の複雑さに見合わないため不採用)。
|
|
2170
2402
|
const eventThreadId = msg?.params?.threadId
|
|
2403
|
+
if (
|
|
2404
|
+
typeof eventThreadId === "string" &&
|
|
2405
|
+
eventThreadId !== session.threadId &&
|
|
2406
|
+
session.subagentInterruptRegistry.has(eventThreadId)
|
|
2407
|
+
) {
|
|
2408
|
+
// lifecycle 以外の message/reasoning/tool delta も子 turn の生存証跡。root 通知は更新しない。
|
|
2409
|
+
this._heartbeatSubagentInterruptRecord(session, eventThreadId)
|
|
2410
|
+
}
|
|
2171
2411
|
const taggedParams = this._tagInjectSourceOnItem(session, msg)
|
|
2172
2412
|
let effectiveParams = taggedParams || msg.params
|
|
2173
2413
|
const childRecord =
|
|
@@ -2350,6 +2590,7 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
2350
2590
|
|
|
2351
2591
|
_onClientExit(session, info) {
|
|
2352
2592
|
if (session.dead) return
|
|
2593
|
+
this._markRunningSubagentsKilled(session)
|
|
2353
2594
|
session.dead = true
|
|
2354
2595
|
// round3: app-server が死んだら barrier を解除し queue を reject する(promise 宙吊り防止)。
|
|
2355
2596
|
this._settleTurnActiveWaiter(session)
|
|
@@ -2614,15 +2855,17 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
2614
2855
|
|
|
2615
2856
|
/**
|
|
2616
2857
|
* @returns {Promise<
|
|
2617
|
-
*
|
|
2618
|
-
*
|
|
2619
|
-
*
|
|
2858
|
+
* ({children: object[], skipped: object[]} & (
|
|
2859
|
+
* | { outcome: "interrupted", turnId: string, corrected: boolean }
|
|
2860
|
+
* | { outcome: "no_active_turn" }
|
|
2861
|
+
* | { outcome: "failed", reason: string }
|
|
2862
|
+
* ))
|
|
2620
2863
|
* >}
|
|
2621
2864
|
*/
|
|
2622
|
-
async interrupt({ stream_id }) {
|
|
2865
|
+
async interrupt({ stream_id, preserve_background_subagents = false }) {
|
|
2623
2866
|
const session = this._require(stream_id)
|
|
2624
2867
|
this._touch(session, stream_id)
|
|
2625
|
-
if (!session.activeTurnId) return { outcome: "no_active_turn", children: [] }
|
|
2868
|
+
if (!session.activeTurnId) return { outcome: "no_active_turn", children: [], skipped: [] }
|
|
2626
2869
|
|
|
2627
2870
|
const turnId = session.activeTurnId
|
|
2628
2871
|
let attemptedRootTurnId = turnId
|
|
@@ -2665,13 +2908,35 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
2665
2908
|
const children = []
|
|
2666
2909
|
const targetRootTurnId = attemptedRootTurnId
|
|
2667
2910
|
const interruptSnapshot = [...session.subagentInterruptRegistry.values()].map((record) => ({ ...record }))
|
|
2911
|
+
// skipped は「今回の owner turn の停止候補」ではなく、応答時点で実際に継続中の
|
|
2912
|
+
// root 直下 background 一覧。frontend の「継続中」表示と一致させるため owner に依存しない。
|
|
2913
|
+
const skipped = preserve_background_subagents === true
|
|
2914
|
+
? interruptSnapshot.flatMap((record) => {
|
|
2915
|
+
const displayed = session.subagents.get(record.threadId)
|
|
2916
|
+
if (
|
|
2917
|
+
!record.turnId ||
|
|
2918
|
+
record.parentThreadId !== session.threadId ||
|
|
2919
|
+
displayed?.background !== true
|
|
2920
|
+
) return []
|
|
2921
|
+
return [{
|
|
2922
|
+
threadId: record.threadId,
|
|
2923
|
+
delegationId: displayed.delegationId,
|
|
2924
|
+
reason: "background",
|
|
2925
|
+
}]
|
|
2926
|
+
})
|
|
2927
|
+
: []
|
|
2668
2928
|
for (const record of interruptSnapshot) {
|
|
2669
2929
|
if (!record.turnId) continue
|
|
2670
2930
|
const ownership = this._subagentOwnership(session, record)
|
|
2671
|
-
if (
|
|
2672
|
-
|
|
2931
|
+
if (preserve_background_subagents === true && ownership.hasBackgroundAncestor) {
|
|
2932
|
+
continue
|
|
2933
|
+
}
|
|
2934
|
+
if (ownership.ownerRootTurnId !== targetRootTurnId) continue
|
|
2935
|
+
children.push(await this._interruptSubagent(session, { ...record }, targetRootTurnId, {
|
|
2936
|
+
allowBackground: preserve_background_subagents !== true,
|
|
2937
|
+
}))
|
|
2673
2938
|
}
|
|
2674
|
-
return { ...result, children }
|
|
2939
|
+
return { ...result, children, skipped }
|
|
2675
2940
|
}
|
|
2676
2941
|
|
|
2677
2942
|
/** サーバー発の承認/入力要求への応答。result の shape は method ごとに呼び出し側が組み立てる。 */
|
|
@@ -3298,13 +3563,57 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
3298
3563
|
// ⚠️ activeTurnId の判定より **前** に置くこと。背景コマンドが残るのは turn/completed の
|
|
3299
3564
|
// 後(= activeTurnId が null に戻った後)なので、下の early return を通すと届かない。
|
|
3300
3565
|
if (this._hasLiveExec(session, now)) return true
|
|
3566
|
+
if (this._hasLiveSubagentTurn(session, now)) return true
|
|
3301
3567
|
if (!session.activeTurnId) return false
|
|
3302
3568
|
return now - (session.lastActivityMs ?? now) <= this._stuckTurnTtlMs
|
|
3303
3569
|
}
|
|
3304
3570
|
|
|
3571
|
+
_hasLiveSubagentTurn(session, now = Date.now()) {
|
|
3572
|
+
const registry = session?.subagentInterruptRegistry
|
|
3573
|
+
if (!registry || registry.size === 0) return false
|
|
3574
|
+
let active = false
|
|
3575
|
+
for (const record of registry.values()) {
|
|
3576
|
+
if (!record?.turnId) continue
|
|
3577
|
+
const elapsed = now - (record.lastSeenAtMs ?? 0)
|
|
3578
|
+
if (elapsed <= this._stuckTurnTtlMs) {
|
|
3579
|
+
active = true
|
|
3580
|
+
continue
|
|
3581
|
+
}
|
|
3582
|
+
// TTL は warm-session 保護だけを失効させる。停止 RPC に必要な turnId は破壊しない。
|
|
3583
|
+
// 長時間無出力の正常な子でも、明示停止を受けたときに同じ handle を利用できるようにする。
|
|
3584
|
+
if (record.protectionExpiredAtMs === undefined) {
|
|
3585
|
+
record.protectionExpiredAtMs = now
|
|
3586
|
+
this.logger?.warn?.(
|
|
3587
|
+
{
|
|
3588
|
+
stream_id: session.stream_id,
|
|
3589
|
+
thread_id: record.threadId,
|
|
3590
|
+
turn_id: record.turnId,
|
|
3591
|
+
elapsed_ms: elapsed,
|
|
3592
|
+
ttl_ms: this._stuckTurnTtlMs,
|
|
3593
|
+
},
|
|
3594
|
+
"codex subagent turn protection expired",
|
|
3595
|
+
)
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
return active
|
|
3599
|
+
}
|
|
3600
|
+
|
|
3601
|
+
_markRunningSubagentsKilled(session, atMs = Date.now()) {
|
|
3602
|
+
for (const record of session?.subagents?.values?.() || []) {
|
|
3603
|
+
if (record.phase !== "started") continue
|
|
3604
|
+
this._setSubagentPhase(
|
|
3605
|
+
session,
|
|
3606
|
+
record.agentThreadId,
|
|
3607
|
+
"killed",
|
|
3608
|
+
{ agentPath: record.agentPath, atMs },
|
|
3609
|
+
)
|
|
3610
|
+
}
|
|
3611
|
+
}
|
|
3612
|
+
|
|
3305
3613
|
/** 実際に app-server を止めてセッションを撤去し exit を emit する(reconcile 共通処理)。 */
|
|
3306
3614
|
_reapSession(session, now, reason) {
|
|
3307
3615
|
if (session.dead) return
|
|
3616
|
+
this._markRunningSubagentsKilled(session, now)
|
|
3308
3617
|
session.dead = true
|
|
3309
3618
|
// round3: reap した session の barrier を解除し queue を reject する(promise 宙吊り防止)。
|
|
3310
3619
|
this._settleTurnActiveWaiter(session)
|
|
@@ -3492,6 +3801,7 @@ export class CodexStreamBridge extends EventEmitter {
|
|
|
3492
3801
|
shutdown() {
|
|
3493
3802
|
// 案B: distinct session で走査(多対一 Map の重複 stop を避ける)。
|
|
3494
3803
|
for (const session of this._liveSessions()) {
|
|
3804
|
+
this._markRunningSubagentsKilled(session)
|
|
3495
3805
|
session.dead = true
|
|
3496
3806
|
// round3: shutdown で barrier を解除し queue 入力を reject(promise 宙吊り防止)。
|
|
3497
3807
|
this._settleTurnActiveWaiter(session)
|
package/src/main.mjs
CHANGED
|
@@ -2123,6 +2123,9 @@ export function buildCodexReadyPayload(streamId, info, queueState) {
|
|
|
2123
2123
|
thread_status: info?.threadStatus || null,
|
|
2124
2124
|
last_turn_status: info?.lastTurnStatus || null,
|
|
2125
2125
|
queue_state: queueState,
|
|
2126
|
+
// Browser は旧 hub-agent との混在期間、この明示 capability が true のときだけ
|
|
2127
|
+
// 子 thread の個別停止 UI を有効化する。省略は未対応を意味する。
|
|
2128
|
+
supports_subagent_interrupt: true,
|
|
2126
2129
|
}
|
|
2127
2130
|
if (Array.isArray(info?.subagents)) {
|
|
2128
2131
|
payload.subagents = info.subagents.map((subagent) => ({
|
|
@@ -3249,26 +3252,69 @@ export function handleCodexChatInput(msg, ctx) {
|
|
|
3249
3252
|
return p
|
|
3250
3253
|
}
|
|
3251
3254
|
|
|
3255
|
+
/**
|
|
3256
|
+
* codex.interrupt を root / individual へ振り分ける。
|
|
3257
|
+
* - delegation_id と agent_thread_id が非空文字列なら individual(request_id 必須・echo する)
|
|
3258
|
+
* - 両 ID が欠落または null なら root(現 frontend 契約。request_id は相関に使わず常に null)
|
|
3259
|
+
* - 片方だけ、または非 null の不正値が混ざれば individual ID validation error
|
|
3260
|
+
*/
|
|
3252
3261
|
export async function handleCodexInterrupt(msg, ctx) {
|
|
3253
3262
|
let result
|
|
3254
|
-
|
|
3255
|
-
|
|
3263
|
+
const hasDelegationId = typeof msg.delegation_id === "string" && msg.delegation_id.length > 0
|
|
3264
|
+
const hasAgentThreadId = typeof msg.agent_thread_id === "string" && msg.agent_thread_id.length > 0
|
|
3265
|
+
const hasEitherId = msg.delegation_id != null || msg.agent_thread_id != null
|
|
3266
|
+
const isSubagentInterrupt = hasDelegationId && hasAgentThreadId
|
|
3267
|
+
const hasRequestId = typeof msg.request_id === "string" && msg.request_id.length > 0
|
|
3268
|
+
if (hasEitherId && !(hasDelegationId && hasAgentThreadId)) {
|
|
3269
|
+
result = { outcome: "failed", reason: "invalid_subagent_interrupt_ids", children: [], skipped: [] }
|
|
3270
|
+
} else if (isSubagentInterrupt && !hasRequestId) {
|
|
3271
|
+
result = { outcome: "failed", reason: "invalid_subagent_interrupt_request_id", children: [], skipped: [] }
|
|
3272
|
+
} else if (!ctx.codexBridge) {
|
|
3273
|
+
result = { outcome: "failed", reason: "bridge_unavailable", children: [], skipped: [] }
|
|
3256
3274
|
} else {
|
|
3257
3275
|
try {
|
|
3258
|
-
|
|
3259
|
-
|
|
3276
|
+
const bridgeResult = hasDelegationId
|
|
3277
|
+
? await ctx.codexBridge.interruptSubagent({
|
|
3278
|
+
stream_id: msg.stream_id,
|
|
3279
|
+
delegation_id: msg.delegation_id,
|
|
3280
|
+
agent_thread_id: msg.agent_thread_id,
|
|
3281
|
+
})
|
|
3282
|
+
: await ctx.codexBridge.interrupt({
|
|
3283
|
+
stream_id: msg.stream_id,
|
|
3284
|
+
preserve_background_subagents: msg.preserve_background_subagents === true,
|
|
3285
|
+
})
|
|
3286
|
+
const validOutcome = ["interrupted", "failed", "no_active_turn"].includes(
|
|
3287
|
+
bridgeResult?.outcome,
|
|
3288
|
+
)
|
|
3289
|
+
if (!bridgeResult || typeof bridgeResult !== "object" || Array.isArray(bridgeResult) || !validOutcome) {
|
|
3290
|
+
ctx.logger?.warn?.(
|
|
3291
|
+
{
|
|
3292
|
+
bridge_result_type: Array.isArray(bridgeResult) ? "array" : typeof bridgeResult,
|
|
3293
|
+
bridge_outcome: typeof bridgeResult?.outcome === "string" ? bridgeResult.outcome : null,
|
|
3294
|
+
},
|
|
3295
|
+
"codex.interrupt bridge returned an invalid result",
|
|
3296
|
+
)
|
|
3297
|
+
result = { outcome: "failed", reason: "invalid_bridge_result", children: [], skipped: [] }
|
|
3298
|
+
} else {
|
|
3299
|
+
result = {
|
|
3300
|
+
...bridgeResult,
|
|
3301
|
+
children: Array.isArray(bridgeResult.children) ? bridgeResult.children : [],
|
|
3302
|
+
skipped: Array.isArray(bridgeResult.skipped) ? bridgeResult.skipped : [],
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
if (result.outcome === "failed" && result.reason !== "invalid_bridge_result") {
|
|
3260
3306
|
ctx.logger?.warn?.({ err: result.reason }, "codex.interrupt failed")
|
|
3261
3307
|
}
|
|
3262
3308
|
} catch (err) {
|
|
3263
3309
|
const reason = err?.message || String(err)
|
|
3264
3310
|
ctx.logger?.warn?.({ err: reason }, "codex.interrupt failed")
|
|
3265
|
-
result = { outcome: "failed", reason, children: [] }
|
|
3311
|
+
result = { outcome: "failed", reason, children: [], skipped: [] }
|
|
3266
3312
|
}
|
|
3267
3313
|
}
|
|
3268
3314
|
ctx.client.send({
|
|
3269
3315
|
type: "codex.interrupt.response",
|
|
3270
3316
|
stream_id: msg.stream_id,
|
|
3271
|
-
request_id: msg.request_id
|
|
3317
|
+
request_id: isSubagentInterrupt && hasRequestId ? msg.request_id : null,
|
|
3272
3318
|
...result,
|
|
3273
3319
|
})
|
|
3274
3320
|
}
|
package/src/usage.mjs
CHANGED
|
@@ -1482,6 +1482,41 @@ function threadIdFromRolloutFilename(fp) {
|
|
|
1482
1482
|
const CODEX_ROLLOUT_HEAD_CHUNK_BYTES = 32 * 1024
|
|
1483
1483
|
const CODEX_ROLLOUT_PREVIEW_MAX_BYTES = 1024 * 1024
|
|
1484
1484
|
|
|
1485
|
+
// sessionsDir ごとの派生インデックス。rollout 自体が正本なので永続化は不要。
|
|
1486
|
+
// _codexTokenCountMemo と同じ mtime+size 無効化規約で、変更ファイルだけを再解析する。
|
|
1487
|
+
// 共通 helper 化は usage 集計側への回帰範囲を広げるため、一覧用 index として独立させる。
|
|
1488
|
+
const _codexThreadListIndex = new Map() // base → Map<fp, {mtimeMs,size,lastSeenAt,meta}>
|
|
1489
|
+
const _codexThreadListLastWarnAt = new Map() // base → wall-clock ms
|
|
1490
|
+
const CODEX_THREAD_LIST_LOG_INTERVAL_MS = 5 * 60 * 1000
|
|
1491
|
+
const CODEX_THREAD_LIST_STALE_ENTRY_MS = 24 * 60 * 60 * 1000
|
|
1492
|
+
const CODEX_ROLLOUT_PARSE_IO_ERROR = Symbol("codex-rollout-parse-io-error")
|
|
1493
|
+
|
|
1494
|
+
function sameCodexThreadListFingerprint(a, b) {
|
|
1495
|
+
return a?.mtimeMs === b?.mtimeMs && a?.size === b?.size
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
function codexThreadListMetaRank(meta) {
|
|
1499
|
+
if (meta?.preview_complete && meta.cwd && meta.preview) return 3
|
|
1500
|
+
if (meta?.preview_complete) return 2
|
|
1501
|
+
return meta ? 1 : 0
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
// observed 後に別呼び出しが新 fingerprint / complete meta を置いた場合は降格させない。
|
|
1505
|
+
function commitCodexThreadListEntry(index, fp, observed, next) {
|
|
1506
|
+
const current = index.get(fp)
|
|
1507
|
+
if (current === observed) {
|
|
1508
|
+
index.set(fp, next)
|
|
1509
|
+
return
|
|
1510
|
+
}
|
|
1511
|
+
if (!current || !sameCodexThreadListFingerprint(current, next)) return
|
|
1512
|
+
const meta =
|
|
1513
|
+
codexThreadListMetaRank(next.meta) > codexThreadListMetaRank(current.meta)
|
|
1514
|
+
? next.meta
|
|
1515
|
+
: current.meta
|
|
1516
|
+
const merged = { ...current, lastSeenAt: Math.max(current.lastSeenAt, next.lastSeenAt), meta }
|
|
1517
|
+
index.set(fp, merged)
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1485
1520
|
/**
|
|
1486
1521
|
* rollout の先頭から一覧用メタ (thread_id / cwd / preview) を取り出す。
|
|
1487
1522
|
*
|
|
@@ -1492,9 +1527,9 @@ const CODEX_ROLLOUT_PREVIEW_MAX_BYTES = 1024 * 1024
|
|
|
1492
1527
|
*
|
|
1493
1528
|
* @param {string} fp
|
|
1494
1529
|
* @param {string} expectedCwd cwd-scoped 一覧の対象。別 cwd は session_meta 読み取り後に早期終了。
|
|
1495
|
-
* @returns {Promise<{thread_id:string,cwd:string,preview:string}|null>}
|
|
1530
|
+
* @returns {Promise<{thread_id:string,cwd:string,preview:string,preview_complete:boolean}|null|symbol>}
|
|
1496
1531
|
*/
|
|
1497
|
-
async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
1532
|
+
async function parseCodexRolloutHead(fp, expectedCwd = "", testHooks = null) {
|
|
1498
1533
|
let thread_id = threadIdFromRolloutFilename(fp)
|
|
1499
1534
|
let cwd = ""
|
|
1500
1535
|
let preview = ""
|
|
@@ -1502,6 +1537,7 @@ async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
|
1502
1537
|
const decoder = new StringDecoder("utf8")
|
|
1503
1538
|
let pending = ""
|
|
1504
1539
|
let offset = 0
|
|
1540
|
+
let result
|
|
1505
1541
|
|
|
1506
1542
|
const parseLine = (line) => {
|
|
1507
1543
|
if (!line) return
|
|
@@ -1528,8 +1564,13 @@ async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
|
1528
1564
|
}
|
|
1529
1565
|
|
|
1530
1566
|
try {
|
|
1567
|
+
if (testHooks?.beforeHeaderOpen) await testHooks.beforeHeaderOpen(fp)
|
|
1568
|
+
testHooks?.onHeaderOpen?.(fp)
|
|
1569
|
+
if (testHooks?.failHeaderOpen?.(fp)) {
|
|
1570
|
+
throw Object.assign(new Error("injected header open failure"), { code: "EIO" })
|
|
1571
|
+
}
|
|
1531
1572
|
fh = await fs.open(fp, "r")
|
|
1532
|
-
while (offset < CODEX_ROLLOUT_PREVIEW_MAX_BYTES) {
|
|
1573
|
+
scan: while (offset < CODEX_ROLLOUT_PREVIEW_MAX_BYTES) {
|
|
1533
1574
|
const readSize = Math.min(
|
|
1534
1575
|
CODEX_ROLLOUT_HEAD_CHUNK_BYTES,
|
|
1535
1576
|
CODEX_ROLLOUT_PREVIEW_MAX_BYTES - offset,
|
|
@@ -1550,12 +1591,14 @@ async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
|
1550
1591
|
pending = pending.slice(newline + 1)
|
|
1551
1592
|
parseLine(line)
|
|
1552
1593
|
// session_meta は先頭行にある。別 workspace の候補に対して巨大な初期コンテキストを
|
|
1553
|
-
//
|
|
1594
|
+
// 読み進めないことで、初回の派生インデックス構築コストを抑える。
|
|
1554
1595
|
if (cwd && expectedCwd && cwd !== expectedCwd) {
|
|
1555
|
-
|
|
1596
|
+
result = thread_id ? { thread_id, cwd, preview, preview_complete: false } : null
|
|
1597
|
+
break scan
|
|
1556
1598
|
}
|
|
1557
1599
|
if (cwd && preview) {
|
|
1558
|
-
|
|
1600
|
+
result = thread_id ? { thread_id, cwd, preview, preview_complete: true } : null
|
|
1601
|
+
break scan
|
|
1559
1602
|
}
|
|
1560
1603
|
}
|
|
1561
1604
|
|
|
@@ -1567,8 +1610,11 @@ async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
|
1567
1610
|
break
|
|
1568
1611
|
}
|
|
1569
1612
|
}
|
|
1613
|
+
if (result === undefined) {
|
|
1614
|
+
result = thread_id ? { thread_id, cwd, preview, preview_complete: true } : null
|
|
1615
|
+
}
|
|
1570
1616
|
} catch {
|
|
1571
|
-
|
|
1617
|
+
result = CODEX_ROLLOUT_PARSE_IO_ERROR
|
|
1572
1618
|
} finally {
|
|
1573
1619
|
try {
|
|
1574
1620
|
await fh?.close()
|
|
@@ -1577,20 +1623,22 @@ async function parseCodexRolloutHead(fp, expectedCwd = "") {
|
|
|
1577
1623
|
}
|
|
1578
1624
|
}
|
|
1579
1625
|
|
|
1580
|
-
if (
|
|
1581
|
-
|
|
1626
|
+
if (result !== CODEX_ROLLOUT_PARSE_IO_ERROR && testHooks?.afterHeaderParse) {
|
|
1627
|
+
await testHooks.afterHeaderParse(fp, result)
|
|
1628
|
+
}
|
|
1629
|
+
return result
|
|
1582
1630
|
}
|
|
1583
1631
|
|
|
1584
1632
|
/**
|
|
1585
1633
|
* cwd 配下の Codex thread 一覧を rollout の disk 走査だけで作る (app-server を spawn しない)。
|
|
1586
1634
|
* 直近 lookbackDays の日付シャードから rollout-*.jsonl を集め、mtime 降順に並べ、header を
|
|
1587
|
-
*
|
|
1635
|
+
* 派生インデックスを更新してから cwd 一致分を limit 件まで返す。返り値の shape は旧 listThreads と同一:
|
|
1588
1636
|
* {thread_id, cwd, preview, updated_at_ms}[]
|
|
1589
1637
|
*
|
|
1590
1638
|
* ⚠️ rollout は「作成日」シャードに置かれ以後追記される。作成が lookbackDays より前で今日
|
|
1591
1639
|
* resume された長寿命スレッドは窓外で拾えない (findCodexRolloutFile の高速パスと同じ限界)。
|
|
1592
|
-
* 一覧 (picker) 用途では直近が支配的なので許容。
|
|
1593
|
-
*
|
|
1640
|
+
* 一覧 (picker) 用途では直近が支配的なので許容。maxScan は既存 API 互換のため受け取っても
|
|
1641
|
+
* 無視し、cwd ごとの一覧欠落を起こすグローバル走査上限には使わない。
|
|
1594
1642
|
*
|
|
1595
1643
|
* @returns {Promise<Array<{thread_id:string, cwd:string, preview:string, updated_at_ms:number}>>}
|
|
1596
1644
|
*/
|
|
@@ -1598,10 +1646,11 @@ export async function listCodexThreadsFromDisk({
|
|
|
1598
1646
|
cwd,
|
|
1599
1647
|
limit,
|
|
1600
1648
|
lookbackDays = 180,
|
|
1601
|
-
maxScan = 800,
|
|
1602
1649
|
now = Date.now(),
|
|
1603
1650
|
logger,
|
|
1604
1651
|
sessionsDir,
|
|
1652
|
+
// テスト専用。production 呼び出しでは指定しない。
|
|
1653
|
+
_testHooks,
|
|
1605
1654
|
} = {}) {
|
|
1606
1655
|
// cwd-scoped picker 用。cwd 未指定なら「全 workspace を返す」ではなく空を返す (cross-review
|
|
1607
1656
|
// major3: 空 cwd で全 workspace の rollout が混入するのを防ぐ。呼び出し側 frontend は常に cwd を
|
|
@@ -1610,13 +1659,16 @@ export async function listCodexThreadsFromDisk({
|
|
|
1610
1659
|
// sessionsDir は self-describing read (Read-plane 案C ①) で呼び出し側が
|
|
1611
1660
|
// 「リクエストの profile_id から解決した読み取り先」を明示注入する経路。
|
|
1612
1661
|
// 未指定なら従来どおり env 由来の active (codexSessionsDir) にフォールバックする。
|
|
1613
|
-
const base = sessionsDir || codexSessionsDir()
|
|
1614
|
-
|
|
1662
|
+
const base = path.resolve(sessionsDir || codexSessionsDir())
|
|
1663
|
+
const seenAt = _testHooks?.nowMs?.() ?? Date.now()
|
|
1664
|
+
// 1. lookback 窓内の全 rollout を stat して {fp, mtimeMs, size} を集める。
|
|
1615
1665
|
// ⚠️ updated_at_ms (= mtime) 降順の契約を守るには「どれが更新新しいか」を知る必要があり、
|
|
1616
1666
|
// それには窓内の全 stat が要る (cross-review round2: 収集を作成日 shard 順で打ち切ると、
|
|
1617
1667
|
// 作成が古く更新が新しい長寿命 rollout を落として mtime 降順に反する)。stat は 1 syscall で
|
|
1618
|
-
// 安価かつ lookbackDays
|
|
1619
|
-
const candidates = [] // {fp, mtimeMs}
|
|
1668
|
+
// 安価かつ lookbackDays で有界。header open+read は下の派生インデックスで変更分だけに抑える。
|
|
1669
|
+
const candidates = [] // {fp, mtimeMs, size}
|
|
1670
|
+
const discovered = new Set() // readdir で存在を確認できた fp (stat 一時失敗でも削除扱いにしない)
|
|
1671
|
+
const scannedDirs = new Set()
|
|
1620
1672
|
for (let i = 0; i < lookbackDays; i++) {
|
|
1621
1673
|
const d = new Date(now - i * 24 * 60 * 60 * 1000)
|
|
1622
1674
|
const y = String(d.getFullYear())
|
|
@@ -1625,27 +1677,97 @@ export async function listCodexThreadsFromDisk({
|
|
|
1625
1677
|
const dir = path.join(base, y, m, day)
|
|
1626
1678
|
const files = await fs.readdir(dir).catch(() => null)
|
|
1627
1679
|
if (!files) continue
|
|
1680
|
+
scannedDirs.add(dir)
|
|
1628
1681
|
for (const f of files) {
|
|
1629
1682
|
if (!f.startsWith("rollout-") || !f.endsWith(".jsonl")) continue
|
|
1630
1683
|
const fp = path.join(dir, f)
|
|
1684
|
+
discovered.add(fp)
|
|
1631
1685
|
const st = await fs.stat(fp).catch(() => null)
|
|
1632
|
-
if (st) candidates.push({ fp, mtimeMs: st.mtimeMs })
|
|
1686
|
+
if (st) candidates.push({ fp, mtimeMs: st.mtimeMs, size: st.size })
|
|
1633
1687
|
}
|
|
1634
1688
|
}
|
|
1635
|
-
// 2. 更新が新しい順 (mtime desc)
|
|
1689
|
+
// 2. 更新が新しい順 (mtime desc)。候補一覧自体は毎回取り直し、削除も検出する。
|
|
1636
1690
|
candidates.sort((a, b) => b.mtimeMs - a.mtimeMs)
|
|
1637
|
-
|
|
1638
|
-
|
|
1691
|
+
let index = _codexThreadListIndex.get(base)
|
|
1692
|
+
if (!index) {
|
|
1693
|
+
index = new Map()
|
|
1694
|
+
_codexThreadListIndex.set(base, index)
|
|
1695
|
+
}
|
|
1696
|
+
for (const fp of index.keys()) {
|
|
1697
|
+
// 今回実際に走査できた shard 内の不在だけを削除扱いにする。短い lookback や別 now の
|
|
1698
|
+
// 呼び出しが、走査範囲外の cache を消して互いに再openを誘発しないようにする。
|
|
1699
|
+
if (scannedDirs.has(path.dirname(fp)) && !discovered.has(fp)) {
|
|
1700
|
+
index.delete(fp)
|
|
1701
|
+
_testHooks?.onIndexPrune?.(fp)
|
|
1702
|
+
} else {
|
|
1703
|
+
const entry = index.get(fp)
|
|
1704
|
+
if (!discovered.has(fp) && seenAt - entry.lastSeenAt >= CODEX_THREAD_LIST_STALE_ENTRY_MS) {
|
|
1705
|
+
index.delete(fp)
|
|
1706
|
+
_testHooks?.onIndexPrune?.(fp)
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
// 3. 呼び出しローカル snapshot を候補取得時点の cache から作る。以後、並行呼び出しが
|
|
1712
|
+
// global index を prune / 更新しても、この呼び出しの出力は snapshot から安定して作れる。
|
|
1713
|
+
const snapshot = new Map()
|
|
1714
|
+
for (const { fp, mtimeMs, size } of candidates) {
|
|
1715
|
+
const cached = index.get(fp)
|
|
1716
|
+
if (cached && cached.mtimeMs === mtimeMs && cached.size === size) {
|
|
1717
|
+
const seen = { ...cached, lastSeenAt: seenAt }
|
|
1718
|
+
commitCodexThreadListEntry(index, fp, cached, seen)
|
|
1719
|
+
// snapshot はこの呼び出しが観測した候補を正本にし、並行書戻しの成否に依存させない。
|
|
1720
|
+
snapshot.set(fp, seen)
|
|
1721
|
+
} else if (
|
|
1722
|
+
cached?.meta?.preview_complete &&
|
|
1723
|
+
cached.meta.cwd &&
|
|
1724
|
+
cached.meta.preview &&
|
|
1725
|
+
size > cached.size &&
|
|
1726
|
+
mtimeMs >= cached.mtimeMs
|
|
1727
|
+
) {
|
|
1728
|
+
// rollout は append-only。preview まで確定済みなら末尾追記で header は変わらないため、
|
|
1729
|
+
// fingerprint だけ進める。truncate / mtime 巻戻し / preview 未完了は下で再解析する。
|
|
1730
|
+
const appended = { mtimeMs, size, lastSeenAt: seenAt, meta: cached.meta }
|
|
1731
|
+
commitCodexThreadListEntry(index, fp, cached, appended)
|
|
1732
|
+
snapshot.set(fp, appended)
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
// 4. 新規・変更分だけ header を解析する。別 cwd は session_meta を得た時点で止め、
|
|
1737
|
+
// その cwd が後で要求された時に一度だけ preview まで補完する。一時 I/O 失敗は cache せず、
|
|
1738
|
+
// 次 poll で再試行する。解析済み不正 header (meta=null) は fingerprint 不変なら再解析しない。
|
|
1739
|
+
let parsedHeaders = 0
|
|
1740
|
+
const headerParseStartedAt = Date.now()
|
|
1741
|
+
for (const { fp, mtimeMs, size } of candidates) {
|
|
1742
|
+
if (snapshot.has(fp)) continue
|
|
1743
|
+
const observed = index.get(fp)
|
|
1744
|
+
parsedHeaders++
|
|
1745
|
+
const meta = await parseCodexRolloutHead(fp, cwd, _testHooks)
|
|
1746
|
+
if (meta === CODEX_ROLLOUT_PARSE_IO_ERROR) continue
|
|
1747
|
+
const entry = { mtimeMs, size, lastSeenAt: seenAt, meta }
|
|
1748
|
+
commitCodexThreadListEntry(index, fp, observed, entry)
|
|
1749
|
+
snapshot.set(fp, entry)
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
// 5. cwd で絞ってから mtime 降順・limit を適用する。グローバル候補数には左右されない。
|
|
1639
1753
|
const out = []
|
|
1640
|
-
let
|
|
1641
|
-
for (const { fp, mtimeMs } of candidates) {
|
|
1754
|
+
let completedPreviews = 0
|
|
1755
|
+
for (const { fp, mtimeMs, size } of candidates) {
|
|
1642
1756
|
if (limit && out.length >= limit) break
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
const meta = await parseCodexRolloutHead(fp, cwd)
|
|
1757
|
+
const entry = snapshot.get(fp)
|
|
1758
|
+
let meta = entry?.meta
|
|
1646
1759
|
if (!meta) continue
|
|
1647
|
-
// header の cwd が一致するものだけ (cwd 未読/不一致は除外 = 無関係スレッドの混入防止)。
|
|
1648
1760
|
if (meta.cwd !== cwd) continue
|
|
1761
|
+
if (!meta.preview_complete) {
|
|
1762
|
+
const observed = index.get(fp)
|
|
1763
|
+
completedPreviews++
|
|
1764
|
+
meta = await parseCodexRolloutHead(fp, "", _testHooks)
|
|
1765
|
+
if (meta === CODEX_ROLLOUT_PARSE_IO_ERROR) continue
|
|
1766
|
+
const completeEntry = { mtimeMs, size, lastSeenAt: seenAt, meta }
|
|
1767
|
+
commitCodexThreadListEntry(index, fp, observed, completeEntry)
|
|
1768
|
+
snapshot.set(fp, completeEntry)
|
|
1769
|
+
if (!meta || meta.cwd !== cwd) continue
|
|
1770
|
+
}
|
|
1649
1771
|
out.push({
|
|
1650
1772
|
thread_id: meta.thread_id,
|
|
1651
1773
|
cwd: meta.cwd,
|
|
@@ -1653,12 +1775,28 @@ export async function listCodexThreadsFromDisk({
|
|
|
1653
1775
|
updated_at_ms: mtimeMs,
|
|
1654
1776
|
})
|
|
1655
1777
|
}
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1778
|
+
|
|
1779
|
+
const elapsedMs = Date.now() - headerParseStartedAt
|
|
1780
|
+
const wallNow = Date.now()
|
|
1781
|
+
const lastWarnAt = _codexThreadListLastWarnAt.get(base)
|
|
1782
|
+
if (
|
|
1783
|
+
typeof logger?.warn === "function" &&
|
|
1784
|
+
(parsedHeaders + completedPreviews > 200 || elapsedMs > 1000) &&
|
|
1785
|
+
(lastWarnAt === undefined || wallNow - lastWarnAt >= CODEX_THREAD_LIST_LOG_INTERVAL_MS)
|
|
1786
|
+
) {
|
|
1787
|
+
_codexThreadListLastWarnAt.set(base, wallNow)
|
|
1788
|
+
logger.warn(
|
|
1789
|
+
{
|
|
1790
|
+
cwd,
|
|
1791
|
+
parsed_headers: parsedHeaders,
|
|
1792
|
+
completed_previews: completedPreviews,
|
|
1793
|
+
total_candidates: candidates.length,
|
|
1794
|
+
elapsed_ms: elapsedMs,
|
|
1795
|
+
},
|
|
1796
|
+
"listCodexThreadsFromDisk: 派生インデックスの header 解析負荷が閾値超過",
|
|
1660
1797
|
)
|
|
1661
1798
|
}
|
|
1799
|
+
|
|
1662
1800
|
return out
|
|
1663
1801
|
}
|
|
1664
1802
|
|
package/src/ws-client.mjs
CHANGED
|
@@ -143,6 +143,7 @@ export class WsClient extends EventEmitter {
|
|
|
143
143
|
this.bundleStatus = {}
|
|
144
144
|
this.bundleManifestPath = opts.bundleManifestPath || null
|
|
145
145
|
this.bundleWatcher = null
|
|
146
|
+
this.bundleWatcherConnectionMeta = null
|
|
146
147
|
this.bundleWatchDebounceTimer = null
|
|
147
148
|
this.hostname = opts.hostname || os.hostname()
|
|
148
149
|
this.platform = opts.platform || "unknown"
|
|
@@ -171,6 +172,10 @@ export class WsClient extends EventEmitter {
|
|
|
171
172
|
this.lastRecvAt = 0
|
|
172
173
|
// heartbeat tick の直前実発火時刻。スリープ復帰のジャンプ検知に使う。
|
|
173
174
|
this.lastTickAt = 0
|
|
175
|
+
// close ログを接続世代ごとに完結させる。古い socket の close が遅れて届いても、
|
|
176
|
+
// 新しい接続の診断状態や timer を壊さないよう socket のクロージャに保持する。
|
|
177
|
+
this.connectionGeneration = 0
|
|
178
|
+
this.connectionMeta = this._newConnectionMeta(0)
|
|
174
179
|
}
|
|
175
180
|
|
|
176
181
|
/** WSS 接続を開始する。`stop()` まで自動で reconnect 続行。 */
|
|
@@ -183,14 +188,21 @@ export class WsClient extends EventEmitter {
|
|
|
183
188
|
Authorization: `Bearer ${this.config.agent_id}:${this.config.agent_token}`,
|
|
184
189
|
}
|
|
185
190
|
const ws = new WebSocket(wsUrl, { headers })
|
|
191
|
+
const connectionMeta = this._newConnectionMeta(++this.connectionGeneration)
|
|
186
192
|
this.ws = ws
|
|
193
|
+
this.connectionMeta = connectionMeta
|
|
187
194
|
|
|
188
|
-
ws.on("open", () =>
|
|
195
|
+
ws.on("open", () => {
|
|
196
|
+
if (!this._isActiveConnection(ws, connectionMeta)) return
|
|
197
|
+
this._onOpen(connectionMeta)
|
|
198
|
+
})
|
|
189
199
|
|
|
190
200
|
ws.on("message", (data) => {
|
|
201
|
+
if (!this._isActiveConnection(ws, connectionMeta)) return
|
|
191
202
|
// 何かを受信した = TCP は生きている。zombie 検知の鮮度を更新する
|
|
192
203
|
// (parse 失敗でも「届いた」事実は生存の証拠なので先に記録する)。
|
|
193
204
|
this.lastRecvAt = Date.now()
|
|
205
|
+
connectionMeta.lastRecvAt = this.lastRecvAt
|
|
194
206
|
let msg
|
|
195
207
|
try {
|
|
196
208
|
msg = JSON.parse(data.toString("utf-8"))
|
|
@@ -202,13 +214,7 @@ export class WsClient extends EventEmitter {
|
|
|
202
214
|
})
|
|
203
215
|
|
|
204
216
|
ws.on("close", (code, reason) => {
|
|
205
|
-
this.
|
|
206
|
-
this._clearStableReset()
|
|
207
|
-
this._stopHeartbeat()
|
|
208
|
-
this._stopBundleWatcher()
|
|
209
|
-
this.logger?.info({ code, reason: reason?.toString() }, "ws close")
|
|
210
|
-
this.emit("close", { code, reason })
|
|
211
|
-
if (!this.stopped) this._scheduleReconnect()
|
|
217
|
+
this._onSocketClose(ws, connectionMeta, code, reason)
|
|
212
218
|
})
|
|
213
219
|
|
|
214
220
|
ws.on("error", (err) => {
|
|
@@ -216,7 +222,10 @@ export class WsClient extends EventEmitter {
|
|
|
216
222
|
// Upgrade で CF/origin が 5xx を返した場合 (典型: "Unexpected server
|
|
217
223
|
// response: 502") は、次回 reconnect の最低 backoff を引き上げる。
|
|
218
224
|
// CF が 5xx を short-cache するため、1s 連発リトライは逆に詰まる。
|
|
219
|
-
if (
|
|
225
|
+
if (
|
|
226
|
+
this._isActiveConnection(ws, connectionMeta) &&
|
|
227
|
+
/Unexpected server response: 5\d\d/.test(err?.message || "")
|
|
228
|
+
) {
|
|
220
229
|
this.lastCloseWas5xx = true
|
|
221
230
|
}
|
|
222
231
|
this.emit("error", err)
|
|
@@ -235,14 +244,23 @@ export class WsClient extends EventEmitter {
|
|
|
235
244
|
* test からも直接呼べるように method として切り出している (`ws.on("open")`
|
|
236
245
|
* のクロージャだと spy しづらい)。
|
|
237
246
|
*/
|
|
238
|
-
_onOpen() {
|
|
247
|
+
_onOpen(connectionMeta = this.connectionMeta) {
|
|
239
248
|
// open 自体が生存の証拠。直前接続の古い lastRecvAt で開幕直後に zombie 誤検知
|
|
240
249
|
// しないようリセットする。⚠️ _armStableReset() の armedAt 捕捉より「前」に設定すること。
|
|
241
250
|
// ack ベース判定 (lastRecvAt > armedAt) を初期状態で偽にするため (順序を逆にすると
|
|
242
251
|
// armedAt < lastRecvAt となり「inbound 受信済み」と誤判定し、half-open でも backoff を
|
|
243
252
|
// リセットしてしまう)。
|
|
244
|
-
|
|
245
|
-
this.
|
|
253
|
+
const now = Date.now()
|
|
254
|
+
this.lastRecvAt = now
|
|
255
|
+
this.lastTickAt = now
|
|
256
|
+
connectionMeta.openedAt = now
|
|
257
|
+
connectionMeta.lastRecvAt = 0
|
|
258
|
+
connectionMeta.lastTickAt = now
|
|
259
|
+
connectionMeta.lastTickGapMs = null
|
|
260
|
+
connectionMeta.lastHeartbeatAt = 0
|
|
261
|
+
connectionMeta.lastHeartbeatOk = null
|
|
262
|
+
connectionMeta.forceReconnectReason = null
|
|
263
|
+
connectionMeta.closeInitiator = null
|
|
246
264
|
// backoff の即時リセットはしない。接続が STABLE_CONNECTION_MS 維持 + inbound 受信できてから
|
|
247
265
|
// _armStableReset() でリセットする (open→即 close フラッピング時の再接続ストロボ対策)。
|
|
248
266
|
// すぐ切れた / half-open の接続では backoff がリセットされず指数バックオフが効き続けるため、
|
|
@@ -275,8 +293,8 @@ export class WsClient extends EventEmitter {
|
|
|
275
293
|
request_id: streamsSyncRequestId,
|
|
276
294
|
})
|
|
277
295
|
this._armStreamsSyncBarrier(streamsSyncRequestId)
|
|
278
|
-
this._startHeartbeat()
|
|
279
|
-
this._startBundleWatcher()
|
|
296
|
+
this._startHeartbeat(connectionMeta)
|
|
297
|
+
this._startBundleWatcher(connectionMeta)
|
|
280
298
|
this.emit("open")
|
|
281
299
|
}
|
|
282
300
|
|
|
@@ -419,6 +437,7 @@ export class WsClient extends EventEmitter {
|
|
|
419
437
|
}
|
|
420
438
|
if (this.ws) {
|
|
421
439
|
try {
|
|
440
|
+
this.connectionMeta.closeInitiator = "client_stop"
|
|
422
441
|
this.ws.close(1000, "client_stop")
|
|
423
442
|
} catch {
|
|
424
443
|
/* ignore */
|
|
@@ -430,6 +449,66 @@ export class WsClient extends EventEmitter {
|
|
|
430
449
|
// internals
|
|
431
450
|
// ------------------------------------------------------------
|
|
432
451
|
|
|
452
|
+
_newConnectionMeta(id) {
|
|
453
|
+
return {
|
|
454
|
+
id,
|
|
455
|
+
openedAt: 0,
|
|
456
|
+
lastRecvAt: 0,
|
|
457
|
+
lastTickAt: 0,
|
|
458
|
+
lastTickGapMs: null,
|
|
459
|
+
lastHeartbeatAt: 0,
|
|
460
|
+
lastHeartbeatOk: null,
|
|
461
|
+
forceReconnectReason: null,
|
|
462
|
+
closeInitiator: null,
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
_isActiveConnection(ws, connectionMeta) {
|
|
467
|
+
return this.ws === ws && this.connectionMeta === connectionMeta
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
_onSocketClose(ws, connectionMeta, code, reason) {
|
|
471
|
+
const now = Date.now()
|
|
472
|
+
const isActive = this._isActiveConnection(ws, connectionMeta)
|
|
473
|
+
this.logger?.info(
|
|
474
|
+
{
|
|
475
|
+
code,
|
|
476
|
+
reason: reason?.toString(),
|
|
477
|
+
connection_id: connectionMeta.id,
|
|
478
|
+
dur_s: connectionMeta.openedAt
|
|
479
|
+
? Math.max(0, Math.round((now - connectionMeta.openedAt) / 100) / 10)
|
|
480
|
+
: null,
|
|
481
|
+
since_recv_s: connectionMeta.lastRecvAt
|
|
482
|
+
? Math.max(0, Math.round((now - connectionMeta.lastRecvAt) / 100) / 10)
|
|
483
|
+
: null,
|
|
484
|
+
received_inbound: connectionMeta.lastRecvAt > 0,
|
|
485
|
+
last_heartbeat_at: connectionMeta.lastHeartbeatAt
|
|
486
|
+
? new Date(connectionMeta.lastHeartbeatAt).toISOString()
|
|
487
|
+
: null,
|
|
488
|
+
last_heartbeat_ok: connectionMeta.lastHeartbeatOk,
|
|
489
|
+
last_tick_at: connectionMeta.lastTickAt
|
|
490
|
+
? new Date(connectionMeta.lastTickAt).toISOString()
|
|
491
|
+
: null,
|
|
492
|
+
last_tick_gap_s: connectionMeta.lastTickGapMs == null
|
|
493
|
+
? null
|
|
494
|
+
: Math.max(0, Math.round(connectionMeta.lastTickGapMs / 100) / 10),
|
|
495
|
+
force_reconnect_reason: connectionMeta.forceReconnectReason,
|
|
496
|
+
close_initiator: connectionMeta.closeInitiator,
|
|
497
|
+
stale_generation: !isActive,
|
|
498
|
+
},
|
|
499
|
+
"ws close",
|
|
500
|
+
)
|
|
501
|
+
// terminate 後の close は次世代の connect/open より遅れて届くことがある。
|
|
502
|
+
// 旧世代から現世代の heartbeat・barrier・watcher・reconnect を変更しない。
|
|
503
|
+
if (!isActive) return
|
|
504
|
+
this._clearStreamsSyncBarrier()
|
|
505
|
+
this._clearStableReset()
|
|
506
|
+
this._stopHeartbeat()
|
|
507
|
+
this._stopBundleWatcher()
|
|
508
|
+
this.emit("close", { code, reason })
|
|
509
|
+
if (!this.stopped) this._scheduleReconnect()
|
|
510
|
+
}
|
|
511
|
+
|
|
433
512
|
_buildWsUrl() {
|
|
434
513
|
const hub = this.config.hub_url.replace(/\/+$/, "")
|
|
435
514
|
const scheme = hub.startsWith("https:") ? "wss:" : "ws:"
|
|
@@ -447,22 +526,39 @@ export class WsClient extends EventEmitter {
|
|
|
447
526
|
}
|
|
448
527
|
}
|
|
449
528
|
|
|
450
|
-
|
|
529
|
+
_sendHeartbeat(connectionMeta, heartbeatAt = Date.now()) {
|
|
530
|
+
if (connectionMeta !== this.connectionMeta) return null
|
|
531
|
+
const ok = this._sendJson({
|
|
532
|
+
type: "heartbeat",
|
|
533
|
+
uptime_sec: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
534
|
+
bundle_version: this.bundleVersion,
|
|
535
|
+
...this.bundleStatus,
|
|
536
|
+
})
|
|
537
|
+
connectionMeta.lastHeartbeatAt = heartbeatAt
|
|
538
|
+
connectionMeta.lastHeartbeatOk = ok
|
|
539
|
+
return ok
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
_startHeartbeat(connectionMeta = this.connectionMeta) {
|
|
451
543
|
this._stopHeartbeat()
|
|
452
544
|
this.lastTickAt = Date.now()
|
|
545
|
+
connectionMeta.lastTickAt = this.lastTickAt
|
|
453
546
|
this.heartbeatTimer = setInterval(() => {
|
|
547
|
+
if (connectionMeta !== this.connectionMeta) return
|
|
454
548
|
const now = Date.now()
|
|
455
549
|
// スリープ復帰検知: スリープ中はプロセスごと止まり、setInterval の実発火間隔が
|
|
456
550
|
// 大きく開く。復帰直後の TCP は half-open でほぼ確実に死んでいるのに send は
|
|
457
551
|
// 成功扱いになるため、鮮度判定 (受信側) を待たず即座に再接続へ倒す。
|
|
458
552
|
const tickGap = this.lastTickAt > 0 ? now - this.lastTickAt : 0
|
|
459
553
|
this.lastTickAt = now
|
|
554
|
+
connectionMeta.lastTickAt = now
|
|
555
|
+
connectionMeta.lastTickGapMs = tickGap
|
|
460
556
|
if (tickGap > TICK_JUMP_MS) {
|
|
461
557
|
this.logger?.warn(
|
|
462
558
|
{ tickGapMs: tickGap },
|
|
463
559
|
"heartbeat tick jump detected (likely sleep wake), forcing reconnect",
|
|
464
560
|
)
|
|
465
|
-
this._forceReconnect()
|
|
561
|
+
this._forceReconnect("sleep_tick_jump", connectionMeta)
|
|
466
562
|
return
|
|
467
563
|
}
|
|
468
564
|
// zombie WS 検知 (inbound 鮮度): backend は heartbeat のたびに ack を返すため、
|
|
@@ -475,7 +571,7 @@ export class WsClient extends EventEmitter {
|
|
|
475
571
|
{ sinceLastRecvMs: now - this.lastRecvAt },
|
|
476
572
|
"ws liveness timeout (zombie suspected), forcing reconnect",
|
|
477
573
|
)
|
|
478
|
-
this._forceReconnect()
|
|
574
|
+
this._forceReconnect("liveness_timeout", connectionMeta)
|
|
479
575
|
return
|
|
480
576
|
}
|
|
481
577
|
// heartbeat 都度 provider 経由で bundle version / CLI 別ステータスを最新化する。
|
|
@@ -485,15 +581,11 @@ export class WsClient extends EventEmitter {
|
|
|
485
581
|
// 送信失敗 (= ws not OPEN) を検知したら即時 close + reconnect 発火。
|
|
486
582
|
// setInterval を放置すると死んだコネクションに heartbeat を投げ続けて
|
|
487
583
|
// 30s 間気付かないので、close を強制トリガーする。
|
|
488
|
-
const ok = this.
|
|
489
|
-
|
|
490
|
-
uptime_sec: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
491
|
-
bundle_version: this.bundleVersion,
|
|
492
|
-
...this.bundleStatus,
|
|
493
|
-
})
|
|
584
|
+
const ok = this._sendHeartbeat(connectionMeta, now)
|
|
585
|
+
if (ok == null) return
|
|
494
586
|
if (!ok) {
|
|
495
587
|
this.logger?.warn("heartbeat send failed, forcing reconnect")
|
|
496
|
-
this._forceReconnect()
|
|
588
|
+
this._forceReconnect("heartbeat_send_failed", connectionMeta)
|
|
497
589
|
}
|
|
498
590
|
}, HEARTBEAT_INTERVAL_MS)
|
|
499
591
|
// unref して node プロセスが timer 1 個で抜けられなくならないように
|
|
@@ -549,13 +641,16 @@ export class WsClient extends EventEmitter {
|
|
|
549
641
|
* Cockpit UI への反映ラグを 30s heartbeat 周期から ~15s (UI poll) に短縮する。
|
|
550
642
|
* fs.watch の起動に失敗しても heartbeat 都度 refresh が保険として動くので no-op。
|
|
551
643
|
*/
|
|
552
|
-
_startBundleWatcher() {
|
|
553
|
-
if (!this.bundleManifestPath
|
|
644
|
+
_startBundleWatcher(connectionMeta = this.connectionMeta) {
|
|
645
|
+
if (!this.bundleManifestPath) return
|
|
646
|
+
if (this.bundleWatcher && this.bundleWatcherConnectionMeta === connectionMeta) return
|
|
647
|
+
if (this.bundleWatcher) this._stopBundleWatcher()
|
|
554
648
|
try {
|
|
649
|
+
this.bundleWatcherConnectionMeta = connectionMeta
|
|
555
650
|
this.bundleWatcher = fs.watch(
|
|
556
651
|
this.bundleManifestPath,
|
|
557
652
|
{ persistent: false },
|
|
558
|
-
() => this._onBundleManifestChange(),
|
|
653
|
+
() => this._onBundleManifestChange(connectionMeta),
|
|
559
654
|
)
|
|
560
655
|
this.bundleWatcher.on("error", (err) => {
|
|
561
656
|
this.logger?.warn(
|
|
@@ -568,6 +663,7 @@ export class WsClient extends EventEmitter {
|
|
|
568
663
|
"bundle manifest watcher started",
|
|
569
664
|
)
|
|
570
665
|
} catch (err) {
|
|
666
|
+
this.bundleWatcherConnectionMeta = null
|
|
571
667
|
this.logger?.warn(
|
|
572
668
|
{ err: err.message, path: this.bundleManifestPath },
|
|
573
669
|
"bundle manifest watcher failed to start",
|
|
@@ -588,43 +684,46 @@ export class WsClient extends EventEmitter {
|
|
|
588
684
|
}
|
|
589
685
|
this.bundleWatcher = null
|
|
590
686
|
}
|
|
687
|
+
this.bundleWatcherConnectionMeta = null
|
|
591
688
|
}
|
|
592
689
|
|
|
593
690
|
/**
|
|
594
691
|
* fs.watch のコールバック。同一書き込みで多重 fire することがある (macOS) ため
|
|
595
692
|
* デバウンスしてから refresh + 即時 heartbeat する。
|
|
596
693
|
*/
|
|
597
|
-
_onBundleManifestChange() {
|
|
694
|
+
_onBundleManifestChange(connectionMeta = this.connectionMeta) {
|
|
695
|
+
if (connectionMeta !== this.connectionMeta) return
|
|
598
696
|
if (this.bundleWatchDebounceTimer) {
|
|
599
697
|
clearTimeout(this.bundleWatchDebounceTimer)
|
|
600
698
|
}
|
|
601
699
|
this.bundleWatchDebounceTimer = setTimeout(() => {
|
|
602
700
|
this.bundleWatchDebounceTimer = null
|
|
701
|
+
if (connectionMeta !== this.connectionMeta) return
|
|
603
702
|
const changed = this._refreshBundleVersion()
|
|
604
703
|
if (!changed) return
|
|
605
704
|
this._refreshBundleStatus()
|
|
606
|
-
const ok = this.
|
|
607
|
-
|
|
608
|
-
uptime_sec: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
609
|
-
bundle_version: this.bundleVersion,
|
|
610
|
-
...this.bundleStatus,
|
|
611
|
-
})
|
|
705
|
+
const ok = this._sendHeartbeat(connectionMeta)
|
|
706
|
+
if (ok == null) return
|
|
612
707
|
if (!ok) {
|
|
613
708
|
this.logger?.warn("bundle-change heartbeat send failed, forcing reconnect")
|
|
614
|
-
this._forceReconnect()
|
|
709
|
+
this._forceReconnect("bundle_heartbeat_send_failed", connectionMeta)
|
|
615
710
|
}
|
|
616
711
|
}, BUNDLE_WATCH_DEBOUNCE_MS)
|
|
617
712
|
this.bundleWatchDebounceTimer.unref?.()
|
|
618
713
|
}
|
|
619
714
|
|
|
620
|
-
|
|
715
|
+
// reason の既定値は、将来の呼出漏れを無情報の null にせずログで検知する fail-soft。
|
|
716
|
+
_forceReconnect(reason = "unspecified", connectionMeta = this.connectionMeta) {
|
|
621
717
|
// 現在の ws を強制 close。close ハンドラが _scheduleReconnect を呼ぶので
|
|
622
718
|
// それ以上の処理は不要。stopped 中は no-op。
|
|
623
719
|
// ws 実装によって terminate (ws npm package) と close (browser WebSocket
|
|
624
720
|
// 互換) どちらが生えているか異なるので両対応する。
|
|
721
|
+
if (connectionMeta !== this.connectionMeta) return
|
|
625
722
|
if (this.stopped) return
|
|
626
723
|
if (!this.ws) return
|
|
627
724
|
if (this.ws.readyState === WebSocket.CLOSED) return
|
|
725
|
+
connectionMeta.forceReconnectReason = reason
|
|
726
|
+
connectionMeta.closeInitiator = "agent_reconnect"
|
|
628
727
|
try {
|
|
629
728
|
if (typeof this.ws.terminate === "function") {
|
|
630
729
|
this.ws.terminate()
|
|
@@ -682,7 +781,7 @@ export class WsClient extends EventEmitter {
|
|
|
682
781
|
{ sinceArmedMs: Date.now() - armedAt },
|
|
683
782
|
"ws stable window elapsed with no inbound (half-open suspected), forcing reconnect",
|
|
684
783
|
)
|
|
685
|
-
this._forceReconnect()
|
|
784
|
+
this._forceReconnect("stable_no_inbound")
|
|
686
785
|
}
|
|
687
786
|
}, STABLE_CONNECTION_MS)
|
|
688
787
|
this.stableResetTimer.unref?.()
|