@paperclipai/server 2026.707.0-canary.9 → 2026.707.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-auth-jwt.d.ts +1 -0
- package/dist/agent-auth-jwt.d.ts.map +1 -1
- package/dist/agent-auth-jwt.js +58 -19
- package/dist/agent-auth-jwt.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +122 -112
- package/dist/index.js.map +1 -1
- package/dist/routes/issues.d.ts.map +1 -1
- package/dist/routes/issues.js +132 -2
- package/dist/routes/issues.js.map +1 -1
- package/dist/services/activity.d.ts +0 -1
- package/dist/services/activity.d.ts.map +1 -1
- package/dist/services/activity.js +0 -1
- package/dist/services/activity.js.map +1 -1
- package/dist/services/external-objects.d.ts +13 -13
- package/dist/services/heartbeat.d.ts +5 -0
- package/dist/services/heartbeat.d.ts.map +1 -1
- package/dist/services/heartbeat.js +35 -0
- package/dist/services/heartbeat.js.map +1 -1
- package/dist/services/index.d.ts +1 -1
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +1 -1
- package/dist/services/index.js.map +1 -1
- package/dist/services/plugin-registry.d.ts +11 -11
- package/dist/services/secrets.d.ts.map +1 -1
- package/dist/services/secrets.js +197 -34
- package/dist/services/secrets.js.map +1 -1
- package/package.json +17 -17
- package/ui-dist/assets/{abnfDiagram-VRR7QNED-CRUUNIUj.js → abnfDiagram-VRR7QNED-Do_TRi_1.js} +1 -1
- package/ui-dist/assets/{arc-BkZXaYgW.js → arc-B3CbhYKx.js} +1 -1
- package/ui-dist/assets/{architectureDiagram-ZJ3FMSHR-D-fx_6yg.js → architectureDiagram-ZJ3FMSHR-BHkeTr8K.js} +1 -1
- package/ui-dist/assets/{blockDiagram-677ZJIJ3-CskS5Ke3.js → blockDiagram-677ZJIJ3-BlF8glgQ.js} +1 -1
- package/ui-dist/assets/{c4Diagram-LMCZKHZV-BpyZOtUQ.js → c4Diagram-LMCZKHZV-CPsUwCPj.js} +1 -1
- package/ui-dist/assets/channel-Blb5QzE9.js +1 -0
- package/ui-dist/assets/{chunk-2Q5K7J3B-CqhWcW6L.js → chunk-2Q5K7J3B-BDpcXrI9.js} +1 -1
- package/ui-dist/assets/{chunk-32BRIVSS-BfiMNxAO.js → chunk-32BRIVSS-DS4eRPEJ.js} +1 -1
- package/ui-dist/assets/{chunk-5VM5RSS4-BMfrlqg_.js → chunk-5VM5RSS4-KPnkyAEU.js} +1 -1
- package/ui-dist/assets/{chunk-EX3LRPZG-CGis75FW.js → chunk-EX3LRPZG-CblZ8LUu.js} +1 -1
- package/ui-dist/assets/{chunk-JWPE2WC7-lrOZGDPV.js → chunk-JWPE2WC7-D7SQlpwg.js} +1 -1
- package/ui-dist/assets/{chunk-MOJQB5TN-CoA0hIGv.js → chunk-MOJQB5TN-BNsZGMhW.js} +1 -1
- package/ui-dist/assets/{chunk-RYQCIY6F-Dfke42dY.js → chunk-RYQCIY6F--f185OVa.js} +1 -1
- package/ui-dist/assets/{chunk-V7JOEXUC-w5OMha1U.js → chunk-V7JOEXUC-Cv1YABi1.js} +1 -1
- package/ui-dist/assets/{chunk-VR4S4FIN-CD9N62c8.js → chunk-VR4S4FIN-BPSIRrGo.js} +1 -1
- package/ui-dist/assets/{chunk-XXDRQBXY-oqDFFdl6.js → chunk-XXDRQBXY-BU3MEjL7.js} +1 -1
- package/ui-dist/assets/classDiagram-OUVF2IWQ-BL7yN9sR.js +1 -0
- package/ui-dist/assets/classDiagram-v2-EOCWNBFH-BL7yN9sR.js +1 -0
- package/ui-dist/assets/{cose-bilkent-JH36ORCC-CUmT6zMB.js → cose-bilkent-JH36ORCC-Q0A3TS3z.js} +1 -1
- package/ui-dist/assets/{cynefin-VYW2F7L2-_OaXHJRG.js → cynefin-VYW2F7L2-C-f0AYNp.js} +1 -1
- package/ui-dist/assets/{cynefinDiagram-TSTJHNR4-D2bqtpma.js → cynefinDiagram-TSTJHNR4-C4MSop3D.js} +1 -1
- package/ui-dist/assets/{dagre-VKFMJZFB-B6wlRq48.js → dagre-VKFMJZFB-F9UNb-Q0.js} +1 -1
- package/ui-dist/assets/{diagram-FQU43EPY-0ZHDgeDK.js → diagram-FQU43EPY-D5tfn73P.js} +1 -1
- package/ui-dist/assets/{diagram-G47NLZAW-ukzjnW7V.js → diagram-G47NLZAW-BWbsH70G.js} +1 -1
- package/ui-dist/assets/{diagram-NH7WQ7WH-BCEfAetY.js → diagram-NH7WQ7WH-tDS24ugE.js} +1 -1
- package/ui-dist/assets/{diagram-OA4YK3LP-B08ve798.js → diagram-OA4YK3LP-DgxJIRAr.js} +1 -1
- package/ui-dist/assets/{diagram-WEI45ONY-C9aZfy6Y.js → diagram-WEI45ONY-CtrSm1NA.js} +1 -1
- package/ui-dist/assets/{ebnfDiagram-CCIWWBDH-Bn0o5_dg.js → ebnfDiagram-CCIWWBDH-DMmA623x.js} +1 -1
- package/ui-dist/assets/{erDiagram-Q63AITRT-CaY5pfG8.js → erDiagram-Q63AITRT-DCsDefdf.js} +1 -1
- package/ui-dist/assets/{flowDiagram-23GEKE2U-CWkwblxp.js → flowDiagram-23GEKE2U-fnIVX0ZN.js} +1 -1
- package/ui-dist/assets/{ganttDiagram-NO4QXBWP-DOz0w5qk.js → ganttDiagram-NO4QXBWP-CPdJRd9c.js} +1 -1
- package/ui-dist/assets/{gitGraphDiagram-IHSO6WYX-Drp-cx1v.js → gitGraphDiagram-IHSO6WYX-CW9Tq6iM.js} +1 -1
- package/ui-dist/assets/{index-BuGSWxB-.js → index-B1mGY1Kk.js} +1 -1
- package/ui-dist/assets/{index-GHchkQe3.js → index-B5YrVPz-.js} +1 -1
- package/ui-dist/assets/{index-yeqh3B6M.js → index-BM9U55my.js} +1 -1
- package/ui-dist/assets/{index-BzxLmFn-.js → index-Bn24W0Sj.js} +1 -1
- package/ui-dist/assets/{index-C17_xFpR.js → index-C0vofs_Q.js} +1 -1
- package/ui-dist/assets/{index-B2fzsM6R.js → index-C5Fyaff7.js} +1 -1
- package/ui-dist/assets/{index-BL8I_FaR.js → index-CB2k56cL.js} +1 -1
- package/ui-dist/assets/{index-CryQfBWJ.js → index-CRHnEybV.js} +1 -1
- package/ui-dist/assets/{index-z8k9vRXH.js → index-CUA8O1P0.js} +1 -1
- package/ui-dist/assets/{index-H_pKdWcw.js → index-CUZzyC56.js} +1 -1
- package/ui-dist/assets/{index-Dde4KVU3.js → index-Cf8V3Yek.js} +1 -1
- package/ui-dist/assets/{index-BJyoaITQ.js → index-D0Sv9g6U.js} +1 -1
- package/ui-dist/assets/{index-D_iCmWrh.js → index-D0kScTTQ.js} +1 -1
- package/ui-dist/assets/{index-CBXpgOXA.js → index-D2OxavQ-.js} +1 -1
- package/ui-dist/assets/{index-w2iIi9Zy.js → index-DYScEjF9.js} +1 -1
- package/ui-dist/assets/{index-BTLtGxHN.js → index-DdwBeD96.js} +163 -163
- package/ui-dist/assets/{index-09e4CbXX.js → index-DrftETSU.js} +1 -1
- package/ui-dist/assets/{index-B2ETwIwN.js → index-DyXbc50h.js} +1 -1
- package/ui-dist/assets/{index-K84tj6B5.js → index-FQbL2wsW.js} +1 -1
- package/ui-dist/assets/{index-CXIzZ6DP.js → index-HeGabcjA.js} +1 -1
- package/ui-dist/assets/{index-BzMOx8G_.js → index-OCUxXBrq.js} +1 -1
- package/ui-dist/assets/{index-Dn0-0dbP.js → index-YYDzkN-f.js} +1 -1
- package/ui-dist/assets/{index-BX6rkSyW.js → index-_kjqW0m7.js} +1 -1
- package/ui-dist/assets/{index-CLzESWGk.js → index-_xe6lO17.js} +1 -1
- package/ui-dist/assets/{infoDiagram-FWYZ7A6U-BbklVX1c.js → infoDiagram-FWYZ7A6U-Dis8S_K7.js} +1 -1
- package/ui-dist/assets/{ishikawaDiagram-FXEZZL3T-CJbaRdbE.js → ishikawaDiagram-FXEZZL3T-BoRk7_U8.js} +1 -1
- package/ui-dist/assets/{journeyDiagram-5HDEW3XC-2x7IPuCU.js → journeyDiagram-5HDEW3XC-DJMkaH8r.js} +1 -1
- package/ui-dist/assets/{kanban-definition-HUTT4EX6-KEIzM1Zk.js → kanban-definition-HUTT4EX6-Bcs8bDpx.js} +1 -1
- package/ui-dist/assets/{linear-B7vT4EQW.js → linear-BbBu9n-4.js} +1 -1
- package/ui-dist/assets/{mermaid.core-1S94fnGm.js → mermaid.core-BgIN7LAd.js} +4 -4
- package/ui-dist/assets/{mindmap-definition-LN4V7U3C-HCfmaoYC.js → mindmap-definition-LN4V7U3C-CCsaVg76.js} +1 -1
- package/ui-dist/assets/{pegDiagram-2B236MQR-BOJN05hz.js → pegDiagram-2B236MQR-bODtpUbV.js} +1 -1
- package/ui-dist/assets/{pieDiagram-ENE6RG2P-URVmHB_B.js → pieDiagram-ENE6RG2P-CHAOhZKS.js} +1 -1
- package/ui-dist/assets/{quadrantDiagram-ABIIQ3AL-D-6CNaGh.js → quadrantDiagram-ABIIQ3AL-Bx5GJMoa.js} +1 -1
- package/ui-dist/assets/{railroadDiagram-RFXS5EU6-BMl-2I6r.js → railroadDiagram-RFXS5EU6-DTIlwwWF.js} +1 -1
- package/ui-dist/assets/{requirementDiagram-TGXJPOKE-DowQ29cU.js → requirementDiagram-TGXJPOKE-Cec4mbEZ.js} +1 -1
- package/ui-dist/assets/{sankeyDiagram-HTMAVEWB-Aih1d0bA.js → sankeyDiagram-HTMAVEWB-CAqu35rx.js} +1 -1
- package/ui-dist/assets/{sequenceDiagram-DBY2YBRQ-ClvmLKnU.js → sequenceDiagram-DBY2YBRQ-RvR8hmx4.js} +1 -1
- package/ui-dist/assets/{sizeCapture-X5ZJPWSS-DhJ8VYqW.js → sizeCapture-X5ZJPWSS-C-nGNlOq.js} +1 -1
- package/ui-dist/assets/{stateDiagram-2N3HPSRC-x6opQ8ip.js → stateDiagram-2N3HPSRC-D_RDwhnD.js} +1 -1
- package/ui-dist/assets/stateDiagram-v2-6OUMAXLB-DGUaDGi8.js +1 -0
- package/ui-dist/assets/{swimlanes-5IMT3BWC-B-vL4RrC.js → swimlanes-5IMT3BWC-CowJgUlh.js} +2 -2
- package/ui-dist/assets/swimlanesDiagram-G3AALYLV-DQtuqz4h.js +8 -0
- package/ui-dist/assets/{timeline-definition-FHXFAJF6-5ETzbGQi.js → timeline-definition-FHXFAJF6-DqG5pp0k.js} +1 -1
- package/ui-dist/assets/{vennDiagram-L72KCM5P-CZ3Cb-Df.js → vennDiagram-L72KCM5P-6yiMOixp.js} +1 -1
- package/ui-dist/assets/{wardleyDiagram-EHGQE667-CCS1DwYT.js → wardleyDiagram-EHGQE667-CZMcBbct.js} +1 -1
- package/ui-dist/assets/{xychartDiagram-FW5EYKEG-Brm2Ikmv.js → xychartDiagram-FW5EYKEG-BFb49f7b.js} +1 -1
- package/ui-dist/index.html +1 -1
- package/ui-dist/assets/channel-DxqB4M3m.js +0 -1
- package/ui-dist/assets/classDiagram-OUVF2IWQ-B-TWzgAX.js +0 -1
- package/ui-dist/assets/classDiagram-v2-EOCWNBFH-B-TWzgAX.js +0 -1
- package/ui-dist/assets/stateDiagram-v2-6OUMAXLB-CPBt6_m0.js +0 -1
- package/ui-dist/assets/swimlanesDiagram-G3AALYLV-CyNSi-YP.js +0 -8
package/dist/agent-auth-jwt.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export interface LocalAgentJwtClaims {
|
|
|
8
8
|
exp: number;
|
|
9
9
|
iss?: string;
|
|
10
10
|
aud?: string;
|
|
11
|
+
instance_id?: string;
|
|
11
12
|
jti?: string;
|
|
12
13
|
}
|
|
13
14
|
export declare function createLocalAgentJwt(agentId: string, companyId: string, adapterType: string, runId: string, responsibleUserId?: string | null): string | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-auth-jwt.d.ts","sourceRoot":"","sources":["../src/agent-auth-jwt.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"agent-auth-jwt.d.ts","sourceRoot":"","sources":["../src/agent-auth-jwt.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA6FD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,iBAgClC;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAwF7E"}
|
package/dist/agent-auth-jwt.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { resolvePaperclipInstanceId } from "./home-paths.js";
|
|
2
3
|
const JWT_ALGORITHM = "HS256";
|
|
3
4
|
function parseNumber(value, fallback) {
|
|
4
5
|
const parsed = Number(value);
|
|
@@ -21,23 +22,43 @@ function jwtConfig() {
|
|
|
21
22
|
ttlSeconds: parseNumber(process.env.PAPERCLIP_AGENT_JWT_TTL_SECONDS, 60 * 60),
|
|
22
23
|
issuer: process.env.PAPERCLIP_AGENT_JWT_ISSUER ?? "paperclip",
|
|
23
24
|
audience: process.env.PAPERCLIP_AGENT_JWT_AUDIENCE ?? "paperclip-api",
|
|
25
|
+
// The control-plane instance this process belongs to. The live plane runs as
|
|
26
|
+
// "default"; every worktree/fork instance gets a distinct id (its worktree
|
|
27
|
+
// name) even though it deliberately shares PAPERCLIP_AGENT_JWT_SECRET with
|
|
28
|
+
// the source instance. Folding this into the signing-key derivation is what
|
|
29
|
+
// prevents a fork-minted token from authenticating against the live plane.
|
|
30
|
+
instanceId: resolvePaperclipInstanceId(),
|
|
24
31
|
disableLegacyFallback: parseBooleanEnv(process.env.PAPERCLIP_AGENT_JWT_DISABLE_LEGACY_FALLBACK),
|
|
25
32
|
};
|
|
26
33
|
}
|
|
27
34
|
/**
|
|
28
|
-
* Derive a per-company signing key from the master JWT secret
|
|
35
|
+
* Derive a per-instance, per-company signing key from the master JWT secret,
|
|
36
|
+
* the control-plane instanceId, and a companyId.
|
|
29
37
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
38
|
+
* Two isolation properties fall out of this derivation:
|
|
39
|
+
* - Per-company: a JWT signed for company A cannot be reused to authenticate
|
|
40
|
+
* as an agent in company B, even if the raw token leaks.
|
|
41
|
+
* - Per-instance: a JWT minted by a worktree/fork control-plane instance
|
|
42
|
+
* cannot authenticate against the live plane, even though forks
|
|
43
|
+
* deliberately share the same master secret (it is copied into worktree
|
|
44
|
+
* envs by provisioning). The live plane derives its key from its own
|
|
45
|
+
* instanceId ("default"), so a fork token — signed under the fork's
|
|
46
|
+
* instanceId — never matches. See PAP-12896 for the incident this closes.
|
|
47
|
+
*
|
|
48
|
+
* The instance-wide master secret is never used to sign new tokens — it is
|
|
49
|
+
* retained only as a verification fallback so that tokens issued before this
|
|
50
|
+
* change continue to validate. NOTE: that legacy fallback is instance-agnostic
|
|
51
|
+
* (it signs with the raw shared secret), so complete cryptographic instance
|
|
52
|
+
* isolation additionally requires disabling it once outstanding legacy tokens
|
|
53
|
+
* have expired (set PAPERCLIP_AGENT_JWT_DISABLE_LEGACY_FALLBACK=true). Normal
|
|
54
|
+
* fork-minted run tokens are already rejected without that step because they
|
|
55
|
+
* are signed with the derived key, not the raw master secret.
|
|
35
56
|
*
|
|
36
57
|
* The derivation domain-separates with the `jwt:` prefix so the same master
|
|
37
58
|
* secret can safely be reused for other HMAC purposes without key reuse.
|
|
38
59
|
*/
|
|
39
|
-
function deriveCompanySigningKey(masterSecret, companyId) {
|
|
40
|
-
return createHmac("sha256", masterSecret).update(`jwt:${companyId}`).digest("hex");
|
|
60
|
+
function deriveCompanySigningKey(masterSecret, companyId, instanceId) {
|
|
61
|
+
return createHmac("sha256", masterSecret).update(`jwt:${instanceId}:${companyId}`).digest("hex");
|
|
41
62
|
}
|
|
42
63
|
function base64UrlEncode(value) {
|
|
43
64
|
return Buffer.from(value, "utf8").toString("base64url");
|
|
@@ -79,15 +100,17 @@ export function createLocalAgentJwt(agentId, companyId, adapterType, runId, resp
|
|
|
79
100
|
exp: now + config.ttlSeconds,
|
|
80
101
|
iss: config.issuer,
|
|
81
102
|
aud: config.audience,
|
|
103
|
+
instance_id: config.instanceId,
|
|
82
104
|
};
|
|
83
105
|
const header = {
|
|
84
106
|
alg: JWT_ALGORITHM,
|
|
85
107
|
typ: "JWT",
|
|
86
108
|
};
|
|
87
109
|
const signingInput = `${base64UrlEncode(JSON.stringify(header))}.${base64UrlEncode(JSON.stringify(claims))}`;
|
|
88
|
-
// Sign with the per-company derived key so a leaked token
|
|
89
|
-
// across tenants
|
|
90
|
-
|
|
110
|
+
// Sign with the per-instance, per-company derived key so a leaked token
|
|
111
|
+
// cannot be reused across tenants and a fork-minted token cannot authenticate
|
|
112
|
+
// against a different control-plane instance.
|
|
113
|
+
const signingKey = deriveCompanySigningKey(config.secret, companyId, config.instanceId);
|
|
91
114
|
const signature = signPayload(signingKey, signingInput);
|
|
92
115
|
return `${signingInput}.${signature}`;
|
|
93
116
|
}
|
|
@@ -111,18 +134,24 @@ export function verifyLocalAgentJwt(token) {
|
|
|
111
134
|
if (!claimedCompanyId)
|
|
112
135
|
return null;
|
|
113
136
|
const signingInput = `${headerB64}.${claimsB64}`;
|
|
114
|
-
// Try the per-company derived key first (current tokens)
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
137
|
+
// Try the per-instance, per-company derived key first (current tokens),
|
|
138
|
+
// deriving under THIS control plane's own instanceId. A token minted by a
|
|
139
|
+
// worktree/fork instance was signed under a different instanceId, so it will
|
|
140
|
+
// not match here — that is the boundary that keeps fork tokens out of the
|
|
141
|
+
// live plane (PAP-12896/PAP-12899). Fall back to the raw master secret so
|
|
142
|
+
// tokens issued before per-company derivation existed continue to verify —
|
|
143
|
+
// this preserves backward compatibility for any outstanding tokens (TTL
|
|
144
|
+
// bounds the legacy window naturally).
|
|
118
145
|
//
|
|
119
146
|
// Operators should set `PAPERCLIP_AGENT_JWT_DISABLE_LEGACY_FALLBACK=true`
|
|
120
147
|
// approximately one JWT TTL (~1h by default, see PAPERCLIP_AGENT_JWT_TTL_SECONDS)
|
|
121
148
|
// after deploying per-company signing. Once set, the master-secret fallback
|
|
122
|
-
// is disabled and only tokens validating under the per-company
|
|
123
|
-
// are accepted — closing the window in which a leaked master
|
|
124
|
-
// be used to forge tokens with arbitrary future `exp` values for
|
|
125
|
-
|
|
149
|
+
// is disabled and only tokens validating under the per-instance/per-company
|
|
150
|
+
// derived key are accepted — closing the window in which a leaked master
|
|
151
|
+
// secret could be used to forge tokens with arbitrary future `exp` values for
|
|
152
|
+
// any tenant, and completing cryptographic isolation between control-plane
|
|
153
|
+
// instances (the raw-secret fallback is instance-agnostic).
|
|
154
|
+
const perCompanyKey = deriveCompanySigningKey(config.secret, claimedCompanyId, config.instanceId);
|
|
126
155
|
const perCompanySig = signPayload(perCompanyKey, signingInput);
|
|
127
156
|
let signatureOk = safeCompare(signature, perCompanySig);
|
|
128
157
|
if (!signatureOk && !config.disableLegacyFallback) {
|
|
@@ -153,6 +182,15 @@ export function verifyLocalAgentJwt(token) {
|
|
|
153
182
|
return null;
|
|
154
183
|
if (audience && audience !== config.audience)
|
|
155
184
|
return null;
|
|
185
|
+
// Enforce the minting instance when the claim is present. The instance-scoped
|
|
186
|
+
// signing key above is the real cryptographic boundary; this claim check is
|
|
187
|
+
// defense-in-depth that yields a clean, cheap rejection (and, once legacy
|
|
188
|
+
// tokens have aged out, guards the master-secret fallback path too). Legacy
|
|
189
|
+
// tokens minted before this claim existed omit it and are still accepted, so
|
|
190
|
+
// enforcement is conditional — matching how iss/aud are handled above.
|
|
191
|
+
const instanceClaim = typeof claims.instance_id === "string" ? claims.instance_id : undefined;
|
|
192
|
+
if (instanceClaim && instanceClaim !== config.instanceId)
|
|
193
|
+
return null;
|
|
156
194
|
return {
|
|
157
195
|
sub,
|
|
158
196
|
company_id: companyId,
|
|
@@ -163,6 +201,7 @@ export function verifyLocalAgentJwt(token) {
|
|
|
163
201
|
exp,
|
|
164
202
|
...(issuer ? { iss: issuer } : {}),
|
|
165
203
|
...(audience ? { aud: audience } : {}),
|
|
204
|
+
...(instanceClaim ? { instance_id: instanceClaim } : {}),
|
|
166
205
|
jti: typeof claims.jti === "string" ? claims.jti : undefined,
|
|
167
206
|
};
|
|
168
207
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-auth-jwt.js","sourceRoot":"","sources":["../src/agent-auth-jwt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"agent-auth-jwt.js","sourceRoot":"","sources":["../src/agent-auth-jwt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAqB7D,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,SAAS,WAAW,CAAC,KAAyB,EAAE,QAAgB;IAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IAChD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,OAAO,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,IAAI,CAAC;AACpG,CAAC;AAED,SAAS,SAAS;IAChB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,CAAC;IACxG,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,OAAO;QACL,MAAM;QACN,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC;QAC7E,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,WAAW;QAC7D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,eAAe;QACrE,6EAA6E;QAC7E,2EAA2E;QAC3E,2EAA2E;QAC3E,4EAA4E;QAC5E,2EAA2E;QAC3E,UAAU,EAAE,0BAA0B,EAAE;QACxC,qBAAqB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC;KAChG,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,uBAAuB,CAAC,YAAoB,EAAE,SAAiB,EAAE,UAAkB;IAC1F,OAAO,UAAU,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,UAAU,IAAI,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnG,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,WAAW,CAAC,MAAc,EAAE,YAAoB;IACvD,OAAO,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS;IACvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAe,EACf,SAAiB,EACjB,WAAmB,EACnB,KAAa,EACb,iBAAiC;IAEjC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAwB;QAClC,GAAG,EAAE,OAAO;QACZ,UAAU,EAAE,SAAS;QACrB,YAAY,EAAE,WAAW;QACzB,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,iBAAiB,EAAE,IAAI,EAAE,IAAI,IAAI;QACtD,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC,UAAU;QAC5B,GAAG,EAAE,MAAM,CAAC,MAAM;QAClB,GAAG,EAAE,MAAM,CAAC,QAAQ;QACpB,WAAW,EAAE,MAAM,CAAC,UAAU;KAC/B,CAAC;IAEF,MAAM,MAAM,GAAG;QACb,GAAG,EAAE,aAAa;QAClB,GAAG,EAAE,KAAK;KACX,CAAC;IAEF,MAAM,YAAY,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;IAC7G,wEAAwE;IACxE,8EAA8E;IAC9E,8CAA8C;IAC9C,MAAM,UAAU,GAAG,uBAAuB,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAExD,OAAO,GAAG,YAAY,IAAI,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC;IAEhD,MAAM,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IAEzD,MAAM,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,gBAAgB,GAAG,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1F,IAAI,CAAC,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAEnC,MAAM,YAAY,GAAG,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC;IACjD,wEAAwE;IACxE,0EAA0E;IAC1E,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,2EAA2E;IAC3E,wEAAwE;IACxE,uCAAuC;IACvC,EAAE;IACF,0EAA0E;IAC1E,kFAAkF;IAClF,4EAA4E;IAC5E,4EAA4E;IAC5E,yEAAyE;IACzE,8EAA8E;IAC9E,2EAA2E;IAC3E,4DAA4D;IAC5D,MAAM,aAAa,GAAG,uBAAuB,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAClG,MAAM,aAAa,GAAG,WAAW,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IAC/D,IAAI,WAAW,GAAG,WAAW,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACxD,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;QAClD,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3D,WAAW,GAAG,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAE9B,MAAM,GAAG,GAAG,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACzF,MAAM,KAAK,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACvE,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC;QACvE,CAAC,CAAC,OAAO,MAAM,CAAC,mBAAmB,KAAK,QAAQ,IAAI,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE;YACnF,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE;YACnC,CAAC,CAAC,IAAI;QACR,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,GAAG,GAAG,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,MAAM,GAAG,GAAG,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAChE,MAAM,SAAS,GAAG,gBAAgB,CAAC;IAEnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACpD,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1D,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,aAAa,GAAG,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,IAAI,aAAa,IAAI,aAAa,KAAK,MAAM,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAEtE,OAAO;QACL,GAAG;QACH,UAAU,EAAE,SAAS;QACrB,YAAY,EAAE,WAAW;QACzB,MAAM,EAAE,KAAK;QACb,GAAG,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG;QACH,GAAG;QACH,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,EAAE,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;KAC7D,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AA0FzC,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,aAAa,CAAC,CAygC1D"}
|
package/dist/index.js
CHANGED
|
@@ -18,7 +18,7 @@ import { loadConfig } from "./config.js";
|
|
|
18
18
|
import { logger } from "./middleware/logger.js";
|
|
19
19
|
import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/environment-custom-image-terminal-ws.js";
|
|
20
20
|
import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";
|
|
21
|
-
import { feedbackService, backfillPrincipalAccessCompatibility, bootstrapExecutionPolicyFromEnv, environmentCustomImageService, heartbeatService, instanceSettingsService, reconcileCloudUpstreamRunsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, } from "./services/index.js";
|
|
21
|
+
import { feedbackService, backfillPrincipalAccessCompatibility, bootstrapExecutionPolicyFromEnv, environmentCustomImageService, heartbeatService, instanceSettingsService, reconcileCloudUpstreamRunsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, resolveHeartbeatSchedulingSuppression, routineService, } from "./services/index.js";
|
|
22
22
|
import { parseAdapterRegistryEnv, reconcileAdapterAvailability, } from "./services/adapter-registry-bootstrap.js";
|
|
23
23
|
import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-share-client.js";
|
|
24
24
|
import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js";
|
|
@@ -628,77 +628,85 @@ export async function startServer() {
|
|
|
628
628
|
const heartbeat = heartbeatService(db, { pluginWorkerManager });
|
|
629
629
|
const environmentCustomImages = environmentCustomImageService(db, { pluginWorkerManager });
|
|
630
630
|
const routines = routineService(db, { pluginWorkerManager });
|
|
631
|
+
const heartbeatSchedulingSuppression = resolveHeartbeatSchedulingSuppression();
|
|
631
632
|
// Reap orphaned runs before timer ticks start so wakeups cannot coalesce
|
|
632
633
|
// into a dead "running" row during startup recovery.
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
634
|
+
if (heartbeatSchedulingSuppression.suppressed) {
|
|
635
|
+
logger.warn({ reason: heartbeatSchedulingSuppression.reason }, "heartbeat scheduling suppressed for this runtime instance");
|
|
636
|
+
}
|
|
637
|
+
else {
|
|
638
|
+
await (async () => {
|
|
639
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
640
|
+
try {
|
|
641
|
+
const result = await heartbeat.reapOrphanedRuns();
|
|
642
|
+
logger.info({ reaped: result.reaped, runIds: result.runIds }, "startup reap of orphaned heartbeat runs complete");
|
|
643
|
+
break;
|
|
643
644
|
}
|
|
644
|
-
|
|
645
|
-
|
|
645
|
+
catch (err) {
|
|
646
|
+
if (attempt < 2) {
|
|
647
|
+
logger.warn({ err, attempt }, "startup reap failed, retrying");
|
|
648
|
+
}
|
|
649
|
+
else {
|
|
650
|
+
logger.error({ err }, "startup reap of orphaned heartbeat runs failed after retry — periodic reaper will serve as degraded backstop");
|
|
651
|
+
}
|
|
646
652
|
}
|
|
647
653
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
logger.
|
|
686
|
-
}
|
|
654
|
+
const promotion = await heartbeat.promoteDueScheduledRetries();
|
|
655
|
+
await heartbeat.resumeQueuedRuns();
|
|
656
|
+
const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
|
|
657
|
+
if (promotion.promoted > 0 ||
|
|
658
|
+
reconciled.assignmentDispatched > 0 ||
|
|
659
|
+
reconciled.dispatchRequeued > 0 ||
|
|
660
|
+
reconciled.continuationRequeued > 0 ||
|
|
661
|
+
reconciled.successfulRunHandoffEscalated > 0 ||
|
|
662
|
+
reconciled.escalated > 0) {
|
|
663
|
+
logger.warn({ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, "startup heartbeat recovery changed assigned issue state");
|
|
664
|
+
}
|
|
665
|
+
const issueGraphReconciled = await heartbeat.reconcileIssueGraphLiveness();
|
|
666
|
+
if (issueGraphReconciled.escalationsCreated > 0 || issueGraphReconciled.dependencyWakesHealed > 0) {
|
|
667
|
+
logger.warn({ ...issueGraphReconciled }, "startup issue-graph liveness reconciliation changed issue graph state");
|
|
668
|
+
}
|
|
669
|
+
const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs();
|
|
670
|
+
if (taskWatchdogsReconciled.triggered > 0) {
|
|
671
|
+
logger.warn({ ...taskWatchdogsReconciled }, "startup task-watchdog reconciliation triggered watchdog work");
|
|
672
|
+
}
|
|
673
|
+
const scanned = await heartbeat.scanSilentActiveRuns();
|
|
674
|
+
if (scanned.created > 0 || scanned.escalated > 0) {
|
|
675
|
+
logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
|
|
676
|
+
}
|
|
677
|
+
const swept = await heartbeat.sweepStaleIssueLocks();
|
|
678
|
+
if (swept.cleared > 0) {
|
|
679
|
+
logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks");
|
|
680
|
+
}
|
|
681
|
+
const reviewed = await heartbeat.reconcileProductivityReviews();
|
|
682
|
+
if (reviewed.created > 0 || reviewed.updated > 0 || reviewed.failed > 0) {
|
|
683
|
+
logger.warn({ ...reviewed }, "startup productivity reconciliation created or updated review work");
|
|
684
|
+
}
|
|
685
|
+
})().catch((err) => {
|
|
686
|
+
logger.error({ err }, "startup heartbeat recovery failed");
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
const setupCleanup = await environmentCustomImages.cleanupExpiredSetupSessions();
|
|
690
|
+
if (setupCleanup.timedOut > 0 || setupCleanup.failed > 0) {
|
|
691
|
+
logger.warn({ ...setupCleanup }, "startup environment customImage setup cleanup changed sessions");
|
|
692
|
+
}
|
|
687
693
|
setInterval(() => {
|
|
688
694
|
const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses();
|
|
689
695
|
if (sweptRuntimeStatuses > 0) {
|
|
690
696
|
logger.info({ swept: sweptRuntimeStatuses }, "heartbeat runtime-status sweeper cleared expired entries");
|
|
691
697
|
}
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
698
|
+
if (!resolveHeartbeatSchedulingSuppression().suppressed) {
|
|
699
|
+
void heartbeat
|
|
700
|
+
.tickTimers(new Date())
|
|
701
|
+
.then((result) => {
|
|
702
|
+
if (result.enqueued > 0) {
|
|
703
|
+
logger.info({ ...result }, "heartbeat timer tick enqueued runs");
|
|
704
|
+
}
|
|
705
|
+
})
|
|
706
|
+
.catch((err) => {
|
|
707
|
+
logger.error({ err }, "heartbeat timer tick failed");
|
|
708
|
+
});
|
|
709
|
+
}
|
|
702
710
|
void routines
|
|
703
711
|
.tickScheduledTriggers(new Date())
|
|
704
712
|
.then((result) => {
|
|
@@ -719,56 +727,58 @@ export async function startServer() {
|
|
|
719
727
|
.catch((err) => {
|
|
720
728
|
logger.error({ err }, "environment customImage setup cleanup failed");
|
|
721
729
|
});
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
730
|
+
if (!resolveHeartbeatSchedulingSuppression().suppressed) {
|
|
731
|
+
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
|
|
732
|
+
// persisted queued work is still being driven forward.
|
|
733
|
+
void heartbeat
|
|
734
|
+
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 })
|
|
735
|
+
.then(() => heartbeat.promoteDueScheduledRetries())
|
|
736
|
+
.then(async (promotion) => {
|
|
737
|
+
await heartbeat.resumeQueuedRuns();
|
|
738
|
+
const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
|
|
739
|
+
if (promotion.promoted > 0 ||
|
|
740
|
+
reconciled.assignmentDispatched > 0 ||
|
|
741
|
+
reconciled.dispatchRequeued > 0 ||
|
|
742
|
+
reconciled.continuationRequeued > 0 ||
|
|
743
|
+
reconciled.successfulRunHandoffEscalated > 0 ||
|
|
744
|
+
reconciled.escalated > 0) {
|
|
745
|
+
logger.warn({ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, "periodic heartbeat recovery changed assigned issue state");
|
|
746
|
+
}
|
|
747
|
+
})
|
|
748
|
+
.then(async () => {
|
|
749
|
+
const reconciled = await heartbeat.reconcileIssueGraphLiveness();
|
|
750
|
+
if (reconciled.escalationsCreated > 0 || reconciled.dependencyWakesHealed > 0) {
|
|
751
|
+
logger.warn({ ...reconciled }, "periodic issue-graph liveness reconciliation changed issue graph state");
|
|
752
|
+
}
|
|
753
|
+
})
|
|
754
|
+
.then(async () => {
|
|
755
|
+
const reconciled = await heartbeat.reconcileTaskWatchdogs();
|
|
756
|
+
if (reconciled.triggered > 0) {
|
|
757
|
+
logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work");
|
|
758
|
+
}
|
|
759
|
+
})
|
|
760
|
+
.then(async () => {
|
|
761
|
+
const scanned = await heartbeat.scanSilentActiveRuns();
|
|
762
|
+
if (scanned.created > 0 || scanned.escalated > 0) {
|
|
763
|
+
logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
|
|
764
|
+
}
|
|
765
|
+
})
|
|
766
|
+
.then(async () => {
|
|
767
|
+
const swept = await heartbeat.sweepStaleIssueLocks();
|
|
768
|
+
if (swept.cleared > 0) {
|
|
769
|
+
logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks");
|
|
770
|
+
}
|
|
771
|
+
})
|
|
772
|
+
.then(async () => {
|
|
773
|
+
const reviewed = await heartbeat.reconcileProductivityReviews();
|
|
774
|
+
if (reviewed.created > 0 || reviewed.updated > 0 || reviewed.failed > 0) {
|
|
775
|
+
logger.warn({ ...reviewed }, "periodic productivity reconciliation created or updated review work");
|
|
776
|
+
}
|
|
777
|
+
})
|
|
778
|
+
.catch((err) => {
|
|
779
|
+
logger.error({ err }, "periodic heartbeat recovery failed");
|
|
780
|
+
});
|
|
781
|
+
}
|
|
772
782
|
}, config.heartbeatSchedulerIntervalMs);
|
|
773
783
|
}
|
|
774
784
|
if (config.databaseBackupEnabled) {
|