crewly 1.20.226 → 1.20.228
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/backend/src/constants.d.ts +30 -1
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +30 -1
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/services/agent/claude-runtime.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/claude-runtime.service.js +15 -0
- package/dist/backend/backend/src/services/agent/claude-runtime.service.js.map +1 -1
- package/dist/backend/backend/src/services/apps/app-wake.service.d.ts +62 -2
- package/dist/backend/backend/src/services/apps/app-wake.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/apps/app-wake.service.js +305 -44
- package/dist/backend/backend/src/services/apps/app-wake.service.js.map +1 -1
- package/dist/backend/backend/src/services/apps/apps.wiring.d.ts.map +1 -1
- package/dist/backend/backend/src/services/apps/apps.wiring.js +10 -1
- package/dist/backend/backend/src/services/apps/apps.wiring.js.map +1 -1
- package/dist/backend/backend/src/services/session/input-guard-known-screens.d.ts +8 -0
- package/dist/backend/backend/src/services/session/input-guard-known-screens.d.ts.map +1 -1
- package/dist/backend/backend/src/services/session/input-guard-known-screens.js +100 -0
- package/dist/backend/backend/src/services/session/input-guard-known-screens.js.map +1 -1
- package/dist/backend/backend/src/services/session/tui-input-guard.d.ts +6 -0
- package/dist/backend/backend/src/services/session/tui-input-guard.d.ts.map +1 -1
- package/dist/backend/backend/src/services/session/tui-input-guard.js +24 -5
- package/dist/backend/backend/src/services/session/tui-input-guard.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +30 -1
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +30 -1
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/claude-runtime.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/claude-runtime.service.js +15 -0
- package/dist/cli/backend/src/services/agent/claude-runtime.service.js.map +1 -1
- package/dist/cli/backend/src/services/session/input-guard-known-screens.d.ts +8 -0
- package/dist/cli/backend/src/services/session/input-guard-known-screens.d.ts.map +1 -1
- package/dist/cli/backend/src/services/session/input-guard-known-screens.js +100 -0
- package/dist/cli/backend/src/services/session/input-guard-known-screens.js.map +1 -1
- package/dist/cli/backend/src/services/session/tui-input-guard.d.ts +6 -0
- package/dist/cli/backend/src/services/session/tui-input-guard.d.ts.map +1 -1
- package/dist/cli/backend/src/services/session/tui-input-guard.js +24 -5
- package/dist/cli/backend/src/services/session/tui-input-guard.js.map +1 -1
- package/package.json +1 -1
|
@@ -27,6 +27,26 @@
|
|
|
27
27
|
* - A failed delivery keeps the batch (and the cursor before it) and is
|
|
28
28
|
* retried with backoff; after WAKE_FAILS_BEFORE_ORC_NOTICE failures the
|
|
29
29
|
* orchestrator is told once.
|
|
30
|
+
* - Nothing may stall the loop (2026-10-05, 科技晨报: one poll pass never
|
|
31
|
+
* finished, the self-rescheduling loop stopped for good, and five owner
|
|
32
|
+
* comments sat in Cloud for hours with nothing logged). Each app's poll
|
|
33
|
+
* has a deadline (POLL_APP_DEADLINE_MS; a late result is discarded), a
|
|
34
|
+
* pass that still has not finished after POLL_PASS_STALL_MS is abandoned
|
|
35
|
+
* and a fresh one starts, and a delivery that has not settled after
|
|
36
|
+
* WAKE_DELIVER_TIMEOUT_MS counts as failed and is retried (a late success
|
|
37
|
+
* still settles the batch).
|
|
38
|
+
* - A batch being delivered is detached: changes arriving meanwhile open the
|
|
39
|
+
* next batch instead of being marked delivered without being in the
|
|
40
|
+
* message.
|
|
41
|
+
* - A watchdog on every scheduled pass re-arms a batch whose timer is gone
|
|
42
|
+
* and, once per batch, logs and tells the orchestrator about owner changes
|
|
43
|
+
* that have not reached their agent after WAKE_STUCK_NOTICE_MS.
|
|
44
|
+
* - Owner changes are delivered as owner-authored (`owner: true`, crewly#1105
|
|
45
|
+
* queue priority): a busy agent gets them queued ahead of system traffic,
|
|
46
|
+
* not after it goes idle.
|
|
47
|
+
* - Batches live in memory; the persisted cursor stays before the first
|
|
48
|
+
* change of every batch not yet delivered, so after a restart the same
|
|
49
|
+
* changes are read again and batched again.
|
|
30
50
|
* - An `ask` reaches a named agent only when it is in the publisher's team
|
|
31
51
|
* and already running; it never starts a stopped agent.
|
|
32
52
|
*
|
|
@@ -46,6 +66,10 @@ export interface AppWakeClient {
|
|
|
46
66
|
export interface WakeDeliveryOptions {
|
|
47
67
|
/** Start the agent when its session is down (only for the app's publisher) */
|
|
48
68
|
activate: boolean;
|
|
69
|
+
/** The owner made (some of) these changes: queue ahead of system traffic (crewly#1105) */
|
|
70
|
+
owner?: boolean;
|
|
71
|
+
/** Identity of this batch (`app:<id>:<first>-<last>`): a second queued copy is a duplicate */
|
|
72
|
+
ref?: string;
|
|
49
73
|
}
|
|
50
74
|
/** Constructor dependencies. */
|
|
51
75
|
export interface AppWakeServiceDeps {
|
|
@@ -78,7 +102,16 @@ export declare class AppWakeService {
|
|
|
78
102
|
private timer;
|
|
79
103
|
private running;
|
|
80
104
|
private ticking;
|
|
105
|
+
/** When the running pass started, and its generation (an abandoned pass stops at its next step) */
|
|
106
|
+
private tickStartedAt;
|
|
107
|
+
private generation;
|
|
108
|
+
private passStallNotified;
|
|
109
|
+
/** Current poll of each app; a poll whose token was replaced or dropped discards its result */
|
|
110
|
+
private readonly pollTokens;
|
|
111
|
+
/** Batches collecting changes, by `appId\0recipient` */
|
|
81
112
|
private readonly batches;
|
|
113
|
+
/** Batches handed to `deliver` and not yet confirmed (in flight, or failed and waiting to retry) */
|
|
114
|
+
private readonly outgoing;
|
|
82
115
|
private readonly lastWakeAt;
|
|
83
116
|
/** Highest seq read per app (the registry holds what is safe to persist) */
|
|
84
117
|
private readonly fetched;
|
|
@@ -103,10 +136,12 @@ export declare class AppWakeService {
|
|
|
103
136
|
private schedule;
|
|
104
137
|
/**
|
|
105
138
|
* One pass over the apps that are due. Exposed for tests and for an immediate poll.
|
|
139
|
+
* A pass still running after POLL_PASS_STALL_MS is abandoned and a fresh one starts.
|
|
106
140
|
*/
|
|
107
141
|
tick(): Promise<void>;
|
|
108
142
|
private runTick;
|
|
109
143
|
private pollOne;
|
|
144
|
+
private isCurrentPoll;
|
|
110
145
|
private deliveredSet;
|
|
111
146
|
private pollApp;
|
|
112
147
|
/**
|
|
@@ -131,14 +166,39 @@ export declare class AppWakeService {
|
|
|
131
166
|
private recipientFor;
|
|
132
167
|
private handle;
|
|
133
168
|
private arm;
|
|
169
|
+
/** Whether a batch holds only visitor submissions (or only a skipped-submissions notice). */
|
|
170
|
+
private static visitorOnly;
|
|
171
|
+
/** Fold `from` (newer) into `into` (older): one message covers both. */
|
|
172
|
+
private static merge;
|
|
134
173
|
/**
|
|
135
|
-
* Send one batch now (timer callback; tests). On failure the
|
|
136
|
-
* pending and is retried with backoff.
|
|
174
|
+
* Send one collecting batch now (timer callback; tests). On failure the
|
|
175
|
+
* batch stays pending and is retried with backoff.
|
|
137
176
|
*
|
|
138
177
|
* @param key - Batch key
|
|
139
178
|
* @returns Whether a message was delivered
|
|
140
179
|
*/
|
|
141
180
|
flush(key: string): Promise<boolean>;
|
|
181
|
+
/** Retry an outgoing batch (its retry timer), folding in whatever collected meanwhile. */
|
|
182
|
+
private retry;
|
|
183
|
+
/**
|
|
184
|
+
* Deliver an outgoing batch once, bounded by WAKE_DELIVER_TIMEOUT_MS.
|
|
185
|
+
*
|
|
186
|
+
* @param key - Batch key
|
|
187
|
+
* @param batch - The batch (already in `outgoing`)
|
|
188
|
+
* @returns Whether it was delivered
|
|
189
|
+
*/
|
|
190
|
+
private send;
|
|
191
|
+
private succeeded;
|
|
192
|
+
/** Tell the orchestrator something, without waiting on it for long. */
|
|
193
|
+
private notifyOrc;
|
|
194
|
+
/**
|
|
195
|
+
* Runs before every scheduled pass: a batch with no timer and no delivery
|
|
196
|
+
* running is re-armed (it would otherwise wait forever), and owner changes
|
|
197
|
+
* that have not reached their agent after WAKE_STUCK_NOTICE_MS are logged
|
|
198
|
+
* and reported to the orchestrator, once per batch. Visitor-only batches
|
|
199
|
+
* waiting for their agent to run are expected to wait and are left alone.
|
|
200
|
+
*/
|
|
201
|
+
watchdog(): void;
|
|
142
202
|
/** @returns Keys of batches waiting to be sent (tests) */
|
|
143
203
|
pendingKeys(): string[];
|
|
144
204
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app-wake.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/apps/app-wake.service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"app-wake.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/apps/app-wake.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AAGH,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAEhF,OAAO,KAAK,EAAE,mBAAmB,EAAsC,MAAM,4BAA4B,CAAC;AAK1G,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,WAAW,IAAI,OAAO,CAAC;IACvB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CAC1I;AAED,+BAA+B;AAC/B,MAAM,WAAW,mBAAmB;IAClC,8EAA8E;IAC9E,QAAQ,EAAE,OAAO,CAAC;IAClB,0FAA0F;IAC1F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8FAA8F;IAC9F,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,gCAAgC;AAChC,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,aAAa,CAAC;IACtB,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,uFAAuF;IACvF,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/F,wFAAwF;IACxF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3E,gDAAgD;IAChD,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IACzC,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,eAAe,CAAC;CAC1B;AAiED,UAAU,UAAU;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,uEAAuE;AACvE,eAAO,MAAM,aAAa,mBAAuB,CAAC;AAElD;;GAEG;AACH,qBAAa,cAAc;IA0Bb,OAAO,CAAC,QAAQ,CAAC,IAAI;IAzBjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAA8B;IAC7C,mGAAmG;IACnG,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAS;IAClC,+FAA+F;IAC/F,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,wDAAwD;IACxD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,4DAA4D;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;IAC5D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAErB,IAAI,EAAE,kBAAkB;IAKrD,qDAAqD;IACrD,KAAK,IAAI,IAAI;IAMb,gHAAgH;IAChH,IAAI,IAAI,IAAI;IAWZ;;;;;OAKG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IAK5C,OAAO,CAAC,QAAQ;IAchB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAyBb,OAAO;YAmBP,OAAO;IAyBrB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,YAAY;YASN,OAAO;IA4BrB;;;;OAIG;YACW,eAAe;IAe7B,OAAO,CAAC,KAAK;IAIb;;;OAGG;IACH,OAAO,CAAC,YAAY;YAgBN,mBAAmB;IAQjC;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAa9B,OAAO,CAAC,QAAQ;YA0BF,YAAY;YAWZ,MAAM;IAiEpB,OAAO,CAAC,GAAG;IAUX,6FAA6F;IAC7F,OAAO,CAAC,MAAM,CAAC,WAAW;IAI1B,wEAAwE;IACxE,OAAO,CAAC,MAAM,CAAC,KAAK;IAiBpB;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAiC1C,0FAA0F;YAC5E,KAAK;IAcnB;;;;;;OAMG;YACW,IAAI;YAoFJ,SAAS;IAiCvB,uEAAuE;IACvE,OAAO,CAAC,SAAS;IAUjB;;;;;;OAMG;IACH,QAAQ,IAAI,IAAI;IAgChB,0DAA0D;IAC1D,WAAW,IAAI,MAAM,EAAE;CAGxB"}
|
|
@@ -27,6 +27,26 @@
|
|
|
27
27
|
* - A failed delivery keeps the batch (and the cursor before it) and is
|
|
28
28
|
* retried with backoff; after WAKE_FAILS_BEFORE_ORC_NOTICE failures the
|
|
29
29
|
* orchestrator is told once.
|
|
30
|
+
* - Nothing may stall the loop (2026-10-05, 科技晨报: one poll pass never
|
|
31
|
+
* finished, the self-rescheduling loop stopped for good, and five owner
|
|
32
|
+
* comments sat in Cloud for hours with nothing logged). Each app's poll
|
|
33
|
+
* has a deadline (POLL_APP_DEADLINE_MS; a late result is discarded), a
|
|
34
|
+
* pass that still has not finished after POLL_PASS_STALL_MS is abandoned
|
|
35
|
+
* and a fresh one starts, and a delivery that has not settled after
|
|
36
|
+
* WAKE_DELIVER_TIMEOUT_MS counts as failed and is retried (a late success
|
|
37
|
+
* still settles the batch).
|
|
38
|
+
* - A batch being delivered is detached: changes arriving meanwhile open the
|
|
39
|
+
* next batch instead of being marked delivered without being in the
|
|
40
|
+
* message.
|
|
41
|
+
* - A watchdog on every scheduled pass re-arms a batch whose timer is gone
|
|
42
|
+
* and, once per batch, logs and tells the orchestrator about owner changes
|
|
43
|
+
* that have not reached their agent after WAKE_STUCK_NOTICE_MS.
|
|
44
|
+
* - Owner changes are delivered as owner-authored (`owner: true`, crewly#1105
|
|
45
|
+
* queue priority): a busy agent gets them queued ahead of system traffic,
|
|
46
|
+
* not after it goes idle.
|
|
47
|
+
* - Batches live in memory; the persisted cursor stays before the first
|
|
48
|
+
* change of every batch not yet delivered, so after a restart the same
|
|
49
|
+
* changes are read again and batched again.
|
|
30
50
|
* - An `ask` reaches a named agent only when it is in the publisher's team
|
|
31
51
|
* and already running; it never starts a stopped agent.
|
|
32
52
|
*
|
|
@@ -37,6 +57,32 @@ import { LoggerService } from '../core/logger.service.js';
|
|
|
37
57
|
import { AppsCloudError } from './apps-cloud.client.js';
|
|
38
58
|
import { buildAppWakeMessage, safeAppName } from './app-wake-message.js';
|
|
39
59
|
const C = CREWLY_APPS_CONSTANTS;
|
|
60
|
+
/** Settles like `promise`, or like `onTimeout()` (its value, or what it throws) after `ms` — whichever comes first. */
|
|
61
|
+
function withDeadline(promise, ms, onTimeout) {
|
|
62
|
+
let timer = null;
|
|
63
|
+
const deadline = new Promise((resolve, reject) => {
|
|
64
|
+
timer = setTimeout(() => {
|
|
65
|
+
try {
|
|
66
|
+
resolve(onTimeout());
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
reject(err);
|
|
70
|
+
}
|
|
71
|
+
}, ms);
|
|
72
|
+
timer.unref?.();
|
|
73
|
+
});
|
|
74
|
+
return Promise.race([promise, deadline]).finally(() => {
|
|
75
|
+
if (timer)
|
|
76
|
+
clearTimeout(timer);
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/** Thrown when one app's poll misses its deadline. */
|
|
80
|
+
class PollDeadlineError extends Error {
|
|
81
|
+
constructor(ms) {
|
|
82
|
+
super(`poll did not finish within ${Math.round(ms / 1000)} s`);
|
|
83
|
+
this.name = 'PollDeadlineError';
|
|
84
|
+
}
|
|
85
|
+
}
|
|
40
86
|
/** Recipient key for the orchestrator (never a valid session name). */
|
|
41
87
|
export const ORC_RECIPIENT = '\u0000orchestrator';
|
|
42
88
|
/**
|
|
@@ -49,7 +95,16 @@ export class AppWakeService {
|
|
|
49
95
|
timer = null;
|
|
50
96
|
running = false;
|
|
51
97
|
ticking = null;
|
|
98
|
+
/** When the running pass started, and its generation (an abandoned pass stops at its next step) */
|
|
99
|
+
tickStartedAt = 0;
|
|
100
|
+
generation = 0;
|
|
101
|
+
passStallNotified = false;
|
|
102
|
+
/** Current poll of each app; a poll whose token was replaced or dropped discards its result */
|
|
103
|
+
pollTokens = new Map();
|
|
104
|
+
/** Batches collecting changes, by `appId\0recipient` */
|
|
52
105
|
batches = new Map();
|
|
106
|
+
/** Batches handed to `deliver` and not yet confirmed (in flight, or failed and waiting to retry) */
|
|
107
|
+
outgoing = new Map();
|
|
53
108
|
lastWakeAt = new Map();
|
|
54
109
|
/** Highest seq read per app (the registry holds what is safe to persist) */
|
|
55
110
|
fetched = new Map();
|
|
@@ -77,10 +132,13 @@ export class AppWakeService {
|
|
|
77
132
|
if (this.timer)
|
|
78
133
|
clearTimeout(this.timer);
|
|
79
134
|
this.timer = null;
|
|
80
|
-
|
|
135
|
+
this.generation++;
|
|
136
|
+
for (const b of [...this.batches.values(), ...this.outgoing.values()])
|
|
81
137
|
if (b.timer)
|
|
82
138
|
clearTimeout(b.timer);
|
|
83
139
|
this.batches.clear();
|
|
140
|
+
this.outgoing.clear();
|
|
141
|
+
this.pollTokens.clear();
|
|
84
142
|
}
|
|
85
143
|
/**
|
|
86
144
|
* Per-app backoff state (tests).
|
|
@@ -97,29 +155,57 @@ export class AppWakeService {
|
|
|
97
155
|
return;
|
|
98
156
|
this.timer = setTimeout(() => {
|
|
99
157
|
this.timer = null;
|
|
100
|
-
|
|
158
|
+
this.watchdog();
|
|
159
|
+
// The loop never waits on a pass for longer than the stall limit: the
|
|
160
|
+
// next scheduled pass then abandons it and starts afresh.
|
|
161
|
+
void withDeadline(this.tick(), C.POLL_PASS_STALL_MS, () => undefined)
|
|
162
|
+
.catch((err) => this.logger.warn('App change poll pass failed', { error: err instanceof Error ? err.message : String(err) }))
|
|
163
|
+
.finally(() => this.schedule());
|
|
101
164
|
}, C.POLL_INTERVAL_MS);
|
|
102
165
|
this.timer.unref?.();
|
|
103
166
|
}
|
|
104
167
|
/**
|
|
105
168
|
* One pass over the apps that are due. Exposed for tests and for an immediate poll.
|
|
169
|
+
* A pass still running after POLL_PASS_STALL_MS is abandoned and a fresh one starts.
|
|
106
170
|
*/
|
|
107
171
|
async tick() {
|
|
108
|
-
if (this.ticking)
|
|
109
|
-
|
|
110
|
-
|
|
172
|
+
if (this.ticking) {
|
|
173
|
+
const runningFor = this.now() - this.tickStartedAt;
|
|
174
|
+
if (runningFor < C.POLL_PASS_STALL_MS)
|
|
175
|
+
return this.ticking;
|
|
176
|
+
this.logger.warn('App change poll pass did not finish; abandoning it and starting a fresh one', { runningForMs: runningFor });
|
|
177
|
+
if (!this.passStallNotified) {
|
|
178
|
+
this.passStallNotified = true;
|
|
179
|
+
this.notifyOrc(`[APP CHANGES] Crewly's poll of app changes stalled for ${Math.round(runningFor / 60_000)} min and was restarted. ` +
|
|
180
|
+
'Owner changes made in that time are being read again now. If this repeats, the backend may need a restart.');
|
|
181
|
+
}
|
|
182
|
+
this.generation++;
|
|
111
183
|
this.ticking = null;
|
|
184
|
+
}
|
|
185
|
+
const gen = this.generation;
|
|
186
|
+
this.tickStartedAt = this.now();
|
|
187
|
+
const pass = this.runTick(gen).finally(() => {
|
|
188
|
+
if (this.ticking === pass)
|
|
189
|
+
this.ticking = null;
|
|
112
190
|
});
|
|
113
|
-
|
|
191
|
+
this.ticking = pass;
|
|
192
|
+
await pass;
|
|
193
|
+
if (gen === this.generation)
|
|
194
|
+
this.passStallNotified = false;
|
|
114
195
|
}
|
|
115
|
-
async runTick() {
|
|
196
|
+
async runTick(gen) {
|
|
116
197
|
if (!this.deps.client.isAvailable())
|
|
117
198
|
return;
|
|
118
199
|
const now = this.now();
|
|
119
|
-
const
|
|
200
|
+
const all = await withDeadline(this.deps.registry.list(), C.POLL_APP_DEADLINE_MS, () => null);
|
|
201
|
+
if (!all) {
|
|
202
|
+
this.logger.warn('Reading the apps registry did not finish; skipping this poll pass');
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
const due = all.filter((e) => !e.deleted && (this.backoff.get(e.appId)?.nextAt ?? 0) <= now);
|
|
120
206
|
let next = 0;
|
|
121
207
|
const worker = async () => {
|
|
122
|
-
while (next < due.length) {
|
|
208
|
+
while (gen === this.generation && next < due.length) {
|
|
123
209
|
const app = due[next++];
|
|
124
210
|
await this.pollOne(app);
|
|
125
211
|
}
|
|
@@ -127,8 +213,12 @@ export class AppWakeService {
|
|
|
127
213
|
await Promise.all(Array.from({ length: Math.min(C.POLL_CONCURRENCY, due.length) }, worker));
|
|
128
214
|
}
|
|
129
215
|
async pollOne(app) {
|
|
216
|
+
const token = Symbol(app.appId);
|
|
217
|
+
this.pollTokens.set(app.appId, token);
|
|
130
218
|
try {
|
|
131
|
-
await this.pollApp(app)
|
|
219
|
+
await withDeadline(this.pollApp(app, token), C.POLL_APP_DEADLINE_MS, () => {
|
|
220
|
+
throw new PollDeadlineError(C.POLL_APP_DEADLINE_MS);
|
|
221
|
+
});
|
|
132
222
|
this.backoff.delete(app.appId);
|
|
133
223
|
}
|
|
134
224
|
catch (err) {
|
|
@@ -143,6 +233,14 @@ export class AppWakeService {
|
|
|
143
233
|
this.backoff.set(app.appId, { failures, nextAt: this.now() + wait });
|
|
144
234
|
this.logger.warn('Polling app changes failed', { appId: app.appId, failures, retryInMs: wait, error: err instanceof Error ? err.message : String(err) });
|
|
145
235
|
}
|
|
236
|
+
finally {
|
|
237
|
+
// A poll that missed its deadline may still finish later: its token is gone, so it changes nothing.
|
|
238
|
+
if (this.pollTokens.get(app.appId) === token)
|
|
239
|
+
this.pollTokens.delete(app.appId);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
isCurrentPoll(appId, token) {
|
|
243
|
+
return this.pollTokens.get(appId) === token;
|
|
146
244
|
}
|
|
147
245
|
deliveredSet(app) {
|
|
148
246
|
let set = this.delivered.get(app.appId);
|
|
@@ -152,11 +250,13 @@ export class AppWakeService {
|
|
|
152
250
|
}
|
|
153
251
|
return set;
|
|
154
252
|
}
|
|
155
|
-
async pollApp(app) {
|
|
253
|
+
async pollApp(app, token) {
|
|
156
254
|
const opts = { timeoutMs: C.POLL_REQUEST_TIMEOUT_MS };
|
|
157
255
|
if (app.cursor === null || app.cursor === undefined) {
|
|
158
256
|
// Never polled: start from now, never replay history.
|
|
159
257
|
const head = await this.deps.client.request('GET', `/apps/${app.appId}/changes`, opts);
|
|
258
|
+
if (!this.isCurrentPoll(app.appId, token))
|
|
259
|
+
return;
|
|
160
260
|
this.fetched.set(app.appId, head.seq);
|
|
161
261
|
await this.deps.registry.setProgress(app.appId, head.seq, []);
|
|
162
262
|
return;
|
|
@@ -166,6 +266,9 @@ export class AppWakeService {
|
|
|
166
266
|
let since = Math.max(app.cursor, this.fetched.get(app.appId) ?? app.cursor);
|
|
167
267
|
for (let page = 0; page < C.POLL_MAX_PAGES; page++) {
|
|
168
268
|
const res = await this.deps.client.request('GET', `/apps/${app.appId}/changes`, { ...opts, query: { since, wait: 0 } });
|
|
269
|
+
// Missed its deadline (or a newer poll of this app started): discard.
|
|
270
|
+
if (!this.isCurrentPoll(app.appId, token))
|
|
271
|
+
return;
|
|
169
272
|
const changes = Array.isArray(res.changes) ? res.changes : [];
|
|
170
273
|
for (const change of changes) {
|
|
171
274
|
if (typeof change.seq === 'number' && change.seq > since && !done.has(change.seq))
|
|
@@ -188,7 +291,7 @@ export class AppWakeService {
|
|
|
188
291
|
if (fetched === undefined)
|
|
189
292
|
return;
|
|
190
293
|
let safe = fetched;
|
|
191
|
-
for (const b of this.batches.values()) {
|
|
294
|
+
for (const b of [...this.batches.values(), ...this.outgoing.values()]) {
|
|
192
295
|
if (b.appId === appId)
|
|
193
296
|
safe = Math.min(safe, b.firstSeq - 1);
|
|
194
297
|
}
|
|
@@ -243,7 +346,7 @@ export class AppWakeService {
|
|
|
243
346
|
return;
|
|
244
347
|
const session = app.agentSession ?? null;
|
|
245
348
|
const key = `${app.appId}\u0000${session ?? ORC_RECIPIENT}`;
|
|
246
|
-
if (this.batches.has(key))
|
|
349
|
+
if (this.batches.has(key) || this.outgoing.has(key))
|
|
247
350
|
return;
|
|
248
351
|
const batch = this.newBatch(app.appId, session, false, Number.POSITIVE_INFINITY);
|
|
249
352
|
batch.skippedNotice = true;
|
|
@@ -270,6 +373,10 @@ export class AppWakeService {
|
|
|
270
373
|
failures: 0,
|
|
271
374
|
orcNotified: false,
|
|
272
375
|
skippedNotice: false,
|
|
376
|
+
openedAt: this.now(),
|
|
377
|
+
inFlight: false,
|
|
378
|
+
reportSkipped: null,
|
|
379
|
+
stuckNotified: false,
|
|
273
380
|
};
|
|
274
381
|
}
|
|
275
382
|
async recipientFor(app, change) {
|
|
@@ -366,12 +473,39 @@ export class AppWakeService {
|
|
|
366
473
|
arm(key, batch, delay) {
|
|
367
474
|
if (batch.timer)
|
|
368
475
|
clearTimeout(batch.timer);
|
|
369
|
-
batch.timer = setTimeout(() =>
|
|
476
|
+
batch.timer = setTimeout(() => {
|
|
477
|
+
batch.timer = null;
|
|
478
|
+
if (this.outgoing.get(key) === batch)
|
|
479
|
+
void this.retry(key);
|
|
480
|
+
else
|
|
481
|
+
void this.flush(key);
|
|
482
|
+
}, Math.max(0, delay));
|
|
370
483
|
batch.timer.unref?.();
|
|
371
484
|
}
|
|
485
|
+
/** Whether a batch holds only visitor submissions (or only a skipped-submissions notice). */
|
|
486
|
+
static visitorOnly(batch) {
|
|
487
|
+
return batch.dataTotal === 0 && batch.eventsTotal === 0 && batch.commentsTotal === 0;
|
|
488
|
+
}
|
|
489
|
+
/** Fold `from` (newer) into `into` (older): one message covers both. */
|
|
490
|
+
static merge(into, from) {
|
|
491
|
+
const keep = (list, max) => (list.length > max ? list.slice(list.length - max) : list);
|
|
492
|
+
into.activate = into.activate || from.activate;
|
|
493
|
+
into.dataChanges = keep([...into.dataChanges, ...from.dataChanges], C.MAX_BATCH_DATA_CHANGES);
|
|
494
|
+
into.events = keep([...into.events, ...from.events], C.MAX_EVENTS_PER_WAKE);
|
|
495
|
+
into.visitorChanges = keep([...into.visitorChanges, ...from.visitorChanges], C.MAX_BATCH_DATA_CHANGES);
|
|
496
|
+
into.comments = keep([...into.comments, ...from.comments], C.COMMENTS.MAX_PER_WAKE);
|
|
497
|
+
into.dataTotal += from.dataTotal;
|
|
498
|
+
into.eventsTotal += from.eventsTotal;
|
|
499
|
+
into.visitorTotal += from.visitorTotal;
|
|
500
|
+
into.commentsTotal += from.commentsTotal;
|
|
501
|
+
into.seqs = keep([...into.seqs, ...from.seqs], C.MAX_DELIVERED_SEQS);
|
|
502
|
+
into.firstSeq = Math.min(into.firstSeq, from.firstSeq);
|
|
503
|
+
into.skippedNotice = into.skippedNotice && from.skippedNotice;
|
|
504
|
+
into.openedAt = Math.min(into.openedAt, from.openedAt);
|
|
505
|
+
}
|
|
372
506
|
/**
|
|
373
|
-
* Send one batch now (timer callback; tests). On failure the
|
|
374
|
-
* pending and is retried with backoff.
|
|
507
|
+
* Send one collecting batch now (timer callback; tests). On failure the
|
|
508
|
+
* batch stays pending and is retried with backoff.
|
|
375
509
|
*
|
|
376
510
|
* @param key - Batch key
|
|
377
511
|
* @returns Whether a message was delivered
|
|
@@ -383,23 +517,71 @@ export class AppWakeService {
|
|
|
383
517
|
if (batch.timer)
|
|
384
518
|
clearTimeout(batch.timer);
|
|
385
519
|
batch.timer = null;
|
|
386
|
-
|
|
387
|
-
|
|
520
|
+
// The previous batch for this agent is still out: fold this one into it
|
|
521
|
+
// when it is waiting to retry (its retry carries both), else check again
|
|
522
|
+
// shortly.
|
|
523
|
+
const out = this.outgoing.get(key);
|
|
524
|
+
if (out) {
|
|
525
|
+
if (out.inFlight) {
|
|
526
|
+
this.arm(key, batch, C.WAKE_INFLIGHT_RECHECK_MS);
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
this.batches.delete(key);
|
|
530
|
+
AppWakeService.merge(out, batch);
|
|
531
|
+
if (!out.timer)
|
|
532
|
+
this.arm(key, out, 0);
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
388
535
|
// Visitor submissions alone never start the agent: wait until it runs.
|
|
389
|
-
|
|
390
|
-
if (visitorOnly && batch.session !== null && this.deps.isRunning && !this.deps.isRunning(batch.session)) {
|
|
536
|
+
if (AppWakeService.visitorOnly(batch) && batch.session !== null && this.deps.isRunning && !this.deps.isRunning(batch.session)) {
|
|
391
537
|
if (this.batches.get(key) === batch)
|
|
392
538
|
this.arm(key, batch, C.VISITOR_WAKE.PENDING_RECHECK_MS);
|
|
393
539
|
return false;
|
|
394
540
|
}
|
|
541
|
+
// Detach: changes arriving while this one is delivered open the next batch.
|
|
542
|
+
this.batches.delete(key);
|
|
543
|
+
this.outgoing.set(key, batch);
|
|
544
|
+
return this.send(key, batch);
|
|
545
|
+
}
|
|
546
|
+
/** Retry an outgoing batch (its retry timer), folding in whatever collected meanwhile. */
|
|
547
|
+
async retry(key) {
|
|
548
|
+
const batch = this.outgoing.get(key);
|
|
549
|
+
if (!batch || batch.inFlight)
|
|
550
|
+
return false;
|
|
551
|
+
if (batch.timer)
|
|
552
|
+
clearTimeout(batch.timer);
|
|
553
|
+
batch.timer = null;
|
|
554
|
+
const newer = this.batches.get(key);
|
|
555
|
+
if (newer) {
|
|
556
|
+
if (newer.timer)
|
|
557
|
+
clearTimeout(newer.timer);
|
|
558
|
+
this.batches.delete(key);
|
|
559
|
+
AppWakeService.merge(batch, newer);
|
|
560
|
+
}
|
|
561
|
+
return this.send(key, batch);
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Deliver an outgoing batch once, bounded by WAKE_DELIVER_TIMEOUT_MS.
|
|
565
|
+
*
|
|
566
|
+
* @param key - Batch key
|
|
567
|
+
* @param batch - The batch (already in `outgoing`)
|
|
568
|
+
* @returns Whether it was delivered
|
|
569
|
+
*/
|
|
570
|
+
async send(key, batch) {
|
|
571
|
+
batch.inFlight = true;
|
|
572
|
+
const app = await this.deps.registry.get(batch.appId).catch(() => null);
|
|
395
573
|
// Skipped visitor submissions are reported in the next message to the
|
|
396
574
|
// agent visitors wake (the publisher, else the orchestrator).
|
|
397
575
|
const isVisitorRecipient = (app?.agentSession ?? null) === batch.session;
|
|
398
576
|
const vstate = app && isVisitorRecipient ? this.visitorState(app) : null;
|
|
399
|
-
|
|
400
|
-
|
|
577
|
+
if (batch.reportSkipped === null)
|
|
578
|
+
batch.reportSkipped = vstate?.skipped ?? 0;
|
|
579
|
+
const skipped = batch.reportSkipped;
|
|
580
|
+
if (AppWakeService.visitorOnly(batch) && batch.visitorTotal === 0 && skipped === 0) {
|
|
401
581
|
// A notice-only batch with nothing left to report.
|
|
402
|
-
|
|
582
|
+
batch.inFlight = false;
|
|
583
|
+
if (this.outgoing.get(key) === batch)
|
|
584
|
+
this.outgoing.delete(key);
|
|
403
585
|
return false;
|
|
404
586
|
}
|
|
405
587
|
const text = buildAppWakeMessage({
|
|
@@ -417,32 +599,62 @@ export class AppWakeService {
|
|
|
417
599
|
visitorSkipped: skipped,
|
|
418
600
|
skillsPath: this.deps.skillsPath,
|
|
419
601
|
});
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
602
|
+
const owner = !AppWakeService.visitorOnly(batch);
|
|
603
|
+
const seqs = batch.seqs.filter((n) => Number.isFinite(n));
|
|
604
|
+
const ref = seqs.length > 0 ? `app:${batch.appId}:${Math.min(...seqs)}-${Math.max(...seqs)}:${seqs.length}` : undefined;
|
|
605
|
+
const sentSeqs = new Set(batch.seqs);
|
|
606
|
+
const attempt = Promise.resolve()
|
|
607
|
+
.then(() => this.deps.deliver(batch.session, text, { activate: batch.activate, ...(owner ? { owner: true } : {}), ...(ref ? { ref } : {}) }))
|
|
608
|
+
.catch((err) => {
|
|
425
609
|
this.logger.warn('App change wake failed', { appId: batch.appId, session: batch.session, error: err instanceof Error ? err.message : String(err) });
|
|
426
|
-
}
|
|
427
|
-
if (!ok) {
|
|
428
|
-
batch.failures++;
|
|
429
|
-
const retry = Math.min(C.WAKE_RETRY_BASE_MS * 2 ** (batch.failures - 1), C.WAKE_RETRY_MAX_MS);
|
|
430
|
-
this.logger.warn('App change wake was not delivered; will retry', { appId: batch.appId, session: batch.session ?? 'orchestrator', failures: batch.failures, retryInMs: retry });
|
|
431
|
-
if (batch.failures >= C.WAKE_FAILS_BEFORE_ORC_NOTICE && !batch.orcNotified && batch.session !== null) {
|
|
432
|
-
batch.orcNotified = true;
|
|
433
|
-
const notice = `[APP CHANGES] Changes in app "${safeAppName(app?.name ?? batch.appId)}" (${batch.appId}) could not be delivered to ` +
|
|
434
|
-
`${batch.session} after ${batch.failures} tries. Crewly keeps retrying. Check whether that agent is stuck or signed out.`;
|
|
435
|
-
await this.deps.deliver(null, notice, { activate: false }).catch(() => false);
|
|
436
|
-
}
|
|
437
|
-
if (this.batches.get(key) === batch)
|
|
438
|
-
this.arm(key, batch, retry);
|
|
439
610
|
return false;
|
|
611
|
+
});
|
|
612
|
+
const outcome = await withDeadline(attempt.then((ok) => (ok ? 'ok' : 'failed')), C.WAKE_DELIVER_TIMEOUT_MS, () => 'timeout');
|
|
613
|
+
batch.inFlight = false;
|
|
614
|
+
if (this.outgoing.get(key) !== batch)
|
|
615
|
+
return false; // stopped meanwhile
|
|
616
|
+
if (outcome === 'ok')
|
|
617
|
+
return this.succeeded(key, batch, app, vstate, skipped);
|
|
618
|
+
if (outcome === 'timeout') {
|
|
619
|
+
this.logger.warn('App change wake did not finish in time; will retry', {
|
|
620
|
+
appId: batch.appId,
|
|
621
|
+
session: batch.session ?? 'orchestrator',
|
|
622
|
+
timeoutMs: C.WAKE_DELIVER_TIMEOUT_MS,
|
|
623
|
+
});
|
|
624
|
+
// A late success still settles the batch, unless a retry is already out
|
|
625
|
+
// or the batch has grown since (then the retry carries everything).
|
|
626
|
+
void attempt.then((ok) => {
|
|
627
|
+
if (!ok || this.outgoing.get(key) !== batch || batch.inFlight)
|
|
628
|
+
return;
|
|
629
|
+
if (!batch.seqs.every((n) => sentSeqs.has(n)))
|
|
630
|
+
return;
|
|
631
|
+
this.logger.info('A timed-out app change wake arrived after all', { appId: batch.appId, session: batch.session ?? 'orchestrator' });
|
|
632
|
+
if (batch.timer)
|
|
633
|
+
clearTimeout(batch.timer);
|
|
634
|
+
batch.timer = null;
|
|
635
|
+
void this.succeeded(key, batch, app, vstate, skipped);
|
|
636
|
+
});
|
|
440
637
|
}
|
|
441
|
-
|
|
638
|
+
batch.failures++;
|
|
639
|
+
const retry = Math.min(C.WAKE_RETRY_BASE_MS * 2 ** (batch.failures - 1), C.WAKE_RETRY_MAX_MS);
|
|
640
|
+
this.logger.warn('App change wake was not delivered; will retry', { appId: batch.appId, session: batch.session ?? 'orchestrator', failures: batch.failures, retryInMs: retry });
|
|
641
|
+
if (batch.failures >= C.WAKE_FAILS_BEFORE_ORC_NOTICE && !batch.orcNotified && batch.session !== null) {
|
|
642
|
+
batch.orcNotified = true;
|
|
643
|
+
this.notifyOrc(`[APP CHANGES] Changes in app "${safeAppName(app?.name ?? batch.appId)}" (${batch.appId}) could not be delivered to ` +
|
|
644
|
+
`${batch.session} after ${batch.failures} tries. Crewly keeps retrying. Check whether that agent is stuck or signed out.`);
|
|
645
|
+
}
|
|
646
|
+
this.arm(key, batch, retry);
|
|
647
|
+
return false;
|
|
648
|
+
}
|
|
649
|
+
async succeeded(key, batch, app, vstate, skipped) {
|
|
650
|
+
if (this.outgoing.get(key) !== batch)
|
|
651
|
+
return false;
|
|
652
|
+
this.outgoing.delete(key);
|
|
653
|
+
const recipient = batch.session ?? ORC_RECIPIENT;
|
|
442
654
|
const at = this.now();
|
|
443
655
|
this.lastWakeAt.set(key, at);
|
|
444
656
|
await this.deps.registry.setLastWake(batch.appId, recipient, at).catch(() => undefined);
|
|
445
|
-
if (vstate) {
|
|
657
|
+
if (vstate && app) {
|
|
446
658
|
if (batch.visitorTotal > 0 || batch.skippedNotice)
|
|
447
659
|
vstate.count++;
|
|
448
660
|
vstate.skipped = Math.max(0, vstate.skipped - skipped);
|
|
@@ -461,13 +673,62 @@ export class AppWakeService {
|
|
|
461
673
|
comments: batch.commentsTotal,
|
|
462
674
|
visitorSubmissions: batch.visitorTotal,
|
|
463
675
|
...(skipped > 0 ? { visitorSkippedReported: skipped } : {}),
|
|
676
|
+
...(batch.failures > 0 ? { afterFailures: batch.failures } : {}),
|
|
464
677
|
});
|
|
678
|
+
// Changes that arrived during the delivery wait for the cooldown it started.
|
|
679
|
+
const next = this.batches.get(key);
|
|
680
|
+
if (next)
|
|
681
|
+
this.arm(key, next, C.COOLDOWN_MS);
|
|
465
682
|
await this.persistProgress(batch.appId).catch(() => undefined);
|
|
466
683
|
return true;
|
|
467
684
|
}
|
|
685
|
+
/** Tell the orchestrator something, without waiting on it for long. */
|
|
686
|
+
notifyOrc(text) {
|
|
687
|
+
void withDeadline(Promise.resolve()
|
|
688
|
+
.then(() => this.deps.deliver(null, text, { activate: false }))
|
|
689
|
+
.catch(() => false), C.WAKE_DELIVER_TIMEOUT_MS, () => false);
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Runs before every scheduled pass: a batch with no timer and no delivery
|
|
693
|
+
* running is re-armed (it would otherwise wait forever), and owner changes
|
|
694
|
+
* that have not reached their agent after WAKE_STUCK_NOTICE_MS are logged
|
|
695
|
+
* and reported to the orchestrator, once per batch. Visitor-only batches
|
|
696
|
+
* waiting for their agent to run are expected to wait and are left alone.
|
|
697
|
+
*/
|
|
698
|
+
watchdog() {
|
|
699
|
+
const now = this.now();
|
|
700
|
+
for (const [key, batch] of [...this.batches.entries(), ...this.outgoing.entries()]) {
|
|
701
|
+
if (!batch.timer && !batch.inFlight) {
|
|
702
|
+
this.logger.warn('App change batch had no timer; re-armed', { appId: batch.appId, session: batch.session ?? 'orchestrator' });
|
|
703
|
+
this.arm(key, batch, 0);
|
|
704
|
+
}
|
|
705
|
+
if (AppWakeService.visitorOnly(batch) || batch.stuckNotified)
|
|
706
|
+
continue;
|
|
707
|
+
const ageMs = now - batch.openedAt;
|
|
708
|
+
if (ageMs < C.WAKE_STUCK_NOTICE_MS)
|
|
709
|
+
continue;
|
|
710
|
+
batch.stuckNotified = true;
|
|
711
|
+
const minutes = Math.round(ageMs / 60_000);
|
|
712
|
+
this.logger.warn('Owner changes in an app have not reached their agent', {
|
|
713
|
+
appId: batch.appId,
|
|
714
|
+
session: batch.session ?? 'orchestrator',
|
|
715
|
+
minutes,
|
|
716
|
+
inFlight: batch.inFlight,
|
|
717
|
+
failures: batch.failures,
|
|
718
|
+
comments: batch.commentsTotal,
|
|
719
|
+
dataChanges: batch.dataTotal,
|
|
720
|
+
events: batch.eventsTotal,
|
|
721
|
+
});
|
|
722
|
+
if (batch.session !== null) {
|
|
723
|
+
this.notifyOrc(`[APP CHANGES] The owner's changes in an app (${batch.appId}: ${batch.commentsTotal} comment(s), ${batch.dataTotal} data change(s), ` +
|
|
724
|
+
`${batch.eventsTotal} message(s)) have waited ${minutes} min to reach ${batch.session}. Crewly keeps trying. ` +
|
|
725
|
+
'Check whether that agent is stuck.');
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
}
|
|
468
729
|
/** @returns Keys of batches waiting to be sent (tests) */
|
|
469
730
|
pendingKeys() {
|
|
470
|
-
return [...this.batches.keys()];
|
|
731
|
+
return [...new Set([...this.batches.keys(), ...this.outgoing.keys()])];
|
|
471
732
|
}
|
|
472
733
|
}
|
|
473
734
|
//# sourceMappingURL=app-wake.service.js.map
|