deepline 0.3.113 → 0.3.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +3 -1
- package/dist/bundling-sources/sdk/src/compat.ts +167 -11
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/skills-version.ts +27 -1
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +6 -2
- package/dist/bundling-sources/shared_libs/observability/queue-health.ts +291 -6
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +24 -24
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +2 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +29 -1
- package/dist/bundling-sources/shared_libs/play-runtime/admission-transport-policy.ts +91 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/direct-admission-deadline.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +325 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +134 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-admission-probe.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-identity.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-cron-fire-task.ts +100 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +313 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +111 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +67 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1394 -22
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +74 -13
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +79 -15
- package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +21 -21
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +64 -0
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +20 -1
- package/dist/bundling-sources/shared_libs/security/receipt-admission-token.ts +224 -0
- package/dist/cli/index.js +574 -114
- package/dist/cli/index.mjs +592 -133
- package/dist/index.d.mts +54 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.js +8 -3
- package/dist/index.mjs +8 -3
- package/dist/install-integrity.json +7 -0
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -558,6 +558,8 @@ function resolveToolExecuteTimeoutMs(
|
|
|
558
558
|
normalized === 'deeplineagent_deeplineagent' ||
|
|
559
559
|
normalized === 'ai_inference' ||
|
|
560
560
|
normalized === 'deeplineagent_ai_inference' ||
|
|
561
|
+
normalized === 'ai_evaluate' ||
|
|
562
|
+
normalized === 'deeplineagent_ai_evaluate' ||
|
|
561
563
|
normalized === 'aiinference'
|
|
562
564
|
? DEEPLINEAGENT_EXECUTE_TIMEOUT_MS
|
|
563
565
|
: undefined;
|
|
@@ -2782,7 +2784,7 @@ export class DeeplineClient {
|
|
|
2782
2784
|
* maximum and contains Deepline credits only.
|
|
2783
2785
|
*/
|
|
2784
2786
|
async quoteInferenceTool(
|
|
2785
|
-
toolId: 'ai_inference' | 'deeplineagent',
|
|
2787
|
+
toolId: 'ai_evaluate' | 'ai_inference' | 'deeplineagent',
|
|
2786
2788
|
payload: Record<string, unknown>,
|
|
2787
2789
|
): Promise<InferenceQuote> {
|
|
2788
2790
|
return this.http.post<InferenceQuote>(
|
|
@@ -62,7 +62,28 @@ export type SdkCompatibilitySkillsUpdate = {
|
|
|
62
62
|
};
|
|
63
63
|
|
|
64
64
|
const CHECK_TIMEOUT_MS = 2_000;
|
|
65
|
-
|
|
65
|
+
/**
|
|
66
|
+
* A cached clean policy younger than this is trusted outright: the command
|
|
67
|
+
* runs without a live request. Policy floors move on a release cadence; the
|
|
68
|
+
* one control designed to move in minutes, a forced rollback, is delayed by
|
|
69
|
+
* at most this window and only for commands run within it. The live check
|
|
70
|
+
* is best-effort already (a network failure runs the command with no policy),
|
|
71
|
+
* so trusting a fresh clean answer is strictly stronger than that path.
|
|
72
|
+
*/
|
|
73
|
+
export const SDK_COMPATIBILITY_TRUST_MS = 10 * 60 * 1000;
|
|
74
|
+
/**
|
|
75
|
+
* A cached entry older than the trust window but younger than this still
|
|
76
|
+
* qualifies for the short live budget and for the offline fallback.
|
|
77
|
+
*/
|
|
78
|
+
export const SDK_COMPATIBILITY_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
79
|
+
/**
|
|
80
|
+
* Blocking budget for the live compatibility check when a fresh, clean policy
|
|
81
|
+
* for the same command is already cached. The live check still runs and still
|
|
82
|
+
* wins whenever the server answers inside this window; on a slow or proxied
|
|
83
|
+
* network the command proceeds on the cached policy instead of waiting the
|
|
84
|
+
* full timeout. Blocking policies never use this budget.
|
|
85
|
+
*/
|
|
86
|
+
export const SDK_COMPATIBILITY_FAST_DEADLINE_MS = 400;
|
|
66
87
|
|
|
67
88
|
type CompatCacheFile = {
|
|
68
89
|
entries?: Record<
|
|
@@ -118,11 +139,83 @@ function legacyCompatibilityCacheKey(
|
|
|
118
139
|
});
|
|
119
140
|
}
|
|
120
141
|
|
|
142
|
+
/**
|
|
143
|
+
* A cached policy is "clean" when it carries no blocking or mutating control:
|
|
144
|
+
* proceeding on it can at most delay an advisory notice, never bypass a
|
|
145
|
+
* required update, a server-directed auto-update, or a CLI-family rollback.
|
|
146
|
+
*/
|
|
147
|
+
export function isCleanCompatibilityPolicy(
|
|
148
|
+
response: SdkCompatibilityResponse | null,
|
|
149
|
+
): boolean {
|
|
150
|
+
if (!response) return false;
|
|
151
|
+
// `ok` is the server's own "this is a policy" marker (426 otherwise); a
|
|
152
|
+
// proxy error body or other junk JSON never earns the trusted tiers.
|
|
153
|
+
return (
|
|
154
|
+
response.ok === true &&
|
|
155
|
+
response.update_required !== true &&
|
|
156
|
+
response.status !== 'unsupported' &&
|
|
157
|
+
!response.auto_update?.should_auto_update &&
|
|
158
|
+
!response.cli_family
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The cached response for this exact command, only when it is inside the
|
|
164
|
+
* cache TTL and clean. Used to decide whether the live check may run with the
|
|
165
|
+
* short deadline instead of its full timeout.
|
|
166
|
+
*/
|
|
167
|
+
export function readFreshCleanCompatibility(
|
|
168
|
+
baseUrl: string,
|
|
169
|
+
command: string | null | undefined,
|
|
170
|
+
skillsVersion: string | null | undefined,
|
|
171
|
+
): SdkCompatibilityResponse | null {
|
|
172
|
+
const cached = readCachedCompatibility(baseUrl, command, skillsVersion);
|
|
173
|
+
return isCleanCompatibilityPolicy(cached) ? cached : null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The cached response for this exact command when it is clean and younger
|
|
178
|
+
* than the trust window: the command may run on it without a live request.
|
|
179
|
+
*/
|
|
180
|
+
export function readTrustedCleanCompatibility(
|
|
181
|
+
baseUrl: string,
|
|
182
|
+
command: string | null | undefined,
|
|
183
|
+
skillsVersion: string | null | undefined,
|
|
184
|
+
now = Date.now(),
|
|
185
|
+
): SdkCompatibilityResponse | null {
|
|
186
|
+
const entry = readCachedCompatibilityEntry(baseUrl, command, skillsVersion);
|
|
187
|
+
if (!entry || !isSavedWithin(entry.savedAt, SDK_COMPATIBILITY_TRUST_MS, now))
|
|
188
|
+
return null;
|
|
189
|
+
return isCleanCompatibilityPolicy(entry.response) ? entry.response : null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* A cache entry is usable only when its age is a real, non-negative number
|
|
194
|
+
* inside the window. A missing timestamp (NaN) or a future one (after a clock
|
|
195
|
+
* correction) would otherwise pass a plain `age > window` check forever.
|
|
196
|
+
*/
|
|
197
|
+
function isSavedWithin(savedAt: unknown, windowMs: number, now: number) {
|
|
198
|
+
if (typeof savedAt !== 'number') return false;
|
|
199
|
+
const age = now - savedAt;
|
|
200
|
+
return Number.isFinite(age) && age >= 0 && age <= windowMs;
|
|
201
|
+
}
|
|
202
|
+
|
|
121
203
|
function readCachedCompatibility(
|
|
122
204
|
baseUrl: string,
|
|
123
205
|
command: string | null | undefined,
|
|
124
206
|
skillsVersion: string | null | undefined,
|
|
125
207
|
): SdkCompatibilityResponse | null {
|
|
208
|
+
return (
|
|
209
|
+
readCachedCompatibilityEntry(baseUrl, command, skillsVersion)?.response ??
|
|
210
|
+
null
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function readCachedCompatibilityEntry(
|
|
215
|
+
baseUrl: string,
|
|
216
|
+
command: string | null | undefined,
|
|
217
|
+
skillsVersion: string | null | undefined,
|
|
218
|
+
): { savedAt: number; response: SdkCompatibilityResponse } | null {
|
|
126
219
|
try {
|
|
127
220
|
const path = sdkCompatibilityCachePath(baseUrl);
|
|
128
221
|
const legacyPath = legacySdkCompatibilityCachePath();
|
|
@@ -141,15 +234,50 @@ function readCachedCompatibility(
|
|
|
141
234
|
(useLegacyCache
|
|
142
235
|
? parsed.entries?.[legacyCompatibilityCacheKey(baseUrl, command)]
|
|
143
236
|
: undefined);
|
|
144
|
-
if (
|
|
237
|
+
if (
|
|
238
|
+
!entry ||
|
|
239
|
+
!isSavedWithin(entry.savedAt, SDK_COMPATIBILITY_CACHE_TTL_MS, Date.now())
|
|
240
|
+
) {
|
|
145
241
|
return null;
|
|
146
242
|
}
|
|
147
|
-
return entry
|
|
243
|
+
return entry;
|
|
148
244
|
} catch {
|
|
149
245
|
return null;
|
|
150
246
|
}
|
|
151
247
|
}
|
|
152
248
|
|
|
249
|
+
/**
|
|
250
|
+
* Drop one cached entry so the next check for this command runs with the full
|
|
251
|
+
* budget. Used after a short-deadline check fell back to the cache: the
|
|
252
|
+
* command proceeds on the cached clean policy once, but a blocking policy the
|
|
253
|
+
* server may have published meanwhile is enforced on the very next command
|
|
254
|
+
* instead of after the cache TTL.
|
|
255
|
+
*/
|
|
256
|
+
function invalidateCachedCompatibility(
|
|
257
|
+
baseUrl: string,
|
|
258
|
+
command: string | null | undefined,
|
|
259
|
+
skillsVersion: string | null | undefined,
|
|
260
|
+
): void {
|
|
261
|
+
try {
|
|
262
|
+
// Mirror the reader: an upgraded client may still be served from the
|
|
263
|
+
// legacy cache file until the host-scoped one is first written.
|
|
264
|
+
const path = sdkCompatibilityCachePath(baseUrl);
|
|
265
|
+
const legacyPath = legacySdkCompatibilityCachePath();
|
|
266
|
+
const cachePath =
|
|
267
|
+
!existsSync(path) && existsSync(legacyPath) ? legacyPath : path;
|
|
268
|
+
if (!existsSync(cachePath)) return;
|
|
269
|
+
const existing = JSON.parse(
|
|
270
|
+
readFileSync(cachePath, 'utf8'),
|
|
271
|
+
) as CompatCacheFile;
|
|
272
|
+
const entries = existing.entries ?? {};
|
|
273
|
+
delete entries[compatibilityCacheKey(baseUrl, command, skillsVersion)];
|
|
274
|
+
delete entries[legacyCompatibilityCacheKey(baseUrl, command)];
|
|
275
|
+
writeFileSync(cachePath, `${JSON.stringify({ entries }, null, 2)}\n`);
|
|
276
|
+
} catch {
|
|
277
|
+
// A cache that cannot be edited simply expires on its own TTL.
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
153
281
|
function writeCachedCompatibility(
|
|
154
282
|
baseUrl: string,
|
|
155
283
|
command: string | null | undefined,
|
|
@@ -176,7 +304,12 @@ function writeCachedCompatibility(
|
|
|
176
304
|
|
|
177
305
|
export async function checkSdkCompatibility(
|
|
178
306
|
baseUrl: string,
|
|
179
|
-
options: {
|
|
307
|
+
options: {
|
|
308
|
+
command?: string | null;
|
|
309
|
+
skillsVersion?: string | null;
|
|
310
|
+
/** Blocking budget for the live request; defaults to the full timeout. */
|
|
311
|
+
timeoutMs?: number;
|
|
312
|
+
} = {},
|
|
180
313
|
): Promise<{
|
|
181
314
|
response: SdkCompatibilityResponse | null;
|
|
182
315
|
error: Error | null;
|
|
@@ -185,7 +318,10 @@ export async function checkSdkCompatibility(
|
|
|
185
318
|
return { response: null, error: null };
|
|
186
319
|
}
|
|
187
320
|
const controller = new AbortController();
|
|
188
|
-
const timeout = setTimeout(
|
|
321
|
+
const timeout = setTimeout(
|
|
322
|
+
() => controller.abort(),
|
|
323
|
+
options.timeoutMs ?? CHECK_TIMEOUT_MS,
|
|
324
|
+
);
|
|
189
325
|
try {
|
|
190
326
|
const url = new URL('/api/v2/sdk/compat', baseUrl);
|
|
191
327
|
url.searchParams.set('version', SDK_VERSION);
|
|
@@ -209,14 +345,24 @@ export async function checkSdkCompatibility(
|
|
|
209
345
|
const data = (await response
|
|
210
346
|
.json()
|
|
211
347
|
.catch(() => null)) as SdkCompatibilityResponse | null;
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
348
|
+
// Only a policy-shaped body counts as a response. A JSON error page from
|
|
349
|
+
// a proxy is a failed check: it is never cached, and it falls through to
|
|
350
|
+
// the cached-policy fallback below like a network error would.
|
|
351
|
+
if (
|
|
352
|
+
!data ||
|
|
353
|
+
typeof data.ok !== 'boolean' ||
|
|
354
|
+
typeof data.status !== 'string'
|
|
355
|
+
) {
|
|
356
|
+
throw new Error(
|
|
357
|
+
`SDK compatibility endpoint returned a non-policy response (HTTP ${response.status}).`,
|
|
218
358
|
);
|
|
219
359
|
}
|
|
360
|
+
writeCachedCompatibility(
|
|
361
|
+
baseUrl,
|
|
362
|
+
options.command,
|
|
363
|
+
options.skillsVersion,
|
|
364
|
+
data,
|
|
365
|
+
);
|
|
220
366
|
return { response: data, error: null };
|
|
221
367
|
} catch (error) {
|
|
222
368
|
// Live compat policy must win when the server is reachable: rollback and
|
|
@@ -228,6 +374,16 @@ export async function checkSdkCompatibility(
|
|
|
228
374
|
options.skillsVersion,
|
|
229
375
|
);
|
|
230
376
|
if (cached) {
|
|
377
|
+
if (
|
|
378
|
+
options.timeoutMs !== undefined &&
|
|
379
|
+
options.timeoutMs < CHECK_TIMEOUT_MS
|
|
380
|
+
) {
|
|
381
|
+
invalidateCachedCompatibility(
|
|
382
|
+
baseUrl,
|
|
383
|
+
options.command,
|
|
384
|
+
options.skillsVersion,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
231
387
|
return { response: cached, error: null };
|
|
232
388
|
}
|
|
233
389
|
return {
|
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.114',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -31,11 +31,37 @@ function readPluginSkillsVersion(): string {
|
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Agent names can reach the CLI from `--agent`. Every state path derived from
|
|
36
|
+
* an agent name goes through this slug so no value can escape the SDK state
|
|
37
|
+
* directory.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* Every agent an installer may write for. `*` targets all of them, so any
|
|
41
|
+
* coordination keyed by agent must treat `*` as this whole set.
|
|
42
|
+
*/
|
|
43
|
+
export const KNOWN_SKILL_AGENTS = [
|
|
44
|
+
'antigravity',
|
|
45
|
+
'claude-code',
|
|
46
|
+
'cline',
|
|
47
|
+
'codex',
|
|
48
|
+
'cursor',
|
|
49
|
+
'gemini-cli',
|
|
50
|
+
'windsurf',
|
|
51
|
+
] as const;
|
|
52
|
+
|
|
53
|
+
export function skillsAgentPathSegment(agent: string): string {
|
|
54
|
+
if (agent === '*') return 'all';
|
|
55
|
+
const slug = agent.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '');
|
|
56
|
+
return slug || 'agent';
|
|
57
|
+
}
|
|
58
|
+
|
|
34
59
|
function sdkSkillsVersionPath(
|
|
35
60
|
baseUrl: string,
|
|
36
61
|
agents: readonly string[] = [],
|
|
37
62
|
): string {
|
|
38
|
-
const suffix =
|
|
63
|
+
const suffix =
|
|
64
|
+
agents.length > 0 ? `-${agents.map(skillsAgentPathSegment).join('-')}` : '';
|
|
39
65
|
return join(sdkCliStateDirPath(baseUrl), `skills${suffix}-version`);
|
|
40
66
|
}
|
|
41
67
|
|
|
@@ -13,8 +13,8 @@ import {
|
|
|
13
13
|
import type { SignalRadarInboundMessage } from '@deepline/monitors/signal-radars/inbound-message';
|
|
14
14
|
|
|
15
15
|
// Public inbound-queue surface used by the thin provider-callback receiver.
|
|
16
|
-
//
|
|
17
|
-
//
|
|
16
|
+
// The receiver stays on this established producer until the receipt dispatch
|
|
17
|
+
// endpoint is already serving in the preceding reader release.
|
|
18
18
|
export const SIGNAL_RADAR_INBOUND_TOPIC = SIGNAL_RADAR_INBOUND_TOPIC_VALUE;
|
|
19
19
|
|
|
20
20
|
function headersToObject(headers: Headers): Record<string, string> {
|
|
@@ -38,6 +38,10 @@ export async function sendSignalRadarInboundMessage(
|
|
|
38
38
|
return sendSignalRadarInbound(message, options);
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
// The reconciliation runner uses the same queue while this reader release
|
|
42
|
+
// waits for the signed receipt endpoint to be live.
|
|
43
|
+
export { send } from '@deepline/monitors/signal-radars/inbound-queue';
|
|
44
|
+
|
|
41
45
|
// Transient (retry) vs permanent (dead-letter) classifier, shared with the
|
|
42
46
|
// worker's retry callback. See classifyInboundFailure for the semantics.
|
|
43
47
|
export function classifySignalRadarInboundError(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/** Scheduler-neutral queue/workset health contract. */
|
|
2
2
|
|
|
3
3
|
import { measuredValue, type Measured } from './scheduled-work-v2';
|
|
4
|
+
import type { Telemetry, TelemetryFields } from './telemetry';
|
|
4
5
|
export type { Measured } from './scheduled-work-v2';
|
|
5
6
|
export {
|
|
6
7
|
exact,
|
|
@@ -448,6 +449,7 @@ export type QueueHealth = {
|
|
|
448
449
|
|
|
449
450
|
export const QUEUE_HEALTH_TELEMETRY_EVENT = 'operations.queue.health' as const;
|
|
450
451
|
export const QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION = 1 as const;
|
|
452
|
+
const UNOBSERVED_SIGNAL_REPEAT_MS = 5 * 60_000;
|
|
451
453
|
|
|
452
454
|
export type QueueHealthSignal = {
|
|
453
455
|
readonly schemaVersion: typeof QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION;
|
|
@@ -478,9 +480,10 @@ export type QueueHealthSignal = {
|
|
|
478
480
|
* Project a durable observation into the one cross-runtime queue signal.
|
|
479
481
|
* Healthy steady state is deliberately silent. Actionable backlog problems
|
|
480
482
|
* repeat on every observation so a log-based monitor cannot resolve while the
|
|
481
|
-
* queue remains unhealthy
|
|
482
|
-
*
|
|
483
|
-
* healthy observation after a problem emits
|
|
483
|
+
* queue remains unhealthy. Persistent adapter absence is repeated on a
|
|
484
|
+
* five-minute boundary so a log-backed dashboard cannot go blank while an
|
|
485
|
+
* adapter remains absent. The first healthy observation after a problem emits
|
|
486
|
+
* recovery.
|
|
484
487
|
*/
|
|
485
488
|
export function deriveQueueHealthSignal(input: {
|
|
486
489
|
definition: QueueCatalogEntry;
|
|
@@ -500,14 +503,19 @@ export function deriveQueueHealthSignal(input: {
|
|
|
500
503
|
previousHealth.status !== 'healthy';
|
|
501
504
|
if (health.status === 'healthy' && !recovering) return null;
|
|
502
505
|
// A permanently unavailable adapter is a catalog/configuration finding, not
|
|
503
|
-
// a one-minute incident.
|
|
504
|
-
//
|
|
506
|
+
// a one-minute incident. Repeat it only when the five-minute bucket changes;
|
|
507
|
+
// this keeps the state visible without recreating one log per observer tick.
|
|
505
508
|
if (
|
|
506
509
|
health.status === 'unobserved' &&
|
|
507
510
|
previousHealth?.status === 'unobserved' &&
|
|
508
511
|
previousHealth.reason === health.reason
|
|
509
512
|
) {
|
|
510
|
-
|
|
513
|
+
const previousBucket = Math.floor(
|
|
514
|
+
(input.previousObservation?.observedAt ?? 0) /
|
|
515
|
+
UNOBSERVED_SIGNAL_REPEAT_MS,
|
|
516
|
+
);
|
|
517
|
+
const currentBucket = Math.floor(now / UNOBSERVED_SIGNAL_REPEAT_MS);
|
|
518
|
+
if (previousBucket === currentBucket) return null;
|
|
511
519
|
}
|
|
512
520
|
return {
|
|
513
521
|
schemaVersion: QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION,
|
|
@@ -676,3 +684,280 @@ export function deriveQueueHealth(
|
|
|
676
684
|
...(worstAgeMs === undefined ? {} : { worstAgeMs, worstAgeRatio: ratio }),
|
|
677
685
|
};
|
|
678
686
|
}
|
|
687
|
+
|
|
688
|
+
// Legacy v1 snapshot helpers remain exported for the webhook/cron collectors
|
|
689
|
+
// during the v1 -> v2 observability rollout. They are intentionally isolated
|
|
690
|
+
// from the scheduler-neutral v2 contract above.
|
|
691
|
+
export const QUEUE_HEALTH_SCHEMA_VERSION = 1 as const;
|
|
692
|
+
export type QueueHealthAvailability = 'available' | 'unavailable';
|
|
693
|
+
export type QueueHealthFreshness = 'fresh' | 'stale';
|
|
694
|
+
export type QueueHealthLayer =
|
|
695
|
+
| 'admission'
|
|
696
|
+
| 'delivery'
|
|
697
|
+
| 'execution'
|
|
698
|
+
| 'projection'
|
|
699
|
+
| 'reconciliation'
|
|
700
|
+
| 'other';
|
|
701
|
+
export type QueueHealthAttentionReason =
|
|
702
|
+
| 'source_unavailable'
|
|
703
|
+
| 'snapshot_stale'
|
|
704
|
+
| 'blocked_work'
|
|
705
|
+
| 'dead_letter_work'
|
|
706
|
+
| 'backlog_aged'
|
|
707
|
+
| 'backlog_count_inexact'
|
|
708
|
+
| 'backlog_age_unavailable';
|
|
709
|
+
export type QueueHealthCount = { value: number | null; exact: boolean };
|
|
710
|
+
export type QueueHealthCapacity = { active: number | null; limit: number | null };
|
|
711
|
+
export type QueueHealthSnapshot = {
|
|
712
|
+
schemaVersion: typeof QUEUE_HEALTH_SCHEMA_VERSION;
|
|
713
|
+
queueId: string;
|
|
714
|
+
owner: string;
|
|
715
|
+
layer: QueueHealthLayer;
|
|
716
|
+
transport: string;
|
|
717
|
+
observedAt: number;
|
|
718
|
+
freshness: { staleAfterMs: number };
|
|
719
|
+
availability: QueueHealthAvailability;
|
|
720
|
+
queued: QueueHealthCount;
|
|
721
|
+
running: QueueHealthCount;
|
|
722
|
+
retryWaiting: QueueHealthCount;
|
|
723
|
+
blocked: QueueHealthCount;
|
|
724
|
+
deadLetter: QueueHealthCount;
|
|
725
|
+
oldestQueuedAgeMs: number | null;
|
|
726
|
+
oldestBlockedAgeMs: number | null;
|
|
727
|
+
capacity: QueueHealthCapacity;
|
|
728
|
+
thresholds: { degradedOldestQueuedAgeMs: number };
|
|
729
|
+
};
|
|
730
|
+
export type QueueHealthSnapshotInput = Omit<
|
|
731
|
+
QueueHealthSnapshot,
|
|
732
|
+
'schemaVersion'
|
|
733
|
+
>;
|
|
734
|
+
type LegacyQueueHealthStatus = 'healthy' | 'degraded' | 'blocked' | 'unknown';
|
|
735
|
+
type LegacyQueueHealthAssessment = {
|
|
736
|
+
status: LegacyQueueHealthStatus;
|
|
737
|
+
freshness: QueueHealthFreshness;
|
|
738
|
+
observedAgeMs: number;
|
|
739
|
+
attentionReasons: readonly QueueHealthAttentionReason[];
|
|
740
|
+
};
|
|
741
|
+
export type QueueHealthAssessment = LegacyQueueHealthAssessment;
|
|
742
|
+
|
|
743
|
+
const legacyLayers = new Set<QueueHealthLayer>([
|
|
744
|
+
'admission',
|
|
745
|
+
'delivery',
|
|
746
|
+
'execution',
|
|
747
|
+
'projection',
|
|
748
|
+
'reconciliation',
|
|
749
|
+
'other',
|
|
750
|
+
]);
|
|
751
|
+
const legacyCountNames = [
|
|
752
|
+
'queued',
|
|
753
|
+
'running',
|
|
754
|
+
'retryWaiting',
|
|
755
|
+
'blocked',
|
|
756
|
+
'deadLetter',
|
|
757
|
+
] as const;
|
|
758
|
+
|
|
759
|
+
export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
|
|
760
|
+
const record = legacyRecord(input, 'Queue health snapshot');
|
|
761
|
+
if (record.schemaVersion !== 1)
|
|
762
|
+
throw new Error('Queue health schemaVersion must be 1.');
|
|
763
|
+
const layer = legacyString(record.layer, 'layer') as QueueHealthLayer;
|
|
764
|
+
if (!legacyLayers.has(layer)) throw new Error('Queue health layer is not supported.');
|
|
765
|
+
const availability = legacyString(
|
|
766
|
+
record.availability,
|
|
767
|
+
'availability',
|
|
768
|
+
) as QueueHealthAvailability;
|
|
769
|
+
if (availability !== 'available' && availability !== 'unavailable')
|
|
770
|
+
throw new Error('Queue health availability must be available or unavailable.');
|
|
771
|
+
const freshness = legacyRecord(record.freshness, 'freshness');
|
|
772
|
+
const capacity = legacyRecord(record.capacity, 'capacity');
|
|
773
|
+
const thresholds = legacyRecord(record.thresholds, 'thresholds');
|
|
774
|
+
const snapshot: QueueHealthSnapshot = {
|
|
775
|
+
schemaVersion: 1,
|
|
776
|
+
queueId: legacyIdentifier(record.queueId, 'queueId'),
|
|
777
|
+
owner: legacyIdentifier(record.owner, 'owner'),
|
|
778
|
+
layer,
|
|
779
|
+
transport: legacyIdentifier(record.transport, 'transport'),
|
|
780
|
+
observedAt: legacyNumber(record.observedAt, 'observedAt'),
|
|
781
|
+
freshness: { staleAfterMs: legacyPositive(freshness.staleAfterMs, 'freshness.staleAfterMs') },
|
|
782
|
+
availability,
|
|
783
|
+
queued: legacyCount(record.queued, 'queued'),
|
|
784
|
+
running: legacyCount(record.running, 'running'),
|
|
785
|
+
retryWaiting: legacyCount(record.retryWaiting, 'retryWaiting'),
|
|
786
|
+
blocked: legacyCount(record.blocked, 'blocked'),
|
|
787
|
+
deadLetter: legacyCount(record.deadLetter, 'deadLetter'),
|
|
788
|
+
oldestQueuedAgeMs: legacyNullableNumber(record.oldestQueuedAgeMs, 'oldestQueuedAgeMs'),
|
|
789
|
+
oldestBlockedAgeMs: legacyNullableNumber(record.oldestBlockedAgeMs, 'oldestBlockedAgeMs'),
|
|
790
|
+
capacity: {
|
|
791
|
+
active: legacyNullableNumber(capacity.active, 'capacity.active'),
|
|
792
|
+
limit: legacyNullableNumber(capacity.limit, 'capacity.limit'),
|
|
793
|
+
},
|
|
794
|
+
thresholds: {
|
|
795
|
+
degradedOldestQueuedAgeMs: legacyPositive(
|
|
796
|
+
thresholds.degradedOldestQueuedAgeMs,
|
|
797
|
+
'thresholds.degradedOldestQueuedAgeMs',
|
|
798
|
+
),
|
|
799
|
+
},
|
|
800
|
+
};
|
|
801
|
+
if (snapshot.availability === 'unavailable') {
|
|
802
|
+
for (const name of legacyCountNames) {
|
|
803
|
+
const count = snapshot[name];
|
|
804
|
+
if (count.value !== null || count.exact)
|
|
805
|
+
throw new Error(`Queue health ${name} must be unknown when availability is unavailable.`);
|
|
806
|
+
}
|
|
807
|
+
if (snapshot.oldestQueuedAgeMs !== null || snapshot.oldestBlockedAgeMs !== null)
|
|
808
|
+
throw new Error('Queue health oldestQueuedAgeMs must be null when availability is unavailable.');
|
|
809
|
+
}
|
|
810
|
+
if (
|
|
811
|
+
snapshot.queued.exact &&
|
|
812
|
+
snapshot.queued.value === 0 &&
|
|
813
|
+
snapshot.retryWaiting.exact &&
|
|
814
|
+
snapshot.retryWaiting.value === 0 &&
|
|
815
|
+
snapshot.oldestQueuedAgeMs !== null
|
|
816
|
+
) {
|
|
817
|
+
throw new Error('Queue health oldestQueuedAgeMs requires queued or retry-waiting work.');
|
|
818
|
+
}
|
|
819
|
+
if (
|
|
820
|
+
snapshot.blocked.exact &&
|
|
821
|
+
snapshot.blocked.value === 0 &&
|
|
822
|
+
snapshot.deadLetter.exact &&
|
|
823
|
+
snapshot.deadLetter.value === 0 &&
|
|
824
|
+
snapshot.oldestBlockedAgeMs !== null
|
|
825
|
+
) {
|
|
826
|
+
throw new Error('Queue health oldestBlockedAgeMs requires blocked or dead-letter work.');
|
|
827
|
+
}
|
|
828
|
+
return snapshot;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
export function createQueueHealthSnapshot(
|
|
832
|
+
input: QueueHealthSnapshotInput,
|
|
833
|
+
): QueueHealthSnapshot {
|
|
834
|
+
return parseQueueHealthSnapshot({ ...input, schemaVersion: 1 });
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
export function assessQueueHealth(
|
|
838
|
+
input: QueueHealthSnapshot,
|
|
839
|
+
now = Date.now(),
|
|
840
|
+
): LegacyQueueHealthAssessment {
|
|
841
|
+
const snapshot = parseQueueHealthSnapshot(input);
|
|
842
|
+
const observedAgeMs = Math.max(0, now - snapshot.observedAt);
|
|
843
|
+
const freshness: QueueHealthFreshness =
|
|
844
|
+
observedAgeMs > snapshot.freshness.staleAfterMs ? 'stale' : 'fresh';
|
|
845
|
+
if (snapshot.availability === 'unavailable')
|
|
846
|
+
return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['source_unavailable'] };
|
|
847
|
+
if (freshness === 'stale')
|
|
848
|
+
return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['snapshot_stale'] };
|
|
849
|
+
const attentionReasons: QueueHealthAttentionReason[] = [];
|
|
850
|
+
if (knownItems(snapshot.blocked)) attentionReasons.push('blocked_work');
|
|
851
|
+
if (knownItems(snapshot.deadLetter)) attentionReasons.push('dead_letter_work');
|
|
852
|
+
if (attentionReasons.length)
|
|
853
|
+
return { status: 'blocked', freshness, observedAgeMs, attentionReasons };
|
|
854
|
+
if (
|
|
855
|
+
snapshot.oldestQueuedAgeMs !== null &&
|
|
856
|
+
snapshot.oldestQueuedAgeMs >= snapshot.thresholds.degradedOldestQueuedAgeMs
|
|
857
|
+
)
|
|
858
|
+
return { status: 'degraded', freshness, observedAgeMs, attentionReasons: ['backlog_aged'] };
|
|
859
|
+
if (
|
|
860
|
+
!snapshot.queued.exact ||
|
|
861
|
+
!snapshot.retryWaiting.exact ||
|
|
862
|
+
!snapshot.blocked.exact ||
|
|
863
|
+
!snapshot.deadLetter.exact
|
|
864
|
+
)
|
|
865
|
+
return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['backlog_count_inexact'] };
|
|
866
|
+
if (
|
|
867
|
+
(knownItems(snapshot.queued) || knownItems(snapshot.retryWaiting)) &&
|
|
868
|
+
snapshot.oldestQueuedAgeMs === null
|
|
869
|
+
)
|
|
870
|
+
return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['backlog_age_unavailable'] };
|
|
871
|
+
return { status: 'healthy', freshness, observedAgeMs, attentionReasons: [] };
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
export function queueHealthTelemetryFields(
|
|
875
|
+
input: QueueHealthSnapshot,
|
|
876
|
+
assessment = assessQueueHealth(input),
|
|
877
|
+
): TelemetryFields {
|
|
878
|
+
const snapshot = parseQueueHealthSnapshot(input);
|
|
879
|
+
return {
|
|
880
|
+
queueHealthSchemaVersion: snapshot.schemaVersion,
|
|
881
|
+
queueId: snapshot.queueId,
|
|
882
|
+
queueOwner: snapshot.owner,
|
|
883
|
+
queueLayer: snapshot.layer,
|
|
884
|
+
queueTransport: snapshot.transport,
|
|
885
|
+
queueAvailability: snapshot.availability,
|
|
886
|
+
queueObservedAt: snapshot.observedAt,
|
|
887
|
+
queueFreshnessStaleAfterMs: snapshot.freshness.staleAfterMs,
|
|
888
|
+
queueFreshness: assessment.freshness,
|
|
889
|
+
queueHealth: assessment.status,
|
|
890
|
+
queueHealthObservedAgeMs: assessment.observedAgeMs,
|
|
891
|
+
queueQueued: snapshot.queued.value,
|
|
892
|
+
queueQueuedExact: snapshot.queued.exact,
|
|
893
|
+
queueRunning: snapshot.running.value,
|
|
894
|
+
queueRunningExact: snapshot.running.exact,
|
|
895
|
+
queueRetryWaiting: snapshot.retryWaiting.value,
|
|
896
|
+
queueRetryWaitingExact: snapshot.retryWaiting.exact,
|
|
897
|
+
queueBlocked: snapshot.blocked.value,
|
|
898
|
+
queueBlockedExact: snapshot.blocked.exact,
|
|
899
|
+
queueDeadLetter: snapshot.deadLetter.value,
|
|
900
|
+
queueDeadLetterExact: snapshot.deadLetter.exact,
|
|
901
|
+
queueOldestQueuedAgeMs: snapshot.oldestQueuedAgeMs,
|
|
902
|
+
queueOldestBlockedAgeMs: snapshot.oldestBlockedAgeMs,
|
|
903
|
+
queueCapacityActive: snapshot.capacity.active,
|
|
904
|
+
queueCapacityLimit: snapshot.capacity.limit,
|
|
905
|
+
queueDegradedOldestQueuedAgeMs: snapshot.thresholds.degradedOldestQueuedAgeMs,
|
|
906
|
+
queueAttentionReasons: assessment.attentionReasons,
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
export async function emitQueueHealthSnapshot(input: {
|
|
911
|
+
telemetry: Pick<Telemetry, 'info' | 'warn'>;
|
|
912
|
+
snapshot: QueueHealthSnapshot;
|
|
913
|
+
now?: number;
|
|
914
|
+
}): Promise<LegacyQueueHealthAssessment> {
|
|
915
|
+
const assessment = assessQueueHealth(input.snapshot, input.now);
|
|
916
|
+
const fields = queueHealthTelemetryFields(input.snapshot, assessment);
|
|
917
|
+
await input.telemetry.info('queue.health.snapshot', fields);
|
|
918
|
+
if (assessment.status !== 'healthy')
|
|
919
|
+
await input.telemetry.warn('queue.health.attention', fields);
|
|
920
|
+
return assessment;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
function legacyRecord(value: unknown, label: string): Record<string, unknown> {
|
|
924
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
925
|
+
throw new Error(`${label} must be an object.`);
|
|
926
|
+
return value as Record<string, unknown>;
|
|
927
|
+
}
|
|
928
|
+
function legacyString(value: unknown, label: string): string {
|
|
929
|
+
if (typeof value !== 'string' || !value.trim())
|
|
930
|
+
throw new Error(`Queue health ${label} must be a non-empty string.`);
|
|
931
|
+
return value.trim();
|
|
932
|
+
}
|
|
933
|
+
function legacyIdentifier(value: unknown, label: string): string {
|
|
934
|
+
const result = legacyString(value, label);
|
|
935
|
+
if (result.length > 96 || !/^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/.test(result))
|
|
936
|
+
throw new Error(`Queue health ${label} must be a stable low-cardinality identifier.`);
|
|
937
|
+
return result;
|
|
938
|
+
}
|
|
939
|
+
function legacyNumber(value: unknown, label: string): number {
|
|
940
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
|
|
941
|
+
throw new Error(`Queue health ${label} must be a finite non-negative number.`);
|
|
942
|
+
return value;
|
|
943
|
+
}
|
|
944
|
+
function legacyPositive(value: unknown, label: string): number {
|
|
945
|
+
const result = legacyNumber(value, label);
|
|
946
|
+
if (result === 0) throw new Error(`Queue health ${label} must be positive.`);
|
|
947
|
+
return result;
|
|
948
|
+
}
|
|
949
|
+
function legacyNullableNumber(value: unknown, label: string): number | null {
|
|
950
|
+
return value === null ? null : legacyNumber(value, label);
|
|
951
|
+
}
|
|
952
|
+
function legacyCount(value: unknown, label: string): QueueHealthCount {
|
|
953
|
+
const record = legacyRecord(value, label);
|
|
954
|
+
if (typeof record.exact !== 'boolean')
|
|
955
|
+
throw new Error(`Queue health ${label}.exact must be a boolean.`);
|
|
956
|
+
const count = legacyNullableNumber(record.value, `${label}.value`);
|
|
957
|
+
if (count === null && record.exact)
|
|
958
|
+
throw new Error(`Queue health ${label}.exact cannot be true when value is unknown.`);
|
|
959
|
+
return { value: count, exact: record.exact };
|
|
960
|
+
}
|
|
961
|
+
function knownItems(count: QueueHealthCount): boolean {
|
|
962
|
+
return count.value !== null && count.value > 0;
|
|
963
|
+
}
|