crewly 1.20.176 → 1.20.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/skills/orchestrator/update-team-member/SKILL.md +1 -1
- package/dist/backend/backend/src/constants.d.ts +79 -3
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +97 -3
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.js +2 -1
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/system/runtime-fallback.controller.d.ts +30 -0
- package/dist/backend/backend/src/controllers/system/runtime-fallback.controller.d.ts.map +1 -0
- package/dist/backend/backend/src/controllers/system/runtime-fallback.controller.js +100 -0
- package/dist/backend/backend/src/controllers/system/runtime-fallback.controller.js.map +1 -0
- package/dist/backend/backend/src/controllers/team/team.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/team/team.controller.js +33 -4
- package/dist/backend/backend/src/controllers/team/team.controller.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +25 -0
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/routes/modules/system.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/routes/modules/system.routes.js +3 -0
- package/dist/backend/backend/src/routes/modules/system.routes.js.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-heartbeat-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-heartbeat-monitor.service.js +17 -0
- package/dist/backend/backend/src/services/agent/agent-heartbeat-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts +10 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.js +103 -7
- package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/antigravity-runtime.service.d.ts +1 -1
- package/dist/backend/backend/src/services/agent/antigravity-runtime.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/antigravity-runtime.service.js +3 -2
- package/dist/backend/backend/src/services/agent/antigravity-runtime.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.d.ts +7 -0
- package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.js +16 -0
- package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/idle-detection.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/idle-detection.service.js +2 -1
- package/dist/backend/backend/src/services/agent/idle-detection.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js +7 -0
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/planned-relaunch.registry.d.ts +61 -0
- package/dist/backend/backend/src/services/agent/planned-relaunch.registry.d.ts.map +1 -0
- package/dist/backend/backend/src/services/agent/planned-relaunch.registry.js +73 -0
- package/dist/backend/backend/src/services/agent/planned-relaunch.registry.js.map +1 -0
- package/dist/backend/backend/src/services/agent/tmux.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/tmux.service.js +2 -1
- package/dist/backend/backend/src/services/agent/tmux.service.js.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-status.service.d.ts +17 -0
- package/dist/backend/backend/src/services/harness/harness-status.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-status.service.js +25 -1
- package/dist/backend/backend/src/services/harness/harness-status.service.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.js +2 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.js.map +1 -1
- package/dist/backend/backend/src/services/monitoring/activity-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/monitoring/activity-monitor.service.js +3 -2
- package/dist/backend/backend/src/services/monitoring/activity-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.js +10 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.d.ts +28 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.js +48 -3
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.js.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.js +3 -2
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.d.ts +100 -0
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.js +115 -0
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.d.ts +66 -0
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.js +211 -0
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-availability.d.ts +39 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-availability.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-availability.js +56 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-availability.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.d.ts +417 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.js +974 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.store.d.ts +53 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.store.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.store.js +109 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.store.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.types.d.ts +134 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.types.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.types.js +152 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.types.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.d.ts +95 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.js +286 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-smoke-test.service.d.ts +203 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-smoke-test.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-smoke-test.service.js +419 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-smoke-test.service.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-usage-probe.d.ts +94 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-usage-probe.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-usage-probe.js +186 -0
- package/dist/backend/backend/src/services/runtime-fallback/runtime-usage-probe.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.d.ts +113 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.js +257 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.js.map +1 -0
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts +8 -0
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js +16 -2
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/backend/backend/src/services/task-pool/claim.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/task-pool/claim.service.js +3 -1
- package/dist/backend/backend/src/services/task-pool/claim.service.js.map +1 -1
- package/dist/backend/backend/src/services/workflow/message-scheduler.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/workflow/message-scheduler.service.js +3 -2
- package/dist/backend/backend/src/services/workflow/message-scheduler.service.js.map +1 -1
- package/dist/backend/backend/src/services/workflow/scheduler.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/workflow/scheduler.service.js +3 -2
- package/dist/backend/backend/src/services/workflow/scheduler.service.js.map +1 -1
- package/dist/backend/backend/src/types/index.d.ts +14 -0
- package/dist/backend/backend/src/types/index.d.ts.map +1 -1
- package/dist/backend/backend/src/types/index.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +79 -3
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +97 -3
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts +10 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.js +103 -7
- package/dist/cli/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/antigravity-runtime.service.d.ts +1 -1
- package/dist/cli/backend/src/services/agent/antigravity-runtime.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/antigravity-runtime.service.js +3 -2
- package/dist/cli/backend/src/services/agent/antigravity-runtime.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.d.ts +7 -0
- package/dist/cli/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.js +16 -0
- package/dist/cli/backend/src/services/agent/crewly-agent/crewly-agent-external-runtime.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js +7 -0
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/planned-relaunch.registry.d.ts +61 -0
- package/dist/cli/backend/src/services/agent/planned-relaunch.registry.d.ts.map +1 -0
- package/dist/cli/backend/src/services/agent/planned-relaunch.registry.js +73 -0
- package/dist/cli/backend/src/services/agent/planned-relaunch.registry.js.map +1 -0
- package/dist/cli/backend/src/services/harness/harness-status.service.d.ts +17 -0
- package/dist/cli/backend/src/services/harness/harness-status.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-status.service.js +25 -1
- package/dist/cli/backend/src/services/harness/harness-status.service.js.map +1 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.js +2 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.js.map +1 -1
- package/dist/cli/backend/src/services/monitoring/activity-monitor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/monitoring/activity-monitor.service.js +3 -2
- package/dist/cli/backend/src/services/monitoring/activity-monitor.service.js.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.d.ts +28 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.js +48 -3
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.js.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.js +3 -2
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.js.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.d.ts +100 -0
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.d.ts.map +1 -0
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.js +115 -0
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.js.map +1 -0
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts +8 -0
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js +16 -2
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/cli/backend/src/services/task-pool/claim.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/task-pool/claim.service.js +3 -1
- package/dist/cli/backend/src/services/task-pool/claim.service.js.map +1 -1
- package/dist/cli/backend/src/types/index.d.ts +14 -0
- package/dist/cli/backend/src/types/index.d.ts.map +1 -1
- package/dist/cli/backend/src/types/index.js.map +1 -1
- package/dist/cli/cli/src/utils/runtime-auth.d.ts +1 -1
- package/dist/cli/cli/src/utils/runtime-auth.js +1 -1
- package/frontend/dist/assets/{index-0726afab.js → index-69af46c3.js} +275 -275
- package/frontend/dist/assets/{index-0c5449b5.css → index-cdc52122.css} +1 -1
- package/frontend/dist/index.html +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime smoke test — prove a runtime really works on this machine by
|
|
3
|
+
* running one agent on it end to end.
|
|
4
|
+
*
|
|
5
|
+
* Through the backend's own REST API (the calls the dashboard makes):
|
|
6
|
+
*
|
|
7
|
+
* 1. `create_team` — temp project + team `zz-runtime-smoke-<runtime>` with
|
|
8
|
+
* one member on the runtime (a leftover of the same name is removed first).
|
|
9
|
+
* 2. `start_member` — start it (as the owner's dashboard would).
|
|
10
|
+
* 3. `agent_ready` — wait until it registered. Antigravity's first-run Terms
|
|
11
|
+
* screen fails the test with "Antigravity needs its terms accepted once"
|
|
12
|
+
* and the screen text — Crewly never accepts terms for the owner.
|
|
13
|
+
* 4. `send_task` — `echo ok > <file>` with bash, then reply with a token.
|
|
14
|
+
* 5. `bash` — the file holds `ok`.
|
|
15
|
+
* 6. `reply` — the token `SMOKE-ok-<nonce>` shows in the agent's output. The
|
|
16
|
+
* task never contains that string verbatim (only its parts), so an echo
|
|
17
|
+
* of the task does not count.
|
|
18
|
+
* 7. `cleanup` — stop and delete the team and the project (always).
|
|
19
|
+
*
|
|
20
|
+
* Bounded by 5 minutes. specs/2026-10-01-runtime-fallback.md
|
|
21
|
+
*
|
|
22
|
+
* @module services/runtime-fallback/runtime-smoke-test.service
|
|
23
|
+
*/
|
|
24
|
+
import { randomBytes } from 'crypto';
|
|
25
|
+
import * as fs from 'fs';
|
|
26
|
+
import * as os from 'os';
|
|
27
|
+
import * as path from 'path';
|
|
28
|
+
import { ANTIGRAVITY_CONSTANTS, RUNTIME_FALLBACK_CONSTANTS, RUNTIME_TYPES } from '../../constants.js';
|
|
29
|
+
import { normalizeTerminalOutput, redactSecrets } from '../harness/login-rules.js';
|
|
30
|
+
import { deriveMemberSessionName } from '../../utils/member-session-name.utils.js';
|
|
31
|
+
import { KNOWN_RUNTIMES } from './runtime-fallback.types.js';
|
|
32
|
+
const S = RUNTIME_FALLBACK_CONSTANTS.SMOKE;
|
|
33
|
+
/** Agent statuses that mean "registered and ready". */
|
|
34
|
+
const READY_STATUSES = new Set(['active']);
|
|
35
|
+
/**
|
|
36
|
+
* Whether a screen shows Antigravity's first-run (Terms) screens.
|
|
37
|
+
*
|
|
38
|
+
* @param screen - Screen text
|
|
39
|
+
* @returns True when the Terms / onboarding screen is up
|
|
40
|
+
*/
|
|
41
|
+
export function showsAntigravityTerms(screen) {
|
|
42
|
+
// Wrapped lines lose the space at the break: compare without whitespace too.
|
|
43
|
+
const bare = screen.replace(/\s+/g, '');
|
|
44
|
+
return ANTIGRAVITY_CONSTANTS.SCREEN.FIRST_RUN_MARKERS.some((m) => screen.includes(m) || bare.includes(m.replace(/\s+/g, '')));
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The task the smoke agent gets.
|
|
48
|
+
*
|
|
49
|
+
* @param file - Proof file to write with bash
|
|
50
|
+
* @param nonce - Code for the reply token
|
|
51
|
+
* @returns Message text (never contains `SMOKE-ok-<nonce>` verbatim)
|
|
52
|
+
*/
|
|
53
|
+
export function buildSmokeTask(file, nonce) {
|
|
54
|
+
return [
|
|
55
|
+
'Crewly runtime smoke test. Do exactly these two steps and nothing else:',
|
|
56
|
+
`1. With your bash / shell tool, run this command: echo ok > ${file}`,
|
|
57
|
+
`2. Reply with one line made of: the word SMOKE, a dash, the text that file now contains, a dash, and the code ${nonce}` +
|
|
58
|
+
' (for code 1234 and file text "hi" the line would be SMOKE-hi-1234). Send it with your reply / report-status skill' +
|
|
59
|
+
' (status done), or as your answer if you have no such skill.',
|
|
60
|
+
].join('\n');
|
|
61
|
+
}
|
|
62
|
+
/** Runs smoke tests and keeps their jobs. */
|
|
63
|
+
export class RuntimeSmokeTestService {
|
|
64
|
+
deps;
|
|
65
|
+
jobs = new Map();
|
|
66
|
+
running = new Map();
|
|
67
|
+
now;
|
|
68
|
+
sleep;
|
|
69
|
+
/**
|
|
70
|
+
* @param deps - Dependencies
|
|
71
|
+
*/
|
|
72
|
+
constructor(deps) {
|
|
73
|
+
this.deps = deps;
|
|
74
|
+
this.now = deps.now ?? (() => Date.now());
|
|
75
|
+
this.sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Start a smoke test (one per runtime at a time).
|
|
79
|
+
*
|
|
80
|
+
* @param runtime - Runtime id
|
|
81
|
+
* @returns The job and a promise of its result
|
|
82
|
+
* @throws Error for an unknown runtime
|
|
83
|
+
*/
|
|
84
|
+
start(runtime) {
|
|
85
|
+
if (!KNOWN_RUNTIMES.includes(runtime))
|
|
86
|
+
throw new Error(`Unknown runtime: ${runtime}`);
|
|
87
|
+
this.prune();
|
|
88
|
+
const existing = [...this.jobs.values()].find((j) => j.runtime === runtime && j.state === 'running');
|
|
89
|
+
if (existing)
|
|
90
|
+
return { job: existing, done: this.running.get(existing.jobId) };
|
|
91
|
+
const job = { jobId: `smoke-${runtime}-${randomBytes(4).toString('hex')}`, runtime, state: 'running', startedAt: new Date(this.now()).toISOString() };
|
|
92
|
+
this.jobs.set(job.jobId, job);
|
|
93
|
+
const done = this.run(runtime).then((result) => {
|
|
94
|
+
job.state = 'done';
|
|
95
|
+
job.result = result;
|
|
96
|
+
this.running.delete(job.jobId);
|
|
97
|
+
return result;
|
|
98
|
+
});
|
|
99
|
+
this.running.set(job.jobId, done);
|
|
100
|
+
return { job, done };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* A job by id.
|
|
104
|
+
*
|
|
105
|
+
* @param jobId - Job id
|
|
106
|
+
* @returns Job, or null
|
|
107
|
+
*/
|
|
108
|
+
get(jobId) {
|
|
109
|
+
return this.jobs.get(jobId) ?? null;
|
|
110
|
+
}
|
|
111
|
+
prune() {
|
|
112
|
+
for (const [id, job] of this.jobs) {
|
|
113
|
+
if (job.state === 'done' && this.now() - Date.parse(job.startedAt) > S.JOB_TTL_MS)
|
|
114
|
+
this.jobs.delete(id);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Run one smoke test.
|
|
119
|
+
*
|
|
120
|
+
* @param runtime - Runtime id
|
|
121
|
+
* @returns Result (never throws)
|
|
122
|
+
*/
|
|
123
|
+
async run(runtime) {
|
|
124
|
+
const api = this.deps.api;
|
|
125
|
+
const started = this.now();
|
|
126
|
+
const deadline = started + (this.deps.timeoutMs ?? S.TIMEOUT_MS);
|
|
127
|
+
const pollMs = this.deps.pollMs ?? S.POLL_MS;
|
|
128
|
+
const name = `${S.TEAM_PREFIX}${runtime}`;
|
|
129
|
+
const nonce = this.deps.nonce?.() ?? randomBytes(3).toString('hex');
|
|
130
|
+
const workDir = path.join(this.deps.workRoot ?? path.join(os.tmpdir(), 'crewly-runtime-smoke'), `${runtime}-${nonce}`);
|
|
131
|
+
const proofFile = path.join(workDir, `smoke-${nonce}.txt`);
|
|
132
|
+
const token = `SMOKE-ok-${nonce}`;
|
|
133
|
+
const steps = [];
|
|
134
|
+
let teamId = null;
|
|
135
|
+
let projectId = null;
|
|
136
|
+
let sessionName = '';
|
|
137
|
+
let screen = '';
|
|
138
|
+
let stepStart = this.now();
|
|
139
|
+
const pass = (step, detail) => {
|
|
140
|
+
steps.push({ step, ok: true, ms: this.now() - stepStart, ...(detail ? { detail } : {}) });
|
|
141
|
+
stepStart = this.now();
|
|
142
|
+
};
|
|
143
|
+
// The live screen; once the session is gone (a refused start), its log.
|
|
144
|
+
const readScreen = async () => {
|
|
145
|
+
if (!sessionName)
|
|
146
|
+
return screen;
|
|
147
|
+
let text = '';
|
|
148
|
+
try {
|
|
149
|
+
text = await api.capture(sessionName, S.CAPTURE_LINES);
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
// session gone — fall back to its log
|
|
153
|
+
}
|
|
154
|
+
if (!text)
|
|
155
|
+
text = await api.sessionLog(sessionName, S.CAPTURE_LINES).catch(() => '');
|
|
156
|
+
if (text)
|
|
157
|
+
screen = redactSecrets(normalizeTerminalOutput(text).text);
|
|
158
|
+
return screen;
|
|
159
|
+
};
|
|
160
|
+
let failure = null;
|
|
161
|
+
const fail = (step, error) => {
|
|
162
|
+
failure = { step, error };
|
|
163
|
+
steps.push({ step, ok: false, ms: this.now() - stepStart, detail: error });
|
|
164
|
+
};
|
|
165
|
+
const termsCheck = (text) => runtime === RUNTIME_TYPES.ANTIGRAVITY_CLI && showsAntigravityTerms(text);
|
|
166
|
+
const TERMS = 'Antigravity needs its terms accepted once';
|
|
167
|
+
try {
|
|
168
|
+
// 1. Team (and a scratch project so the agent never works in a real one)
|
|
169
|
+
await this.removeLeftovers(name);
|
|
170
|
+
fs.mkdirSync(workDir, { recursive: true });
|
|
171
|
+
projectId = await api.createProject(name, workDir);
|
|
172
|
+
const created = await api.createTeam({
|
|
173
|
+
name,
|
|
174
|
+
description: 'Temporary team for a runtime smoke test (deleted afterwards).',
|
|
175
|
+
projectIds: [projectId],
|
|
176
|
+
members: [
|
|
177
|
+
{
|
|
178
|
+
name: S.MEMBER_NAME,
|
|
179
|
+
role: S.MEMBER_ROLE,
|
|
180
|
+
runtimeType: runtime,
|
|
181
|
+
...(runtime === RUNTIME_TYPES.CREWLY_AGENT
|
|
182
|
+
? { modelId: this.deps.crewlyAgentModel?.() ?? RUNTIME_FALLBACK_CONSTANTS.DEFAULT_CREWLY_AGENT_MODEL }
|
|
183
|
+
: {}),
|
|
184
|
+
systemPrompt: 'You are a smoke-test agent. Do exactly what each message asks and nothing else.',
|
|
185
|
+
},
|
|
186
|
+
],
|
|
187
|
+
});
|
|
188
|
+
teamId = created.id;
|
|
189
|
+
pass('create_team', name);
|
|
190
|
+
// 2. Start
|
|
191
|
+
const startResult = await api.startMember(created.id, created.memberId);
|
|
192
|
+
const member = await api.getMember(created.id, created.memberId).catch(() => null);
|
|
193
|
+
sessionName = member?.sessionName || deriveMemberSessionName(name, S.MEMBER_NAME, created.memberId);
|
|
194
|
+
if (!startResult.ok) {
|
|
195
|
+
const text = await readScreen();
|
|
196
|
+
if (termsCheck(text) || /first-run screens|Terms of Service/i.test(startResult.error ?? ''))
|
|
197
|
+
fail('start_member', TERMS);
|
|
198
|
+
else
|
|
199
|
+
fail('start_member', startResult.error ?? 'The member did not start');
|
|
200
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
201
|
+
}
|
|
202
|
+
pass('start_member');
|
|
203
|
+
// 3. Registered (watch for the Terms screen meanwhile)
|
|
204
|
+
let ready = false;
|
|
205
|
+
while (this.now() < deadline) {
|
|
206
|
+
const m = await api.getMember(created.id, created.memberId).catch(() => null);
|
|
207
|
+
if (m?.sessionName)
|
|
208
|
+
sessionName = m.sessionName;
|
|
209
|
+
if (termsCheck(await readScreen())) {
|
|
210
|
+
fail('agent_ready', TERMS);
|
|
211
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
212
|
+
}
|
|
213
|
+
if (m && READY_STATUSES.has(m.agentStatus)) {
|
|
214
|
+
ready = true;
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
await this.sleep(pollMs);
|
|
218
|
+
}
|
|
219
|
+
if (!ready) {
|
|
220
|
+
fail('agent_ready', 'The agent did not register in time');
|
|
221
|
+
await readScreen();
|
|
222
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
223
|
+
}
|
|
224
|
+
pass('agent_ready', sessionName);
|
|
225
|
+
// 4. Task
|
|
226
|
+
const delivered = await api.deliver(sessionName, buildSmokeTask(proofFile, nonce));
|
|
227
|
+
if (!delivered.ok) {
|
|
228
|
+
fail('send_task', delivered.error ?? 'The task could not be delivered');
|
|
229
|
+
await readScreen();
|
|
230
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
231
|
+
}
|
|
232
|
+
pass('send_task');
|
|
233
|
+
// 5. bash ran, 6. reply came back
|
|
234
|
+
let bashOk = false;
|
|
235
|
+
let replied = false;
|
|
236
|
+
while (this.now() < deadline && !(bashOk && replied)) {
|
|
237
|
+
if (!bashOk && readProof(proofFile) === 'ok') {
|
|
238
|
+
bashOk = true;
|
|
239
|
+
pass('bash');
|
|
240
|
+
}
|
|
241
|
+
const text = await readScreen();
|
|
242
|
+
if (termsCheck(text)) {
|
|
243
|
+
fail(bashOk ? 'reply' : 'bash', TERMS);
|
|
244
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
245
|
+
}
|
|
246
|
+
if (!replied && (text.includes(token) || text.replace(/\s+/g, '').includes(token)))
|
|
247
|
+
replied = true;
|
|
248
|
+
if (bashOk && replied)
|
|
249
|
+
break;
|
|
250
|
+
await this.sleep(pollMs);
|
|
251
|
+
}
|
|
252
|
+
if (!bashOk)
|
|
253
|
+
fail('bash', `The agent did not run "echo ok" with bash (no ${path.basename(proofFile)})`);
|
|
254
|
+
else if (!replied)
|
|
255
|
+
fail('reply', `No reply with the token ${token}`);
|
|
256
|
+
else
|
|
257
|
+
pass('reply', token);
|
|
258
|
+
if (failure)
|
|
259
|
+
await readScreen();
|
|
260
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
const step = !teamId ? 'create_team' : !sessionName ? 'start_member' : 'send_task';
|
|
264
|
+
fail(step, err instanceof Error ? err.message : String(err));
|
|
265
|
+
return this.result(runtime, started, steps, failure, screen);
|
|
266
|
+
}
|
|
267
|
+
finally {
|
|
268
|
+
stepStart = this.now();
|
|
269
|
+
const cleanupErrors = [];
|
|
270
|
+
if (teamId) {
|
|
271
|
+
await api.stopTeam(teamId).catch((e) => cleanupErrors.push(String(e)));
|
|
272
|
+
await api.deleteTeam(teamId).catch((e) => cleanupErrors.push(String(e)));
|
|
273
|
+
}
|
|
274
|
+
// A start the runtime refused (e.g. Antigravity's first-run screen)
|
|
275
|
+
// leaves a PTY the team no longer knows about.
|
|
276
|
+
if (sessionName && (await api.sessionExists(sessionName).catch(() => false))) {
|
|
277
|
+
await api.killSession(sessionName).catch((e) => cleanupErrors.push(String(e)));
|
|
278
|
+
}
|
|
279
|
+
if (projectId)
|
|
280
|
+
await api.deleteProject(projectId).catch((e) => cleanupErrors.push(String(e)));
|
|
281
|
+
try {
|
|
282
|
+
fs.rmSync(workDir, { recursive: true, force: true });
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
// scratch only
|
|
286
|
+
}
|
|
287
|
+
steps.push({ step: 'cleanup', ok: cleanupErrors.length === 0, ms: this.now() - stepStart, ...(cleanupErrors.length ? { detail: cleanupErrors.join('; ') } : {}) });
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
/** Remove a team / project left over from an earlier run. */
|
|
291
|
+
async removeLeftovers(name) {
|
|
292
|
+
const api = this.deps.api;
|
|
293
|
+
for (const team of (await api.listTeams().catch(() => [])).filter((t) => t.name === name)) {
|
|
294
|
+
await api.stopTeam(team.id).catch(() => undefined);
|
|
295
|
+
await api.deleteTeam(team.id).catch(() => undefined);
|
|
296
|
+
}
|
|
297
|
+
for (const project of (await api.listProjects().catch(() => [])).filter((p) => p.name === name)) {
|
|
298
|
+
await api.deleteProject(project.id).catch(() => undefined);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
result(runtime, started, steps, failure, screen) {
|
|
302
|
+
const tail = screen.split('\n').slice(-S.SCREEN_LINES).join('\n').trim();
|
|
303
|
+
return {
|
|
304
|
+
runtime,
|
|
305
|
+
passed: failure === null,
|
|
306
|
+
...(failure ? { failedStep: failure.step, error: failure.error } : {}),
|
|
307
|
+
steps,
|
|
308
|
+
...(failure && tail ? { screen: tail } : {}),
|
|
309
|
+
durationMs: this.now() - started,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Read the proof file.
|
|
315
|
+
*
|
|
316
|
+
* @param file - Path
|
|
317
|
+
* @returns Its trimmed text, or null
|
|
318
|
+
*/
|
|
319
|
+
function readProof(file) {
|
|
320
|
+
try {
|
|
321
|
+
return fs.readFileSync(file, 'utf-8').trim();
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
/** Fetch-based client for the local API (the default {@link SmokeApi}). */
|
|
328
|
+
export class LocalSmokeApi {
|
|
329
|
+
baseUrl;
|
|
330
|
+
token;
|
|
331
|
+
/**
|
|
332
|
+
* @param baseUrl - e.g. http://127.0.0.1:8787
|
|
333
|
+
* @param token - Crewly API token
|
|
334
|
+
*/
|
|
335
|
+
constructor(baseUrl, token) {
|
|
336
|
+
this.baseUrl = baseUrl;
|
|
337
|
+
this.token = token;
|
|
338
|
+
}
|
|
339
|
+
async call(method, route, body) {
|
|
340
|
+
const res = await fetch(`${this.baseUrl()}/api${route}`, {
|
|
341
|
+
method,
|
|
342
|
+
headers: {
|
|
343
|
+
'content-type': 'application/json',
|
|
344
|
+
'x-crewly-token': this.token(),
|
|
345
|
+
// The owner's dashboard: the wake / commitment gates are for agents.
|
|
346
|
+
'x-crewly-caller': 'dashboard',
|
|
347
|
+
},
|
|
348
|
+
...(body !== undefined ? { body: JSON.stringify(body) } : {}),
|
|
349
|
+
});
|
|
350
|
+
const json = (await res.json().catch(() => ({})));
|
|
351
|
+
if (!res.ok || json.success === false)
|
|
352
|
+
throw new Error(json.error ?? `${method} ${route} → HTTP ${res.status}`);
|
|
353
|
+
return json;
|
|
354
|
+
}
|
|
355
|
+
async listTeams() {
|
|
356
|
+
const r = await this.call('GET', '/teams');
|
|
357
|
+
return r.data ?? [];
|
|
358
|
+
}
|
|
359
|
+
async listProjects() {
|
|
360
|
+
const r = await this.call('GET', '/projects');
|
|
361
|
+
return r.data ?? [];
|
|
362
|
+
}
|
|
363
|
+
async createProject(name, projectPath) {
|
|
364
|
+
const r = await this.call('POST', '/projects', { name, path: projectPath, description: 'Runtime smoke test (temporary)' });
|
|
365
|
+
return r.data.id;
|
|
366
|
+
}
|
|
367
|
+
async deleteProject(projectId) {
|
|
368
|
+
await this.call('DELETE', `/projects/${encodeURIComponent(projectId)}`);
|
|
369
|
+
}
|
|
370
|
+
async createTeam(body) {
|
|
371
|
+
const r = await this.call('POST', '/teams', body);
|
|
372
|
+
return { id: r.data.id, memberId: r.data.members[0].id };
|
|
373
|
+
}
|
|
374
|
+
async startMember(teamId, memberId) {
|
|
375
|
+
try {
|
|
376
|
+
await this.call('POST', `/teams/${encodeURIComponent(teamId)}/members/${encodeURIComponent(memberId)}/start`, {});
|
|
377
|
+
return { ok: true };
|
|
378
|
+
}
|
|
379
|
+
catch (err) {
|
|
380
|
+
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
async getMember(teamId, memberId) {
|
|
384
|
+
const r = await this.call('GET', `/teams/${encodeURIComponent(teamId)}`);
|
|
385
|
+
const m = r.data.members.find((x) => x.id === memberId);
|
|
386
|
+
return m ? { agentStatus: m.agentStatus, sessionName: m.sessionName } : null;
|
|
387
|
+
}
|
|
388
|
+
async capture(sessionName, lines) {
|
|
389
|
+
const r = await this.call('GET', `/terminal/${encodeURIComponent(sessionName)}/output?lines=${lines}`);
|
|
390
|
+
return typeof r.data === 'string' ? r.data : r.data?.output ?? '';
|
|
391
|
+
}
|
|
392
|
+
async sessionLog(sessionName, lines) {
|
|
393
|
+
const r = await this.call('GET', `/sessions/${encodeURIComponent(sessionName)}/logs?lines=${lines}`);
|
|
394
|
+
return (r.data?.lines ?? []).join('\n');
|
|
395
|
+
}
|
|
396
|
+
async deliver(sessionName, message) {
|
|
397
|
+
try {
|
|
398
|
+
await this.call('POST', `/terminal/${encodeURIComponent(sessionName)}/deliver`, { message });
|
|
399
|
+
return { ok: true };
|
|
400
|
+
}
|
|
401
|
+
catch (err) {
|
|
402
|
+
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
async killSession(sessionName) {
|
|
406
|
+
await this.call('DELETE', `/terminal/${encodeURIComponent(sessionName)}`);
|
|
407
|
+
}
|
|
408
|
+
async sessionExists(sessionName) {
|
|
409
|
+
const r = await this.call('GET', `/terminal/${encodeURIComponent(sessionName)}/exists`);
|
|
410
|
+
return r.data?.exists === true;
|
|
411
|
+
}
|
|
412
|
+
async stopTeam(teamId) {
|
|
413
|
+
await this.call('POST', `/teams/${encodeURIComponent(teamId)}/stop`, {});
|
|
414
|
+
}
|
|
415
|
+
async deleteTeam(teamId) {
|
|
416
|
+
await this.call('DELETE', `/teams/${encodeURIComponent(teamId)}`);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
//# sourceMappingURL=runtime-smoke-test.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-smoke-test.service.js","sourceRoot":"","sources":["../../../../../../backend/src/services/runtime-fallback/runtime-smoke-test.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtG,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AACnF,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AACnF,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,MAAM,CAAC,GAAG,0BAA0B,CAAC,KAAK,CAAC;AAgE3C,uDAAuD;AACvD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE3C;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAc;IACnD,6EAA6E;IAC7E,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,qBAAqB,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/H,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa;IACzD,OAAO;QACN,yEAAyE;QACzE,+DAA+D,IAAI,EAAE;QACrE,iHAAiH,KAAK,EAAE;YACvH,oHAAoH;YACpH,8DAA8D;KAC/D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,6CAA6C;AAC7C,MAAM,OAAO,uBAAuB;IASN;IARZ,IAAI,GAAG,IAAI,GAAG,EAAwB,CAAC;IACvC,OAAO,GAAG,IAAI,GAAG,EAAoC,CAAC;IACtD,GAAG,CAAe;IAClB,KAAK,CAAgC;IAEtD;;OAEG;IACH,YAA6B,IAAmB;QAAnB,SAAI,GAAJ,IAAI,CAAe;QAC/C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAe;QACpB,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;QACtF,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QACrG,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAA6B,EAAE,CAAC;QAC3G,MAAM,GAAG,GAAiB,EAAE,KAAK,EAAE,SAAS,OAAO,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACpK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9C,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC;YACnB,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAClC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,GAAG,CAAC,KAAa;QAChB,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAEO,KAAK;QACZ,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG,CAAC,OAAe;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC;QAC7C,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,WAAW,GAAG,OAAO,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;QACvH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,KAAK,MAAM,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,YAAY,KAAK,EAAE,CAAC;QAClC,MAAM,KAAK,GAA6B,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,MAAM,IAAI,GAAG,CAAC,IAAe,EAAE,MAAe,EAAQ,EAAE;YACvD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1F,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,CAAC,CAAC;QACF,wEAAwE;QACxE,MAAM,UAAU,GAAG,KAAK,IAAqB,EAAE;YAC9C,IAAI,CAAC,WAAW;gBAAE,OAAO,MAAM,CAAC;YAChC,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACR,sCAAsC;YACvC,CAAC;YACD,IAAI,CAAC,IAAI;gBAAE,IAAI,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrF,IAAI,IAAI;gBAAE,MAAM,GAAG,aAAa,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YACrE,OAAO,MAAM,CAAC;QACf,CAAC,CAAC;QAEF,IAAI,OAAO,GAA8C,IAAI,CAAC;QAC9D,MAAM,IAAI,GAAG,CAAC,IAAe,EAAE,KAAa,EAAQ,EAAE;YACrD,OAAO,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC;QACF,MAAM,UAAU,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,OAAO,KAAK,aAAa,CAAC,eAAe,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACvH,MAAM,KAAK,GAAG,2CAA2C,CAAC;QAE1D,IAAI,CAAC;YACJ,yEAAyE;YACzE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACjC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3C,SAAS,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC;gBACpC,IAAI;gBACJ,WAAW,EAAE,+DAA+D;gBAC5E,UAAU,EAAE,CAAC,SAAS,CAAC;gBACvB,OAAO,EAAE;oBACR;wBACC,IAAI,EAAE,CAAC,CAAC,WAAW;wBACnB,IAAI,EAAE,CAAC,CAAC,WAAW;wBACnB,WAAW,EAAE,OAAO;wBACpB,GAAG,CAAC,OAAO,KAAK,aAAa,CAAC,YAAY;4BACzC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,0BAA0B,CAAC,0BAA0B,EAAE;4BACtG,CAAC,CAAC,EAAE,CAAC;wBACN,YAAY,EAAE,iFAAiF;qBAC/F;iBACD;aACD,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAE1B,WAAW;YACX,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YACnF,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,uBAAuB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YACpG,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,MAAM,UAAU,EAAE,CAAC;gBAChC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,qCAAqC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,CAAC;oBAAE,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;;oBACpH,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,KAAK,IAAI,0BAA0B,CAAC,CAAC;gBAC3E,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,CAAC;YAErB,uDAAuD;YACvD,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC9E,IAAI,CAAC,EAAE,WAAW;oBAAE,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;gBAChD,IAAI,UAAU,CAAC,MAAM,UAAU,EAAE,CAAC,EAAE,CAAC;oBACpC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;oBAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC;gBACD,IAAI,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC5C,KAAK,GAAG,IAAI,CAAC;oBACb,MAAM;gBACP,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,IAAI,CAAC,aAAa,EAAE,oCAAoC,CAAC,CAAC;gBAC1D,MAAM,UAAU,EAAE,CAAC;gBACnB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAEjC,UAAU;YACV,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;YACnF,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACnB,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,KAAK,IAAI,iCAAiC,CAAC,CAAC;gBACxE,MAAM,UAAU,EAAE,CAAC;gBACnB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,CAAC;YAElB,kCAAkC;YAClC,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;gBACtD,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,GAAG,IAAI,CAAC;oBACd,IAAI,CAAC,MAAM,CAAC,CAAC;gBACd,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,UAAU,EAAE,CAAC;gBAChC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACvC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC;gBACD,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAAE,OAAO,GAAG,IAAI,CAAC;gBACnG,IAAI,MAAM,IAAI,OAAO;oBAAE,MAAM;gBAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,CAAC,MAAM;gBAAE,IAAI,CAAC,MAAM,EAAE,iDAAiD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;iBACnG,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,2BAA2B,KAAK,EAAE,CAAC,CAAC;;gBAChE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC1B,IAAI,OAAO;gBAAE,MAAM,UAAU,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,GAAc,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC;YAC9F,IAAI,CAAC,IAAI,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,CAAC;gBAAS,CAAC;YACV,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,aAAa,GAAa,EAAE,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChF,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,oEAAoE;YACpE,+CAA+C;YAC/C,IAAI,WAAW,IAAI,CAAC,MAAM,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC9E,MAAM,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;YACD,IAAI,SAAS;gBAAE,MAAM,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvG,IAAI,CAAC;gBACJ,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACR,eAAe;YAChB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpK,CAAC;IACF,CAAC;IAED,6DAA6D;IACrD,KAAK,CAAC,eAAe,CAAC,IAAY;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YAC3F,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACnD,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACjG,MAAM,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,CAAC;IACF,CAAC;IAEO,MAAM,CACb,OAAe,EACf,OAAe,EACf,KAA+B,EAC/B,OAAkD,EAClD,MAAc;QAEd,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,OAAO;YACN,OAAO;YACP,MAAM,EAAE,OAAO,KAAK,IAAI;YACxB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,KAAK;YACL,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SAChC,CAAC;IACH,CAAC;CACD;AAED;;;;;GAKG;AACH,SAAS,SAAS,CAAC,IAAY;IAC9B,IAAI,CAAC;QACJ,OAAO,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,2EAA2E;AAC3E,MAAM,OAAO,aAAa;IAMP;IACA;IANlB;;;OAGG;IACH,YACkB,OAAqB,EACrB,KAAmB;QADnB,YAAO,GAAP,OAAO,CAAc;QACrB,UAAK,GAAL,KAAK,CAAc;IAClC,CAAC;IAEI,KAAK,CAAC,IAAI,CAAI,MAAc,EAAE,KAAa,EAAE,IAAc;QAClE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,KAAK,EAAE,EAAE;YACxD,MAAM;YACN,OAAO,EAAE;gBACR,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC9B,qEAAqE;gBACrE,iBAAiB,EAAE,WAAW;aAC9B;YACD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAA0D,CAAC;QAC3G,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI,KAAK,WAAW,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,OAAO,IAAS,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAgD,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,YAAY;QACjB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAgD,KAAK,EAAE,WAAW,CAAC,CAAC;QAC7F,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,IAAY,EAAE,WAAmB;QACpD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAA2B,MAAM,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,gCAAgC,EAAE,CAAC,CAAC;QACrJ,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB;QACpC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAA6B;QAC7C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAA2D,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC5G,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,QAAgB;QACjD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,kBAAkB,CAAC,MAAM,CAAC,YAAY,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAClH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,QAAgB;QAC/C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAyF,KAAK,EAAE,UAAU,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjK,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QACxD,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,WAAmB,EAAE,KAAa;QAC/C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAyC,KAAK,EAAE,aAAa,kBAAkB,CAAC,WAAW,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;QAC/I,OAAO,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,KAAa;QAClD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiC,KAAK,EAAE,aAAa,kBAAkB,CAAC,WAAW,CAAC,eAAe,KAAK,EAAE,CAAC,CAAC;QACrI,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,WAAmB,EAAE,OAAe;QACjD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,kBAAkB,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7F,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,WAAmB;QACpC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,kBAAkB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACtC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiC,KAAK,EAAE,aAAa,kBAAkB,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACxH,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC5B,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;CACD"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage probe — does a runtime have usage right now?
|
|
3
|
+
*
|
|
4
|
+
* Reuses the harness-login-probe approach: for Claude Code one tiny
|
|
5
|
+
* print-mode turn (`claude -p … --model haiku --no-session-persistence`) in
|
|
6
|
+
* a scratch directory, with the env an agent gets. Output that matches a
|
|
7
|
+
* usage-limit rule → `limited`; exit 0 → `available`; anything else
|
|
8
|
+
* (network, login, timeout) → `unknown`.
|
|
9
|
+
*
|
|
10
|
+
* For the in-process Crewly Agent: one tiny API call (`max_tokens: 1`) to
|
|
11
|
+
* each provider its agents use, with the stored key. HTTP 402 / a usage or
|
|
12
|
+
* billing error → `limited`; 2xx → `available`; anything else → `unknown`.
|
|
13
|
+
*
|
|
14
|
+
* Other runtimes return `unsupported` (a probe would spend their usage or
|
|
15
|
+
* needs a login Crewly cannot check cheaply); the fallback then relies on the
|
|
16
|
+
* parsed reset time. `unknown` (the probe ran but could not tell) never
|
|
17
|
+
* switches an agent back.
|
|
18
|
+
*
|
|
19
|
+
* The probe never prints or logs a secret.
|
|
20
|
+
*
|
|
21
|
+
* @module services/runtime-fallback/runtime-usage-probe
|
|
22
|
+
*/
|
|
23
|
+
import { type HarnessCredentialsStore } from '../harness/harness-credentials.store.js';
|
|
24
|
+
import type { RunCommand } from '../harness/harness.types.js';
|
|
25
|
+
import type { ProbeResult } from './runtime-fallback.service.js';
|
|
26
|
+
/** One provider the in-process Crewly Agent uses, with its key. */
|
|
27
|
+
export interface CrewlyAgentProbeTarget {
|
|
28
|
+
/** Provider id (`deepseek`, `openai`, `anthropic`, `google`, …) */
|
|
29
|
+
provider: string;
|
|
30
|
+
/** API key (never logged) */
|
|
31
|
+
apiKey: string;
|
|
32
|
+
/** Model id without the provider prefix, when an agent names one */
|
|
33
|
+
model?: string;
|
|
34
|
+
}
|
|
35
|
+
/** Minimal fetch. */
|
|
36
|
+
export type ProbeFetch = (url: string, init: {
|
|
37
|
+
method: string;
|
|
38
|
+
headers: Record<string, string>;
|
|
39
|
+
body: string;
|
|
40
|
+
signal?: AbortSignal;
|
|
41
|
+
}) => Promise<{
|
|
42
|
+
status: number;
|
|
43
|
+
text(): Promise<string>;
|
|
44
|
+
}>;
|
|
45
|
+
/** Injectable dependencies. */
|
|
46
|
+
export interface UsageProbeDeps {
|
|
47
|
+
/** Providers (with keys) the Crewly Agent runtime uses; empty → `unknown` */
|
|
48
|
+
crewlyAgentTargets?: () => Promise<CrewlyAgentProbeTarget[]>;
|
|
49
|
+
fetch?: ProbeFetch;
|
|
50
|
+
run?: RunCommand;
|
|
51
|
+
env?: NodeJS.ProcessEnv;
|
|
52
|
+
homeDir?: string;
|
|
53
|
+
resolveCommand?: (command: string) => string | null;
|
|
54
|
+
credentials?: Pick<HarnessCredentialsStore, 'harnessEnvForAgents'>;
|
|
55
|
+
scratchDir?: string;
|
|
56
|
+
timeoutMs?: number;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Classify a Claude probe turn.
|
|
60
|
+
*
|
|
61
|
+
* @param result - Exit code and output
|
|
62
|
+
* @returns `limited`, `available` or `unknown`
|
|
63
|
+
*
|
|
64
|
+
* @example
|
|
65
|
+
* ```ts
|
|
66
|
+
* classifyClaudeUsageProbe({ code: 1, stdout: "You've hit your limit · resets 3pm", stderr: '' }); // 'limited'
|
|
67
|
+
* ```
|
|
68
|
+
*/
|
|
69
|
+
export declare function classifyClaudeUsageProbe(result: {
|
|
70
|
+
code: number | null;
|
|
71
|
+
stdout: string;
|
|
72
|
+
stderr: string;
|
|
73
|
+
}): ProbeResult;
|
|
74
|
+
/**
|
|
75
|
+
* Classify a provider's answer to a one-token request.
|
|
76
|
+
*
|
|
77
|
+
* @param status - HTTP status
|
|
78
|
+
* @param body - Response body (never logged)
|
|
79
|
+
* @returns `available` on 2xx, `limited` on 402 or a usage/billing error, else `unknown`
|
|
80
|
+
*
|
|
81
|
+
* @example
|
|
82
|
+
* ```ts
|
|
83
|
+
* classifyProviderProbe(402, '{"error":{"message":"Insufficient Balance"}}'); // 'limited'
|
|
84
|
+
* ```
|
|
85
|
+
*/
|
|
86
|
+
export declare function classifyProviderProbe(status: number, body: string): ProbeResult;
|
|
87
|
+
/**
|
|
88
|
+
* Build the probe.
|
|
89
|
+
*
|
|
90
|
+
* @param deps - Dependencies (defaults: the real CLI and stored credentials)
|
|
91
|
+
* @returns `probe(runtime)`; never rejects
|
|
92
|
+
*/
|
|
93
|
+
export declare function createRuntimeUsageProbe(deps?: UsageProbeDeps): (runtime: string) => Promise<ProbeResult>;
|
|
94
|
+
//# sourceMappingURL=runtime-usage-probe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-usage-probe.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/runtime-fallback/runtime-usage-probe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH,OAAO,EAA8B,KAAK,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAEnH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAEjE,mEAAmE;AACnE,MAAM,WAAW,sBAAsB;IACtC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAqB;AACrB,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,KAAK,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC,CAAC;AAE9L,+BAA+B;AAC/B,MAAM,WAAW,cAAc;IAC9B,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAC;IAC7D,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACpD,WAAW,CAAC,EAAE,IAAI,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,CAAC;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAKD;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAMrH;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAM/E;AA8ED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,GAAE,cAAmB,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CA4B5G"}
|