dsh-ops 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/LICENSE +30 -0
- package/NOTICE +106 -0
- package/PROVENANCE.md +435 -0
- package/README.en.md +126 -0
- package/README.md +115 -2
- package/README.zh.md +116 -0
- package/bin/dsh-ops.mjs +1216 -0
- package/cordis.patch.yml +160 -0
- package/docs/manual-validation.md +53 -0
- package/docs/release-0.2.1.md +72 -0
- package/docs/schema-baseline.json +64 -0
- package/docs/schema-current.json +84 -0
- package/docs/schema-measurement.md +17 -0
- package/dsh-plugin.json +88 -0
- package/icon.svg +12 -0
- package/lib/binary.js +409 -0
- package/lib/config.js +198 -0
- package/lib/handshake.js +252 -0
- package/lib/index.js +108 -0
- package/lib/jobs.js +42 -0
- package/lib/policy.js +64 -0
- package/lib/presentation.js +63 -0
- package/lib/profile-install.js +61 -0
- package/lib/rust.js +194 -0
- package/lib/session-shells.js +78 -0
- package/lib/shells.js +998 -0
- package/lib/tools.js +657 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +114 -4
- package/vendor/fastctx/Cargo.lock +3210 -0
- package/vendor/fastctx/Cargo.toml +94 -0
- package/vendor/fastctx/FORK.md +119 -0
- package/vendor/fastctx/LICENSE-APACHE +201 -0
- package/vendor/fastctx/NOTICE +40 -0
- package/vendor/fastctx/README.md +439 -0
- package/vendor/fastctx/THIRD_PARTY_LICENSES.md +17 -0
- package/vendor/fastctx/THIRD_PARTY_LICENSES_RUST.md +7914 -0
- package/vendor/fastctx/UPSTREAM.md +49 -0
- package/vendor/fastctx/build.rs +413 -0
- package/vendor/fastctx/src/background_status.rs +403 -0
- package/vendor/fastctx/src/binary.rs +75 -0
- package/vendor/fastctx/src/bounded_sort.rs +500 -0
- package/vendor/fastctx/src/budget.rs +781 -0
- package/vendor/fastctx/src/cli/mod.rs +110 -0
- package/vendor/fastctx/src/context_guard.rs +289 -0
- package/vendor/fastctx/src/control/mod.rs +6 -0
- package/vendor/fastctx/src/control/paths.rs +49 -0
- package/vendor/fastctx/src/control/settings.rs +753 -0
- package/vendor/fastctx/src/control/transaction.rs +531 -0
- package/vendor/fastctx/src/edit/document.rs +535 -0
- package/vendor/fastctx/src/edit/locks.rs +371 -0
- package/vendor/fastctx/src/edit/mod.rs +213 -0
- package/vendor/fastctx/src/edit/private_storage/unix.rs +315 -0
- package/vendor/fastctx/src/edit/private_storage/windows.rs +793 -0
- package/vendor/fastctx/src/edit/private_storage.rs +234 -0
- package/vendor/fastctx/src/edit/replace.rs +1030 -0
- package/vendor/fastctx/src/edit_server.rs +53 -0
- package/vendor/fastctx/src/encoding/reference_v011.rs +587 -0
- package/vendor/fastctx/src/encoding/snapshot_pipeline.rs +1678 -0
- package/vendor/fastctx/src/encoding.rs +1118 -0
- package/vendor/fastctx/src/file_executor.rs +1151 -0
- package/vendor/fastctx/src/file_snapshot.rs +1491 -0
- package/vendor/fastctx/src/glob_filter.rs +98 -0
- package/vendor/fastctx/src/glob_tool.rs +653 -0
- package/vendor/fastctx/src/grep_sink.rs +1162 -0
- package/vendor/fastctx/src/grep_tool.rs +2449 -0
- package/vendor/fastctx/src/lib.rs +45 -0
- package/vendor/fastctx/src/main.rs +15 -0
- package/vendor/fastctx/src/model.rs +51 -0
- package/vendor/fastctx/src/model_guidance.rs +62 -0
- package/vendor/fastctx/src/operation.rs +356 -0
- package/vendor/fastctx/src/ordered_window.rs +1235 -0
- package/vendor/fastctx/src/os_environment.rs +414 -0
- package/vendor/fastctx/src/path_codec.rs +850 -0
- package/vendor/fastctx/src/paths.rs +244 -0
- package/vendor/fastctx/src/process_identity.rs +763 -0
- package/vendor/fastctx/src/process_policy.rs +74 -0
- package/vendor/fastctx/src/read_tool/batch.rs +496 -0
- package/vendor/fastctx/src/read_tool/hex_file.rs +141 -0
- package/vendor/fastctx/src/read_tool/image_file.rs +88 -0
- package/vendor/fastctx/src/read_tool/mod.rs +245 -0
- package/vendor/fastctx/src/read_tool/pdf.rs +470 -0
- package/vendor/fastctx/src/read_tool/pdf_disabled.rs +47 -0
- package/vendor/fastctx/src/read_tool/pdf_engine.rs +664 -0
- package/vendor/fastctx/src/read_tool/text_file.rs +351 -0
- package/vendor/fastctx/src/render_plan.rs +468 -0
- package/vendor/fastctx/src/runtime/activity.rs +159 -0
- package/vendor/fastctx/src/runtime/hosts.rs +99 -0
- package/vendor/fastctx/src/runtime/journal.rs +556 -0
- package/vendor/fastctx/src/runtime/local_ipc.rs +186 -0
- package/vendor/fastctx/src/runtime/mod.rs +746 -0
- package/vendor/fastctx/src/runtime/protocol.rs +296 -0
- package/vendor/fastctx/src/runtime/session.rs +536 -0
- package/vendor/fastctx/src/runtime/windows_process.rs +66 -0
- package/vendor/fastctx/src/search_parallelism.rs +106 -0
- package/vendor/fastctx/src/search_text.rs +227 -0
- package/vendor/fastctx/src/server.rs +359 -0
- package/vendor/fastctx/src/server_manifest.rs +468 -0
- package/vendor/fastctx/src/server_support.rs +826 -0
- package/vendor/fastctx/src/session.rs +629 -0
- package/vendor/fastctx/src/shell/apply_patch_hint.rs +41 -0
- package/vendor/fastctx/src/shell/bash.rs +263 -0
- package/vendor/fastctx/src/shell/buffer.rs +108 -0
- package/vendor/fastctx/src/shell/encoding.rs +403 -0
- package/vendor/fastctx/src/shell/foreground.rs +115 -0
- package/vendor/fastctx/src/shell/jobs/admission.rs +91 -0
- package/vendor/fastctx/src/shell/jobs/background.rs +146 -0
- package/vendor/fastctx/src/shell/jobs/host.rs +830 -0
- package/vendor/fastctx/src/shell/jobs/identity.rs +29 -0
- package/vendor/fastctx/src/shell/jobs/mod.rs +1513 -0
- package/vendor/fastctx/src/shell/jobs/model.rs +244 -0
- package/vendor/fastctx/src/shell/jobs/output_log.rs +1148 -0
- package/vendor/fastctx/src/shell/jobs/store.rs +1300 -0
- package/vendor/fastctx/src/shell/mod.rs +345 -0
- package/vendor/fastctx/src/shell/normalize.rs +389 -0
- package/vendor/fastctx/src/shell/output.rs +406 -0
- package/vendor/fastctx/src/shell/process.rs +493 -0
- package/vendor/fastctx/src/shell_server.rs +156 -0
- package/vendor/fastctx/src/skip_report.rs +83 -0
- package/vendor/fastctx/src/stdio_transport.rs +177 -0
- package/vendor/fastctx/src/tool_schema.rs +204 -0
- package/vendor/fastctx/src/traversal.rs +846 -0
- package/vendor/fastctx/third-party/pdfium-7763/LICENSE +9 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/abseil.txt +202 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/agg23.txt +14 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/fast_float.txt +27 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/freetype.txt +169 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/icu.txt +542 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/lcms.txt +27 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.ijg +260 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.md +135 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libopenjpeg.txt +32 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libpng.txt +134 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libtiff.txt +21 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/llvm-libc.txt +278 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/pdfium.txt +230 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/simdutf.txt +18 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/zlib.txt +29 -0
|
@@ -0,0 +1,1513 @@
|
|
|
1
|
+
// Modified by dsh-ops (https://github.com/T-Auto/dsh-ops): removed the TUI job-tail reader, the Apply/Unapply admission and kill helpers, and the permission-denied classification only the TUI read.
|
|
2
|
+
//! Persistent background jobs whose supervisors and records outlive every MCP server session.
|
|
3
|
+
|
|
4
|
+
pub(crate) mod admission;
|
|
5
|
+
mod background;
|
|
6
|
+
mod host;
|
|
7
|
+
mod identity;
|
|
8
|
+
mod model;
|
|
9
|
+
mod output_log;
|
|
10
|
+
mod store;
|
|
11
|
+
|
|
12
|
+
use crate::budget::{
|
|
13
|
+
GLOBAL_TOKEN_BUDGET_ENV, JOB_OUTPUT_TOKEN_BUDGET_ENV, TokenBudget, estimate_tokens,
|
|
14
|
+
relax_tool_token_budget, tool_token_budget_for_required,
|
|
15
|
+
};
|
|
16
|
+
use crate::control::paths::ControlPaths;
|
|
17
|
+
use crate::model::ToolResponse;
|
|
18
|
+
use crate::paths::display_path;
|
|
19
|
+
use crate::shell::JobListStatus;
|
|
20
|
+
use crate::shell::encoding::{
|
|
21
|
+
OutputEncoding, decode_job, job_garble_note, validate_output_encoding,
|
|
22
|
+
};
|
|
23
|
+
use crate::shell::output::{
|
|
24
|
+
budget_too_small_message, compose_response_with_tail, global_token_budget,
|
|
25
|
+
job_output_token_budget, plural, terminal_response,
|
|
26
|
+
};
|
|
27
|
+
use model::{JobRecord, JobStatus, LaunchSpec, StoredLine};
|
|
28
|
+
use std::collections::HashMap;
|
|
29
|
+
use std::fmt::{Display, Formatter};
|
|
30
|
+
use std::path::{Path, PathBuf};
|
|
31
|
+
use std::sync::{Arc, Mutex};
|
|
32
|
+
use std::time::{Duration, Instant, SystemTime};
|
|
33
|
+
|
|
34
|
+
const KILL_ACK_TIMEOUT: Duration = Duration::from_secs(6);
|
|
35
|
+
const REGISTRY_POLL: Duration = Duration::from_millis(20);
|
|
36
|
+
|
|
37
|
+
#[derive(Clone, Debug)]
|
|
38
|
+
pub(crate) struct JobManager {
|
|
39
|
+
paths: Result<ControlPaths, String>,
|
|
40
|
+
executable: Result<PathBuf, String>,
|
|
41
|
+
admission_generation: Result<u64, String>,
|
|
42
|
+
cursors: Arc<Mutex<HashMap<String, u64>>>,
|
|
43
|
+
background: background::BackgroundTracker,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
pub(crate) struct BackgroundLaunch<'a> {
|
|
47
|
+
pub(crate) bash: &'a Path,
|
|
48
|
+
pub(crate) command: &'a str,
|
|
49
|
+
pub(crate) cwd: &'a Path,
|
|
50
|
+
pub(crate) login_shell: bool,
|
|
51
|
+
pub(crate) encoding: Option<OutputEncoding>,
|
|
52
|
+
pub(crate) environment: &'a crate::session::SessionEnvironment,
|
|
53
|
+
pub(crate) utf8_locale: &'a str,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
#[derive(Clone, Debug)]
|
|
57
|
+
struct OutputSnapshot {
|
|
58
|
+
status: JobStatus,
|
|
59
|
+
head: Vec<StoredLine>,
|
|
60
|
+
tail: Vec<StoredLine>,
|
|
61
|
+
unread_first: u64,
|
|
62
|
+
unread_last: u64,
|
|
63
|
+
all_unread_loaded: bool,
|
|
64
|
+
total_lines: u64,
|
|
65
|
+
legacy_loss: bool,
|
|
66
|
+
capture_error: Option<model::CaptureErrorRecord>,
|
|
67
|
+
output_truncation: Option<model::OutputTruncationRecord>,
|
|
68
|
+
default_encoding: Option<OutputEncoding>,
|
|
69
|
+
anchor: u64,
|
|
70
|
+
direct_log: Option<PathBuf>,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
#[derive(Clone, Debug)]
|
|
74
|
+
struct FormattedPage {
|
|
75
|
+
response: String,
|
|
76
|
+
cursor_seq: Option<u64>,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Stable control-plane view of one persistent job record.
|
|
80
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
81
|
+
pub(crate) struct JobSummary {
|
|
82
|
+
pub(crate) id: String,
|
|
83
|
+
pub(crate) command: String,
|
|
84
|
+
pub(crate) cwd: String,
|
|
85
|
+
pub(crate) started_at: String,
|
|
86
|
+
pub(crate) status: JobSummaryStatus,
|
|
87
|
+
pub(crate) source: JobSourceSummary,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/// Stable best-effort source identity for grouping jobs from distinct server sessions.
|
|
91
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
92
|
+
pub(crate) struct JobSourceSummary {
|
|
93
|
+
pub(crate) key: String,
|
|
94
|
+
pub(crate) tag: String,
|
|
95
|
+
pub(crate) server_pid: u32,
|
|
96
|
+
pub(crate) parent_executable: Option<String>,
|
|
97
|
+
pub(crate) server_cwd: String,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Public three-state lifecycle used by CLI and TUI without exposing storage internals.
|
|
101
|
+
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
102
|
+
pub(crate) enum JobSummaryStatus {
|
|
103
|
+
Running,
|
|
104
|
+
Exited(i32),
|
|
105
|
+
Interrupted,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// Diagnosable registry failure.
|
|
109
|
+
#[derive(Debug)]
|
|
110
|
+
pub(crate) struct JobRegistryError {
|
|
111
|
+
message: String,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
impl JobRegistryError {
|
|
115
|
+
pub(super) fn from_io(context: String, error: std::io::Error) -> Self {
|
|
116
|
+
Self {
|
|
117
|
+
message: format!("{context}: {error}"),
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
pub(super) fn data(message: String) -> Self {
|
|
122
|
+
Self { message }
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
impl Display for JobRegistryError {
|
|
127
|
+
fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
|
|
128
|
+
formatter.write_str(&self.message)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
impl std::error::Error for JobRegistryError {}
|
|
133
|
+
|
|
134
|
+
impl From<JobRegistryError> for String {
|
|
135
|
+
fn from(error: JobRegistryError) -> Self {
|
|
136
|
+
error.message
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
141
|
+
enum KillState {
|
|
142
|
+
Killed,
|
|
143
|
+
AlreadyExited(i32),
|
|
144
|
+
AlreadyInterrupted,
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
impl JobManager {
|
|
148
|
+
pub(crate) fn new() -> Self {
|
|
149
|
+
Self::with_session(crate::session::SessionContext::library_default())
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
pub(crate) fn with_session(session: Arc<crate::session::SessionContext>) -> Self {
|
|
153
|
+
let paths = Ok(session.control_paths.clone());
|
|
154
|
+
let admission_generation = paths
|
|
155
|
+
.as_ref()
|
|
156
|
+
.map_err(Clone::clone)
|
|
157
|
+
.and_then(admission::observe_generation);
|
|
158
|
+
Self {
|
|
159
|
+
paths,
|
|
160
|
+
executable: std::env::current_exe()
|
|
161
|
+
.map_err(|error| format!("Cannot locate the running fastctx binary: {error}")),
|
|
162
|
+
admission_generation,
|
|
163
|
+
cursors: Arc::new(Mutex::new(HashMap::new())),
|
|
164
|
+
background: background::BackgroundTracker::default(),
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
pub(crate) fn start(&self, launch: BackgroundLaunch<'_>) -> ToolResponse {
|
|
169
|
+
let BackgroundLaunch {
|
|
170
|
+
bash,
|
|
171
|
+
command,
|
|
172
|
+
cwd,
|
|
173
|
+
login_shell,
|
|
174
|
+
encoding,
|
|
175
|
+
environment,
|
|
176
|
+
utf8_locale,
|
|
177
|
+
} = launch;
|
|
178
|
+
let paths = match self.paths() {
|
|
179
|
+
Ok(paths) => paths,
|
|
180
|
+
Err(error) => return ToolResponse::error(error),
|
|
181
|
+
};
|
|
182
|
+
let executable = match self.executable.as_ref() {
|
|
183
|
+
Ok(executable) => executable,
|
|
184
|
+
Err(error) => return ToolResponse::error(error.clone()),
|
|
185
|
+
};
|
|
186
|
+
let admission_generation = match self.admission_generation.as_ref() {
|
|
187
|
+
Ok(generation) => *generation,
|
|
188
|
+
Err(error) => return ToolResponse::error(error.clone()),
|
|
189
|
+
};
|
|
190
|
+
let _admission = match admission::AdmissionGuard::acquire(paths) {
|
|
191
|
+
Ok(guard) if guard.generation() == admission_generation => guard,
|
|
192
|
+
Ok(_) => {
|
|
193
|
+
return ToolResponse::error(
|
|
194
|
+
"This FastCtx server predates the current configuration. Start a new session and retry run_background."
|
|
195
|
+
.to_string(),
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
Err(error) => return ToolResponse::error(error),
|
|
199
|
+
};
|
|
200
|
+
let limits = match store::effective_limits(paths) {
|
|
201
|
+
Ok(limits) => limits,
|
|
202
|
+
Err(error) => return ToolResponse::error(error),
|
|
203
|
+
};
|
|
204
|
+
if let Err(error) = store::reap(paths, limits.storage_limit_mib) {
|
|
205
|
+
return ToolResponse::error(error);
|
|
206
|
+
}
|
|
207
|
+
let (job_id, job_dir) = match store::reserve_job(&paths.jobs_dir) {
|
|
208
|
+
Ok(reservation) => reservation,
|
|
209
|
+
Err(error) => return ToolResponse::error(error),
|
|
210
|
+
};
|
|
211
|
+
let registry = match store::scan_registry(&paths.jobs_dir) {
|
|
212
|
+
Ok(registry) => registry,
|
|
213
|
+
Err(error) => {
|
|
214
|
+
store::remove_reserved_job(&job_dir);
|
|
215
|
+
return ToolResponse::error(error);
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
let active = registry
|
|
219
|
+
.records
|
|
220
|
+
.iter()
|
|
221
|
+
.filter(|record| record.status.is_running())
|
|
222
|
+
.count() as u64
|
|
223
|
+
+ registry.pending_reservations;
|
|
224
|
+
if active > limits.max_running_jobs {
|
|
225
|
+
store::remove_reserved_job(&job_dir);
|
|
226
|
+
return ToolResponse::error(format!(
|
|
227
|
+
"Too many running jobs: the limit is {} across all FastCtx sessions for the current user. Kill or wait out an existing job first.",
|
|
228
|
+
limits.max_running_jobs
|
|
229
|
+
));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
let log_path = job_dir.join(model::OUTPUT_LOG_FILE);
|
|
233
|
+
let terminal = format!(
|
|
234
|
+
"(Complete: job {job_id} started; log at {}.)",
|
|
235
|
+
display_path(&log_path)
|
|
236
|
+
);
|
|
237
|
+
let budget = match tool_token_budget_for_required(
|
|
238
|
+
GLOBAL_TOKEN_BUDGET_ENV,
|
|
239
|
+
estimate_tokens(&terminal),
|
|
240
|
+
) {
|
|
241
|
+
Ok(budget) => budget,
|
|
242
|
+
Err(error) => {
|
|
243
|
+
store::remove_reserved_job(&job_dir);
|
|
244
|
+
return ToolResponse::error(error);
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
if estimate_tokens(&terminal) > budget.value {
|
|
248
|
+
store::remove_reserved_job(&job_dir);
|
|
249
|
+
return ToolResponse::error(budget_too_small_message(budget));
|
|
250
|
+
}
|
|
251
|
+
let spec = LaunchSpec {
|
|
252
|
+
job_id: job_id.clone(),
|
|
253
|
+
job_dir: job_dir.clone(),
|
|
254
|
+
bash: bash.to_path_buf(),
|
|
255
|
+
command: command.to_string(),
|
|
256
|
+
cwd: cwd.to_path_buf(),
|
|
257
|
+
login_shell,
|
|
258
|
+
encoding: encoding.map(|encoding| encoding.label().to_string()),
|
|
259
|
+
environment: environment.clone(),
|
|
260
|
+
utf8_locale: utf8_locale.to_string(),
|
|
261
|
+
output_limit_bytes: limits.storage_limit_mib.saturating_mul(1024 * 1024),
|
|
262
|
+
origin: store::origin_snapshot(environment.cwd()),
|
|
263
|
+
};
|
|
264
|
+
match host::launch_supervisor(executable, &spec) {
|
|
265
|
+
Ok(()) => {
|
|
266
|
+
self.background.track_id(&job_id, SystemTime::now());
|
|
267
|
+
ToolResponse::text(terminal)
|
|
268
|
+
}
|
|
269
|
+
Err(error) => {
|
|
270
|
+
let live = store::read_json::<model::JobMeta>(
|
|
271
|
+
&job_dir.join(model::META_FILE),
|
|
272
|
+
"job metadata",
|
|
273
|
+
)
|
|
274
|
+
.ok()
|
|
275
|
+
.flatten()
|
|
276
|
+
.is_some_and(|meta| identity::identity_is_alive(&meta.supervisor));
|
|
277
|
+
if !live {
|
|
278
|
+
store::remove_reserved_job(&job_dir);
|
|
279
|
+
}
|
|
280
|
+
ToolResponse::error(error)
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
pub(crate) fn output_until_cancelled(
|
|
286
|
+
&self,
|
|
287
|
+
job_id: &str,
|
|
288
|
+
wait_ms: u64,
|
|
289
|
+
after_seq: Option<u64>,
|
|
290
|
+
encoding: Option<OutputEncoding>,
|
|
291
|
+
cancelled: impl Fn() -> bool,
|
|
292
|
+
) -> ToolResponse {
|
|
293
|
+
let paths = match self.paths() {
|
|
294
|
+
Ok(paths) => paths,
|
|
295
|
+
Err(error) => return ToolResponse::error(error),
|
|
296
|
+
};
|
|
297
|
+
let mut budget = match job_output_token_budget() {
|
|
298
|
+
Ok(budget) => budget,
|
|
299
|
+
Err(error) => return ToolResponse::error(error),
|
|
300
|
+
};
|
|
301
|
+
let started = Instant::now();
|
|
302
|
+
let wait = Duration::from_millis(wait_ms);
|
|
303
|
+
let anchor = after_seq.unwrap_or_else(|| {
|
|
304
|
+
self.cursors
|
|
305
|
+
.lock()
|
|
306
|
+
.unwrap()
|
|
307
|
+
.get(job_id)
|
|
308
|
+
.copied()
|
|
309
|
+
.unwrap_or(0)
|
|
310
|
+
});
|
|
311
|
+
let record = loop {
|
|
312
|
+
if cancelled() {
|
|
313
|
+
return ToolResponse::error(
|
|
314
|
+
"The job output wait was cancelled because the MCP request or server session ended."
|
|
315
|
+
.to_string(),
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
let record = match store::find_record(&paths.jobs_dir, job_id) {
|
|
319
|
+
Ok(Some(record)) => {
|
|
320
|
+
self.background.track_record(&record, SystemTime::now());
|
|
321
|
+
record
|
|
322
|
+
}
|
|
323
|
+
Ok(None) => {
|
|
324
|
+
self.background.remove(job_id);
|
|
325
|
+
return missing_job(job_id);
|
|
326
|
+
}
|
|
327
|
+
Err(error) => return ToolResponse::error(error),
|
|
328
|
+
};
|
|
329
|
+
let capture_failed = match store::capture_error(&record) {
|
|
330
|
+
Ok(capture_error) => capture_error.is_some(),
|
|
331
|
+
Err(error) => return ToolResponse::error(error),
|
|
332
|
+
};
|
|
333
|
+
let output_truncated = match store::output_truncation(&record) {
|
|
334
|
+
Ok(truncation) => truncation.is_some(),
|
|
335
|
+
Err(error) => return ToolResponse::error(error),
|
|
336
|
+
};
|
|
337
|
+
if !record.status.is_running()
|
|
338
|
+
|| capture_failed
|
|
339
|
+
|| output_truncated
|
|
340
|
+
|| started.elapsed() >= wait
|
|
341
|
+
{
|
|
342
|
+
break record;
|
|
343
|
+
}
|
|
344
|
+
let remaining = wait.saturating_sub(started.elapsed());
|
|
345
|
+
std::thread::sleep(remaining.min(REGISTRY_POLL));
|
|
346
|
+
};
|
|
347
|
+
let default_encoding = match record
|
|
348
|
+
.meta
|
|
349
|
+
.encoding
|
|
350
|
+
.as_deref()
|
|
351
|
+
.map(validate_output_encoding)
|
|
352
|
+
.transpose()
|
|
353
|
+
{
|
|
354
|
+
Ok(encoding) => encoding,
|
|
355
|
+
Err(error) => {
|
|
356
|
+
return ToolResponse::error(format!(
|
|
357
|
+
"Cannot read job {job_id}: its stored output encoding is invalid ({error})"
|
|
358
|
+
));
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
let page = loop {
|
|
362
|
+
let snapshot = match load_output_snapshot(&record, anchor, default_encoding, budget) {
|
|
363
|
+
Ok(snapshot) => snapshot,
|
|
364
|
+
Err(error) => {
|
|
365
|
+
if error.to_ascii_lowercase().contains("too small to return") {
|
|
366
|
+
match relax_tool_token_budget(JOB_OUTPUT_TOKEN_BUDGET_ENV) {
|
|
367
|
+
Ok(Some(expanded)) => {
|
|
368
|
+
budget = expanded;
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
Ok(None) => {}
|
|
372
|
+
Err(config_error) => return ToolResponse::error(config_error),
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return ToolResponse::error(error);
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
match format_snapshot(job_id, wait_ms, &snapshot, encoding, budget) {
|
|
379
|
+
Ok(page) => break page,
|
|
380
|
+
Err(error) => {
|
|
381
|
+
if error.to_ascii_lowercase().contains("too small to return") {
|
|
382
|
+
match relax_tool_token_budget(JOB_OUTPUT_TOKEN_BUDGET_ENV) {
|
|
383
|
+
Ok(Some(expanded)) => {
|
|
384
|
+
budget = expanded;
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
Ok(None) => {}
|
|
388
|
+
Err(config_error) => return ToolResponse::error(config_error),
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return ToolResponse::error(error);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
if let Some(cursor_seq) = page.cursor_seq {
|
|
396
|
+
let mut cursors = self.cursors.lock().unwrap();
|
|
397
|
+
let cursor = cursors.entry(job_id.to_string()).or_insert(0);
|
|
398
|
+
*cursor = (*cursor).max(cursor_seq);
|
|
399
|
+
}
|
|
400
|
+
if !record.status.is_running() {
|
|
401
|
+
self.background.remove(job_id);
|
|
402
|
+
}
|
|
403
|
+
ToolResponse::text(page.response)
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
pub(crate) fn kill(&self, job_id: &str) -> ToolResponse {
|
|
407
|
+
let paths = match self.paths() {
|
|
408
|
+
Ok(paths) => paths,
|
|
409
|
+
Err(error) => return ToolResponse::error(error),
|
|
410
|
+
};
|
|
411
|
+
let killed = format!("(Complete: job {job_id} killed.)");
|
|
412
|
+
let required = [
|
|
413
|
+
estimate_tokens(&killed),
|
|
414
|
+
estimate_tokens(&format!(
|
|
415
|
+
"(Complete: job {job_id} had already exited with code {}.)",
|
|
416
|
+
i32::MIN
|
|
417
|
+
)),
|
|
418
|
+
estimate_tokens(&format!(
|
|
419
|
+
"(Complete: job {job_id} had already been interrupted.)"
|
|
420
|
+
)),
|
|
421
|
+
]
|
|
422
|
+
.into_iter()
|
|
423
|
+
.max()
|
|
424
|
+
.unwrap_or(0);
|
|
425
|
+
let budget = match tool_token_budget_for_required(GLOBAL_TOKEN_BUDGET_ENV, required) {
|
|
426
|
+
Ok(budget) => budget,
|
|
427
|
+
Err(error) => return ToolResponse::error(error),
|
|
428
|
+
};
|
|
429
|
+
if estimate_tokens(&killed) > budget.value {
|
|
430
|
+
return ToolResponse::error(budget_too_small_message(budget));
|
|
431
|
+
}
|
|
432
|
+
let response = match terminate(paths, job_id) {
|
|
433
|
+
Ok(KillState::Killed) => ToolResponse::text(killed),
|
|
434
|
+
Ok(KillState::AlreadyExited(code)) => global_terminal(format!(
|
|
435
|
+
"(Complete: job {job_id} had already exited with code {code}.)"
|
|
436
|
+
)),
|
|
437
|
+
Ok(KillState::AlreadyInterrupted) => global_terminal(format!(
|
|
438
|
+
"(Complete: job {job_id} had already been interrupted.)"
|
|
439
|
+
)),
|
|
440
|
+
Err(error) => {
|
|
441
|
+
if matches!(store::find_record(&paths.jobs_dir, job_id), Ok(None)) {
|
|
442
|
+
self.background.remove(job_id);
|
|
443
|
+
}
|
|
444
|
+
return ToolResponse::error(error);
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
if !response.is_error {
|
|
448
|
+
self.background.remove(job_id);
|
|
449
|
+
}
|
|
450
|
+
response
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
pub(crate) fn list(
|
|
454
|
+
&self,
|
|
455
|
+
status: JobListStatus,
|
|
456
|
+
offset: u64,
|
|
457
|
+
limit: Option<u64>,
|
|
458
|
+
) -> ToolResponse {
|
|
459
|
+
let paths = match self.paths() {
|
|
460
|
+
Ok(paths) => paths,
|
|
461
|
+
Err(error) => return ToolResponse::error(error),
|
|
462
|
+
};
|
|
463
|
+
let registry = match store::scan_registry(&paths.jobs_dir) {
|
|
464
|
+
Ok(registry) => registry,
|
|
465
|
+
Err(error) => return ToolResponse::error(error),
|
|
466
|
+
};
|
|
467
|
+
let limit = match limit {
|
|
468
|
+
Some(limit) => limit,
|
|
469
|
+
None => match crate::control::settings::load(paths) {
|
|
470
|
+
Ok(settings) => settings.fastshell.job_list_limit,
|
|
471
|
+
Err(error) => return ToolResponse::error(error),
|
|
472
|
+
},
|
|
473
|
+
};
|
|
474
|
+
format_job_list(registry.records, status, offset, limit)
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
fn paths(&self) -> Result<&ControlPaths, String> {
|
|
478
|
+
self.paths.as_ref().map_err(Clone::clone)
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
pub(crate) fn background_status_at(
|
|
482
|
+
&self,
|
|
483
|
+
exclude: Option<&str>,
|
|
484
|
+
now: SystemTime,
|
|
485
|
+
) -> Option<crate::background_status::BackgroundStatus> {
|
|
486
|
+
if !self.background.has_candidates(exclude) {
|
|
487
|
+
return None;
|
|
488
|
+
}
|
|
489
|
+
let paths = self.paths().ok()?;
|
|
490
|
+
self.background.snapshot(paths, exclude, now)
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
fn terminate(paths: &ControlPaths, job_id: &str) -> Result<KillState, String> {
|
|
495
|
+
let record =
|
|
496
|
+
store::find_record(&paths.jobs_dir, job_id)?.ok_or_else(|| missing_job_text(job_id))?;
|
|
497
|
+
match &record.status {
|
|
498
|
+
JobStatus::Exited(exit) => return Ok(KillState::AlreadyExited(exit.exit_code)),
|
|
499
|
+
JobStatus::Interrupted => return Ok(KillState::AlreadyInterrupted),
|
|
500
|
+
JobStatus::Running => {}
|
|
501
|
+
}
|
|
502
|
+
store::request_kill(&record)?;
|
|
503
|
+
let deadline = Instant::now() + KILL_ACK_TIMEOUT;
|
|
504
|
+
loop {
|
|
505
|
+
let record =
|
|
506
|
+
store::find_record(&paths.jobs_dir, job_id)?.ok_or_else(|| missing_job_text(job_id))?;
|
|
507
|
+
match record.status {
|
|
508
|
+
JobStatus::Running if Instant::now() < deadline => {}
|
|
509
|
+
JobStatus::Running => {
|
|
510
|
+
return Err(format!(
|
|
511
|
+
"Cannot kill job {job_id}: its supervisor did not acknowledge within 6 seconds. Retry job_kill or stop the supervisor process manually."
|
|
512
|
+
));
|
|
513
|
+
}
|
|
514
|
+
JobStatus::Exited(exit) if exit.was_killed() => {
|
|
515
|
+
return Ok(KillState::Killed);
|
|
516
|
+
}
|
|
517
|
+
JobStatus::Exited(exit) => return Ok(KillState::AlreadyExited(exit.exit_code)),
|
|
518
|
+
JobStatus::Interrupted => return Ok(KillState::AlreadyInterrupted),
|
|
519
|
+
}
|
|
520
|
+
std::thread::sleep(REGISTRY_POLL);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
impl Default for JobManager {
|
|
525
|
+
fn default() -> Self {
|
|
526
|
+
Self::new()
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
fn format_snapshot(
|
|
531
|
+
job_id: &str,
|
|
532
|
+
wait_ms: u64,
|
|
533
|
+
snapshot: &OutputSnapshot,
|
|
534
|
+
call_encoding: Option<OutputEncoding>,
|
|
535
|
+
budget: TokenBudget,
|
|
536
|
+
) -> Result<FormattedPage, String> {
|
|
537
|
+
if snapshot.head.is_empty() && snapshot.tail.is_empty() {
|
|
538
|
+
let candidate = render_candidate(job_id, wait_ms, snapshot, call_encoding, 0, 0);
|
|
539
|
+
if estimate_tokens(&candidate.response) > budget.value {
|
|
540
|
+
return Err(budget_too_small_message(budget));
|
|
541
|
+
}
|
|
542
|
+
return Ok(FormattedPage {
|
|
543
|
+
response: candidate.response,
|
|
544
|
+
cursor_seq: (snapshot.unread_last > snapshot.anchor).then_some(snapshot.unread_last),
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
if snapshot.all_unread_loaded {
|
|
549
|
+
let candidate = render_candidate(
|
|
550
|
+
job_id,
|
|
551
|
+
wait_ms,
|
|
552
|
+
snapshot,
|
|
553
|
+
call_encoding,
|
|
554
|
+
snapshot.head.len(),
|
|
555
|
+
0,
|
|
556
|
+
);
|
|
557
|
+
if estimate_tokens(&candidate.response) <= budget.value {
|
|
558
|
+
return Ok(FormattedPage {
|
|
559
|
+
response: candidate.response,
|
|
560
|
+
cursor_seq: snapshot
|
|
561
|
+
.direct_log
|
|
562
|
+
.as_ref()
|
|
563
|
+
.map(|_| snapshot.unread_last)
|
|
564
|
+
.or(candidate.last_seq),
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if snapshot.direct_log.is_none() {
|
|
570
|
+
return format_legacy_page(job_id, wait_ms, snapshot, call_encoding, budget);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
format_direct_window(job_id, wait_ms, snapshot, call_encoding, budget)
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
#[derive(Debug)]
|
|
577
|
+
struct RenderedCandidate {
|
|
578
|
+
response: String,
|
|
579
|
+
last_seq: Option<u64>,
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
fn load_output_snapshot(
|
|
583
|
+
record: &JobRecord,
|
|
584
|
+
anchor: u64,
|
|
585
|
+
default_encoding: Option<OutputEncoding>,
|
|
586
|
+
budget: TokenBudget,
|
|
587
|
+
) -> Result<OutputSnapshot, String> {
|
|
588
|
+
let mut log = store::open_log(record)?;
|
|
589
|
+
let direct_log = log.direct_path().map(Path::to_path_buf);
|
|
590
|
+
let total_lines = log.total_lines();
|
|
591
|
+
let requested_first = anchor.saturating_add(1);
|
|
592
|
+
let unread_first = requested_first.max(log.oldest_seq());
|
|
593
|
+
let max_lines = budget.value.saturating_mul(4).saturating_add(64);
|
|
594
|
+
let max_bytes = budget.value.saturating_mul(16).saturating_add(64 * 1024);
|
|
595
|
+
let mut head = Vec::new();
|
|
596
|
+
let mut tail = Vec::new();
|
|
597
|
+
let mut all_unread_loaded = true;
|
|
598
|
+
if unread_first <= total_lines {
|
|
599
|
+
let prefix = log.read_prefix_bounded(unread_first, total_lines, max_lines, max_bytes)?;
|
|
600
|
+
all_unread_loaded = prefix.complete;
|
|
601
|
+
head = prefix.lines;
|
|
602
|
+
if !all_unread_loaded && direct_log.is_some() {
|
|
603
|
+
if anchor != 0 {
|
|
604
|
+
head.clear();
|
|
605
|
+
}
|
|
606
|
+
let suffix =
|
|
607
|
+
log.read_suffix_bounded(unread_first, total_lines, max_lines, max_bytes)?;
|
|
608
|
+
tail = suffix.lines;
|
|
609
|
+
if let Some(last_head) = head.last().map(|line| line.seq) {
|
|
610
|
+
tail.retain(|line| line.seq > last_head);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
let legacy_loss = log.had_irretrievable_loss() || unread_first > requested_first;
|
|
615
|
+
Ok(OutputSnapshot {
|
|
616
|
+
status: record.status.clone(),
|
|
617
|
+
head,
|
|
618
|
+
tail,
|
|
619
|
+
unread_first,
|
|
620
|
+
unread_last: total_lines,
|
|
621
|
+
all_unread_loaded,
|
|
622
|
+
total_lines,
|
|
623
|
+
legacy_loss,
|
|
624
|
+
capture_error: log.capture_error.clone(),
|
|
625
|
+
output_truncation: log.output_truncation.clone(),
|
|
626
|
+
default_encoding,
|
|
627
|
+
anchor,
|
|
628
|
+
direct_log,
|
|
629
|
+
})
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
fn format_legacy_page(
|
|
633
|
+
job_id: &str,
|
|
634
|
+
wait_ms: u64,
|
|
635
|
+
snapshot: &OutputSnapshot,
|
|
636
|
+
call_encoding: Option<OutputEncoding>,
|
|
637
|
+
budget: TokenBudget,
|
|
638
|
+
) -> Result<FormattedPage, String> {
|
|
639
|
+
let mut low = 1_usize;
|
|
640
|
+
let mut high = snapshot.head.len();
|
|
641
|
+
let mut best = None;
|
|
642
|
+
while low <= high {
|
|
643
|
+
let shown = low + (high - low) / 2;
|
|
644
|
+
let candidate = render_candidate(job_id, wait_ms, snapshot, call_encoding, shown, 0);
|
|
645
|
+
if estimate_tokens(&candidate.response) <= budget.value {
|
|
646
|
+
best = Some(candidate);
|
|
647
|
+
low = shown.saturating_add(1);
|
|
648
|
+
} else if shown == 1 {
|
|
649
|
+
break;
|
|
650
|
+
} else {
|
|
651
|
+
high = shown - 1;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
let candidate = best.ok_or_else(|| budget_too_small_message(budget))?;
|
|
655
|
+
Ok(FormattedPage {
|
|
656
|
+
response: candidate.response,
|
|
657
|
+
cursor_seq: candidate.last_seq,
|
|
658
|
+
})
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
fn format_direct_window(
|
|
662
|
+
job_id: &str,
|
|
663
|
+
wait_ms: u64,
|
|
664
|
+
snapshot: &OutputSnapshot,
|
|
665
|
+
call_encoding: Option<OutputEncoding>,
|
|
666
|
+
budget: TokenBudget,
|
|
667
|
+
) -> Result<FormattedPage, String> {
|
|
668
|
+
let tail_available = if snapshot.all_unread_loaded {
|
|
669
|
+
snapshot.head.len()
|
|
670
|
+
} else {
|
|
671
|
+
snapshot.tail.len()
|
|
672
|
+
};
|
|
673
|
+
if tail_available == 0 {
|
|
674
|
+
return Err(budget_too_small_message(budget));
|
|
675
|
+
}
|
|
676
|
+
let head_available = if snapshot.anchor == 0 {
|
|
677
|
+
if snapshot.all_unread_loaded {
|
|
678
|
+
snapshot.head.len().saturating_sub(1)
|
|
679
|
+
} else {
|
|
680
|
+
snapshot.head.len()
|
|
681
|
+
}
|
|
682
|
+
} else {
|
|
683
|
+
0
|
|
684
|
+
};
|
|
685
|
+
let preferred_head = preferred_head_count(
|
|
686
|
+
snapshot,
|
|
687
|
+
call_encoding,
|
|
688
|
+
head_available,
|
|
689
|
+
budget.value.saturating_div(10).max(1),
|
|
690
|
+
);
|
|
691
|
+
let mut low = 0_usize;
|
|
692
|
+
let mut high = preferred_head;
|
|
693
|
+
let mut head_that_fits = None;
|
|
694
|
+
while low <= high {
|
|
695
|
+
let head = low + (high - low) / 2;
|
|
696
|
+
let candidate = render_candidate(job_id, wait_ms, snapshot, call_encoding, head, 1);
|
|
697
|
+
if estimate_tokens(&candidate.response) <= budget.value {
|
|
698
|
+
head_that_fits = Some(head);
|
|
699
|
+
low = head.saturating_add(1);
|
|
700
|
+
} else if head == 0 {
|
|
701
|
+
break;
|
|
702
|
+
} else {
|
|
703
|
+
high = head - 1;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
let head = head_that_fits.ok_or_else(|| budget_too_small_message(budget))?;
|
|
707
|
+
let tail_limit = if snapshot.all_unread_loaded {
|
|
708
|
+
tail_available.saturating_sub(head)
|
|
709
|
+
} else {
|
|
710
|
+
tail_available
|
|
711
|
+
};
|
|
712
|
+
let mut low = 1_usize;
|
|
713
|
+
let mut high = tail_limit;
|
|
714
|
+
let mut best = None;
|
|
715
|
+
while low <= high {
|
|
716
|
+
let tail = low + (high - low) / 2;
|
|
717
|
+
let candidate = render_candidate(job_id, wait_ms, snapshot, call_encoding, head, tail);
|
|
718
|
+
if estimate_tokens(&candidate.response) <= budget.value {
|
|
719
|
+
best = Some(candidate);
|
|
720
|
+
low = tail.saturating_add(1);
|
|
721
|
+
} else if tail == 1 {
|
|
722
|
+
break;
|
|
723
|
+
} else {
|
|
724
|
+
high = tail - 1;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
let candidate = best.ok_or_else(|| budget_too_small_message(budget))?;
|
|
728
|
+
Ok(FormattedPage {
|
|
729
|
+
response: candidate.response,
|
|
730
|
+
cursor_seq: Some(snapshot.unread_last),
|
|
731
|
+
})
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
fn preferred_head_count(
|
|
735
|
+
snapshot: &OutputSnapshot,
|
|
736
|
+
call_encoding: Option<OutputEncoding>,
|
|
737
|
+
available: usize,
|
|
738
|
+
token_target: usize,
|
|
739
|
+
) -> usize {
|
|
740
|
+
let mut low = 0_usize;
|
|
741
|
+
let mut high = available;
|
|
742
|
+
let mut best = 0_usize;
|
|
743
|
+
while low <= high {
|
|
744
|
+
let count = low + (high - low) / 2;
|
|
745
|
+
let selected = select_lines(snapshot, count, 0);
|
|
746
|
+
let encoded = selected
|
|
747
|
+
.iter()
|
|
748
|
+
.map(|line| line.encoded_line())
|
|
749
|
+
.collect::<Vec<_>>();
|
|
750
|
+
let decoded = decode_job(&encoded, call_encoding, snapshot.default_encoding);
|
|
751
|
+
if estimate_tokens(&decoded.lines.join("\n")) <= token_target {
|
|
752
|
+
best = count;
|
|
753
|
+
low = count.saturating_add(1);
|
|
754
|
+
} else if count == 0 {
|
|
755
|
+
break;
|
|
756
|
+
} else {
|
|
757
|
+
high = count - 1;
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
best
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
fn render_candidate(
|
|
764
|
+
job_id: &str,
|
|
765
|
+
wait_ms: u64,
|
|
766
|
+
snapshot: &OutputSnapshot,
|
|
767
|
+
call_encoding: Option<OutputEncoding>,
|
|
768
|
+
head_count: usize,
|
|
769
|
+
tail_count: usize,
|
|
770
|
+
) -> RenderedCandidate {
|
|
771
|
+
let selected = select_lines(snapshot, head_count, tail_count);
|
|
772
|
+
let encoded = selected
|
|
773
|
+
.iter()
|
|
774
|
+
.map(|line| line.encoded_line())
|
|
775
|
+
.collect::<Vec<_>>();
|
|
776
|
+
let decoded = decode_job(&encoded, call_encoding, snapshot.default_encoding);
|
|
777
|
+
let mut notes = Vec::new();
|
|
778
|
+
if let Some(path) = snapshot.direct_log.as_ref() {
|
|
779
|
+
for (first, last) in omitted_ranges(snapshot, &selected) {
|
|
780
|
+
notes.push(omission_note(first, last, path));
|
|
781
|
+
}
|
|
782
|
+
} else if snapshot.legacy_loss {
|
|
783
|
+
notes.push(legacy_loss_note(snapshot));
|
|
784
|
+
}
|
|
785
|
+
if let Some(error) = &snapshot.capture_error {
|
|
786
|
+
notes.push(capture_failure_note(error, snapshot.direct_log.as_deref()));
|
|
787
|
+
}
|
|
788
|
+
if let Some(truncation) = &snapshot.output_truncation {
|
|
789
|
+
notes.push(output_truncation_note(
|
|
790
|
+
truncation,
|
|
791
|
+
snapshot.direct_log.as_deref(),
|
|
792
|
+
));
|
|
793
|
+
}
|
|
794
|
+
if let Some(note) = job_garble_note(decoded.invalid_sequences, snapshot.anchor) {
|
|
795
|
+
notes.push(note);
|
|
796
|
+
}
|
|
797
|
+
if let Some(path) = snapshot.direct_log.as_ref() {
|
|
798
|
+
for (line, truncated) in selected.iter().zip(&decoded.truncated_per_line) {
|
|
799
|
+
if *truncated {
|
|
800
|
+
notes.push(format!(
|
|
801
|
+
"(Note: line {} was truncated at 2000 chars in this response; read the complete line at {} with offset={}, or inspect a fragment with grep or the inspect_local_file tool's hex view.)",
|
|
802
|
+
line.seq,
|
|
803
|
+
display_path(path),
|
|
804
|
+
line.seq
|
|
805
|
+
));
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
let leading = (!notes.is_empty()).then(|| notes.join("\n\n"));
|
|
810
|
+
let last_seq = selected.last().map(|line| line.seq);
|
|
811
|
+
let terminal = output_terminal(job_id, wait_ms, snapshot, selected.len(), last_seq);
|
|
812
|
+
RenderedCandidate {
|
|
813
|
+
response: compose_response_with_tail(
|
|
814
|
+
leading.as_deref(),
|
|
815
|
+
&decoded.lines,
|
|
816
|
+
decoded.transcoding_note.as_deref(),
|
|
817
|
+
&terminal,
|
|
818
|
+
),
|
|
819
|
+
last_seq,
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
fn select_lines(
|
|
824
|
+
snapshot: &OutputSnapshot,
|
|
825
|
+
head_count: usize,
|
|
826
|
+
tail_count: usize,
|
|
827
|
+
) -> Vec<&StoredLine> {
|
|
828
|
+
let mut selected = Vec::new();
|
|
829
|
+
if snapshot.all_unread_loaded {
|
|
830
|
+
let head = head_count.min(snapshot.head.len());
|
|
831
|
+
selected.extend(snapshot.head.iter().take(head));
|
|
832
|
+
let tail = tail_count.min(snapshot.head.len().saturating_sub(head));
|
|
833
|
+
if tail > 0 {
|
|
834
|
+
selected.extend(snapshot.head[snapshot.head.len() - tail..].iter());
|
|
835
|
+
}
|
|
836
|
+
return selected;
|
|
837
|
+
}
|
|
838
|
+
selected.extend(snapshot.head.iter().take(head_count));
|
|
839
|
+
let tail = tail_count.min(snapshot.tail.len());
|
|
840
|
+
if tail > 0 {
|
|
841
|
+
let last_head = selected.last().map(|line| line.seq).unwrap_or(0);
|
|
842
|
+
selected.extend(
|
|
843
|
+
snapshot.tail[snapshot.tail.len() - tail..]
|
|
844
|
+
.iter()
|
|
845
|
+
.filter(|line| line.seq > last_head),
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
selected
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
fn omitted_ranges(snapshot: &OutputSnapshot, selected: &[&StoredLine]) -> Vec<(u64, u64)> {
|
|
852
|
+
if snapshot.unread_first > snapshot.unread_last {
|
|
853
|
+
return Vec::new();
|
|
854
|
+
}
|
|
855
|
+
let mut ranges = Vec::new();
|
|
856
|
+
let mut next = snapshot.unread_first;
|
|
857
|
+
for line in selected {
|
|
858
|
+
if line.seq > next {
|
|
859
|
+
ranges.push((next, line.seq - 1));
|
|
860
|
+
}
|
|
861
|
+
next = line.seq.saturating_add(1);
|
|
862
|
+
}
|
|
863
|
+
if next <= snapshot.unread_last {
|
|
864
|
+
ranges.push((next, snapshot.unread_last));
|
|
865
|
+
}
|
|
866
|
+
ranges
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
fn omission_note(first: u64, last: u64, path: &Path) -> String {
|
|
870
|
+
if first == last {
|
|
871
|
+
format!(
|
|
872
|
+
"(Note: line {first} was omitted from this response; read it at {} with offset={first}.)",
|
|
873
|
+
display_path(path)
|
|
874
|
+
)
|
|
875
|
+
} else {
|
|
876
|
+
format!(
|
|
877
|
+
"(Note: lines {first}-{last} were omitted from this response; read them at {} with offset={first}.)",
|
|
878
|
+
display_path(path)
|
|
879
|
+
)
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
fn legacy_loss_note(snapshot: &OutputSnapshot) -> String {
|
|
884
|
+
let expected = snapshot.anchor.saturating_add(1);
|
|
885
|
+
let missing = snapshot.unread_first.saturating_sub(expected);
|
|
886
|
+
if missing > 0 {
|
|
887
|
+
format!(
|
|
888
|
+
"(Note: {missing} earlier {} {} dropped from this legacy job record and cannot be retrieved.)",
|
|
889
|
+
plural(missing, "line", "lines"),
|
|
890
|
+
if missing == 1 { "was" } else { "were" }
|
|
891
|
+
)
|
|
892
|
+
} else {
|
|
893
|
+
"(Note: this legacy job record lost or truncated output that cannot be retrieved.)"
|
|
894
|
+
.to_string()
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
fn capture_failure_note(error: &model::CaptureErrorRecord, direct_log: Option<&Path>) -> String {
|
|
899
|
+
match direct_log {
|
|
900
|
+
Some(path) => format!(
|
|
901
|
+
"(Note: output capture failed after seq {}: {}. This does not kill the process; its exit status remains available, but the log at {} stops here.)",
|
|
902
|
+
error.after_seq,
|
|
903
|
+
error.reason,
|
|
904
|
+
display_path(path)
|
|
905
|
+
),
|
|
906
|
+
None => format!(
|
|
907
|
+
"(Note: output capture failed after seq {}: {}. This did not kill the process; its exit status remains available, but this legacy record stops here.)",
|
|
908
|
+
error.after_seq, error.reason
|
|
909
|
+
),
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
fn output_truncation_note(
|
|
914
|
+
truncation: &model::OutputTruncationRecord,
|
|
915
|
+
direct_log: Option<&Path>,
|
|
916
|
+
) -> String {
|
|
917
|
+
match direct_log {
|
|
918
|
+
Some(path) => format!(
|
|
919
|
+
"(Note: this job reached its {}-byte combined output.log + output.idx hard limit after seq {}. The supervisor kept draining output and did not stop the command, but later output was not persisted. The preserved prefix is at {}.)",
|
|
920
|
+
truncation.limit_bytes,
|
|
921
|
+
truncation.after_seq,
|
|
922
|
+
display_path(path)
|
|
923
|
+
),
|
|
924
|
+
None => format!(
|
|
925
|
+
"(Note: this job reached its {}-byte output hard limit after seq {}. The supervisor kept draining output and did not stop the command, but later output was not persisted.)",
|
|
926
|
+
truncation.limit_bytes, truncation.after_seq
|
|
927
|
+
),
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
fn output_terminal(
|
|
932
|
+
job_id: &str,
|
|
933
|
+
wait_ms: u64,
|
|
934
|
+
snapshot: &OutputSnapshot,
|
|
935
|
+
shown: usize,
|
|
936
|
+
last_seq: Option<u64>,
|
|
937
|
+
) -> String {
|
|
938
|
+
if let JobStatus::Running = snapshot.status {
|
|
939
|
+
if shown > 0 {
|
|
940
|
+
return format!(
|
|
941
|
+
"(Partial: job {job_id} is running; {shown} new {} shown. Call job_output again for more, or move on and check back.)",
|
|
942
|
+
plural(shown as u64, "line", "lines")
|
|
943
|
+
);
|
|
944
|
+
}
|
|
945
|
+
if wait_ms < crate::shell::MAX_BLOCKING_CALL_MS {
|
|
946
|
+
return format!(
|
|
947
|
+
"(Partial: job {job_id} is running; no new output within {wait_ms} ms. Move on and check back, or raise wait_ms if you have nothing else to do.)"
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
return format!(
|
|
951
|
+
"(Partial: job {job_id} is running; no new output within {wait_ms} ms. It may stay quiet for a long time, or never exit — move on and check back.)"
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
if let Some(path) = snapshot.direct_log.as_ref() {
|
|
955
|
+
return match &snapshot.status {
|
|
956
|
+
JobStatus::Exited(exit) if exit.was_killed() => format!(
|
|
957
|
+
"(Complete: job {job_id} was killed; {} {} total. Full log: {})",
|
|
958
|
+
snapshot.total_lines,
|
|
959
|
+
plural(snapshot.total_lines, "line", "lines"),
|
|
960
|
+
display_path(path)
|
|
961
|
+
),
|
|
962
|
+
JobStatus::Exited(exit) => format!(
|
|
963
|
+
"(Complete: job {job_id} exited {}; {} {} total. Full log: {})",
|
|
964
|
+
exit.exit_code,
|
|
965
|
+
snapshot.total_lines,
|
|
966
|
+
plural(snapshot.total_lines, "line", "lines"),
|
|
967
|
+
display_path(path)
|
|
968
|
+
),
|
|
969
|
+
JobStatus::Interrupted => format!(
|
|
970
|
+
"(Complete: job {job_id} was interrupted: its process ended without an exit record (machine restart or external kill); {} {} preserved. Full log: {})",
|
|
971
|
+
snapshot.total_lines,
|
|
972
|
+
plural(snapshot.total_lines, "line", "lines"),
|
|
973
|
+
display_path(path)
|
|
974
|
+
),
|
|
975
|
+
JobStatus::Running => unreachable!(),
|
|
976
|
+
};
|
|
977
|
+
}
|
|
978
|
+
let next = last_seq.unwrap_or(snapshot.anchor);
|
|
979
|
+
let more = next < snapshot.unread_last
|
|
980
|
+
&& (!snapshot.all_unread_loaded
|
|
981
|
+
|| snapshot.head.last().is_some_and(|line| line.seq > next));
|
|
982
|
+
if more {
|
|
983
|
+
return match &snapshot.status {
|
|
984
|
+
JobStatus::Exited(exit) if exit.was_killed() => format!(
|
|
985
|
+
"(Partial: job {job_id} was killed; more legacy output remains. Call job_output again with after_seq={next}.)"
|
|
986
|
+
),
|
|
987
|
+
JobStatus::Exited(exit) => format!(
|
|
988
|
+
"(Partial: job {job_id} exited {}; more legacy output remains. Call job_output again with after_seq={next}.)",
|
|
989
|
+
exit.exit_code
|
|
990
|
+
),
|
|
991
|
+
JobStatus::Interrupted => format!(
|
|
992
|
+
"(Partial: job {job_id} was interrupted; more legacy output remains. Call job_output again with after_seq={next}.)"
|
|
993
|
+
),
|
|
994
|
+
JobStatus::Running => unreachable!(),
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
let loss = if snapshot.legacy_loss {
|
|
998
|
+
", but this legacy record lost or truncated output that cannot be retrieved"
|
|
999
|
+
} else {
|
|
1000
|
+
""
|
|
1001
|
+
};
|
|
1002
|
+
match &snapshot.status {
|
|
1003
|
+
JobStatus::Exited(exit) if exit.was_killed() => format!(
|
|
1004
|
+
"(Complete: job {job_id} was killed; {} {} total{loss}.)",
|
|
1005
|
+
snapshot.total_lines,
|
|
1006
|
+
plural(snapshot.total_lines, "line", "lines")
|
|
1007
|
+
),
|
|
1008
|
+
JobStatus::Exited(exit) => format!(
|
|
1009
|
+
"(Complete: job {job_id} exited {}; {} {} total{loss}.)",
|
|
1010
|
+
exit.exit_code,
|
|
1011
|
+
snapshot.total_lines,
|
|
1012
|
+
plural(snapshot.total_lines, "line", "lines")
|
|
1013
|
+
),
|
|
1014
|
+
JobStatus::Interrupted => format!(
|
|
1015
|
+
"(Complete: job {job_id} was interrupted: its process ended without an exit record (machine restart or external kill); {} {} preserved{loss}.)",
|
|
1016
|
+
snapshot.total_lines,
|
|
1017
|
+
plural(snapshot.total_lines, "line", "lines")
|
|
1018
|
+
),
|
|
1019
|
+
JobStatus::Running => unreachable!(),
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
fn format_job_list(
|
|
1024
|
+
records: Vec<JobRecord>,
|
|
1025
|
+
status: JobListStatus,
|
|
1026
|
+
offset: u64,
|
|
1027
|
+
limit: u64,
|
|
1028
|
+
) -> ToolResponse {
|
|
1029
|
+
let mut budget = match global_token_budget() {
|
|
1030
|
+
Ok(budget) => budget,
|
|
1031
|
+
Err(error) => return ToolResponse::error(error),
|
|
1032
|
+
};
|
|
1033
|
+
loop {
|
|
1034
|
+
let response = format_job_list_with_budget(records.clone(), status, offset, limit, budget);
|
|
1035
|
+
let starved = response.is_error
|
|
1036
|
+
&& response.content.iter().any(|content| {
|
|
1037
|
+
matches!(content, crate::ToolContent::Text(text) if text.to_ascii_lowercase().contains("too small to return"))
|
|
1038
|
+
});
|
|
1039
|
+
if starved {
|
|
1040
|
+
match relax_tool_token_budget(GLOBAL_TOKEN_BUDGET_ENV) {
|
|
1041
|
+
Ok(Some(expanded)) => {
|
|
1042
|
+
budget = expanded;
|
|
1043
|
+
continue;
|
|
1044
|
+
}
|
|
1045
|
+
Ok(None) => {}
|
|
1046
|
+
Err(error) => return ToolResponse::error(error),
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
return response;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
fn format_job_list_with_budget(
|
|
1054
|
+
mut records: Vec<JobRecord>,
|
|
1055
|
+
status: JobListStatus,
|
|
1056
|
+
offset: u64,
|
|
1057
|
+
limit: u64,
|
|
1058
|
+
budget: TokenBudget,
|
|
1059
|
+
) -> ToolResponse {
|
|
1060
|
+
records.retain(|record| match status {
|
|
1061
|
+
JobListStatus::Running => record.status.is_running(),
|
|
1062
|
+
JobListStatus::Finished => !record.status.is_running(),
|
|
1063
|
+
JobListStatus::All => true,
|
|
1064
|
+
});
|
|
1065
|
+
if records.is_empty() {
|
|
1066
|
+
return terminal_response(empty_job_list_terminal(status), budget);
|
|
1067
|
+
}
|
|
1068
|
+
records.sort_by(
|
|
1069
|
+
|left, right| match (left.status.is_running(), right.status.is_running()) {
|
|
1070
|
+
(true, false) => std::cmp::Ordering::Less,
|
|
1071
|
+
(false, true) => std::cmp::Ordering::Greater,
|
|
1072
|
+
(true, true) => store::started_sort_key(right).cmp(&store::started_sort_key(left)),
|
|
1073
|
+
(false, false) => right
|
|
1074
|
+
.ended_sort_key
|
|
1075
|
+
.cmp(&left.ended_sort_key)
|
|
1076
|
+
.then_with(|| right.id.cmp(&left.id)),
|
|
1077
|
+
},
|
|
1078
|
+
);
|
|
1079
|
+
let running = records
|
|
1080
|
+
.iter()
|
|
1081
|
+
.filter(|record| record.status.is_running())
|
|
1082
|
+
.count() as u64;
|
|
1083
|
+
let finished = records.len() as u64 - running;
|
|
1084
|
+
let total = records.len();
|
|
1085
|
+
let start = usize::try_from(offset).unwrap_or(usize::MAX).min(total);
|
|
1086
|
+
if start == total {
|
|
1087
|
+
return terminal_response(
|
|
1088
|
+
format!(
|
|
1089
|
+
"(Complete: no {} at offset={offset}; {total} available.)",
|
|
1090
|
+
job_list_scope(status)
|
|
1091
|
+
),
|
|
1092
|
+
budget,
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
let page_end = start
|
|
1096
|
+
.saturating_add(usize::try_from(limit).unwrap_or(usize::MAX))
|
|
1097
|
+
.min(total);
|
|
1098
|
+
let entries = records[start..page_end]
|
|
1099
|
+
.iter()
|
|
1100
|
+
.map(format_job_entry)
|
|
1101
|
+
.collect::<Vec<_>>();
|
|
1102
|
+
let complete_terminal = complete_job_list_terminal(status, running, finished);
|
|
1103
|
+
let terminal = if page_end < total {
|
|
1104
|
+
partial_job_list_terminal(status, start, entries.len(), total, limit)
|
|
1105
|
+
} else {
|
|
1106
|
+
complete_terminal.clone()
|
|
1107
|
+
};
|
|
1108
|
+
let complete = compose_list(&entries, &terminal);
|
|
1109
|
+
if estimate_tokens(&complete) <= budget.value {
|
|
1110
|
+
return ToolResponse::text(complete);
|
|
1111
|
+
}
|
|
1112
|
+
let mut low = 1_usize;
|
|
1113
|
+
let mut high = entries.len();
|
|
1114
|
+
let mut best = None;
|
|
1115
|
+
while low <= high {
|
|
1116
|
+
let shown = low + (high - low) / 2;
|
|
1117
|
+
let terminal = partial_job_list_terminal(status, start, shown, total, limit);
|
|
1118
|
+
let response = compose_list(&entries[..shown], &terminal);
|
|
1119
|
+
if estimate_tokens(&response) <= budget.value {
|
|
1120
|
+
best = Some(response);
|
|
1121
|
+
low = shown.saturating_add(1);
|
|
1122
|
+
} else if shown == 1 {
|
|
1123
|
+
break;
|
|
1124
|
+
} else {
|
|
1125
|
+
high = shown - 1;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
best.map_or_else(
|
|
1129
|
+
|| ToolResponse::error(budget_too_small_message(budget)),
|
|
1130
|
+
ToolResponse::text,
|
|
1131
|
+
)
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
fn empty_job_list_terminal(status: JobListStatus) -> String {
|
|
1135
|
+
match status {
|
|
1136
|
+
JobListStatus::Running => "(Complete: no running jobs.)",
|
|
1137
|
+
JobListStatus::Finished => "(Complete: no finished records.)",
|
|
1138
|
+
JobListStatus::All => "(Complete: no jobs.)",
|
|
1139
|
+
}
|
|
1140
|
+
.to_string()
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
fn complete_job_list_terminal(status: JobListStatus, running: u64, finished: u64) -> String {
|
|
1144
|
+
match status {
|
|
1145
|
+
JobListStatus::Running => format!(
|
|
1146
|
+
"(Complete: {running} running {}.)",
|
|
1147
|
+
plural(running, "job", "jobs")
|
|
1148
|
+
),
|
|
1149
|
+
JobListStatus::Finished => format!(
|
|
1150
|
+
"(Complete: {finished} finished {}.)",
|
|
1151
|
+
plural(finished, "record", "records")
|
|
1152
|
+
),
|
|
1153
|
+
JobListStatus::All => format!(
|
|
1154
|
+
"(Complete: {running} running {}, {finished} finished {}.)",
|
|
1155
|
+
plural(running, "job", "jobs"),
|
|
1156
|
+
plural(finished, "record", "records")
|
|
1157
|
+
),
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
fn partial_job_list_terminal(
|
|
1162
|
+
status: JobListStatus,
|
|
1163
|
+
start: usize,
|
|
1164
|
+
shown: usize,
|
|
1165
|
+
total: usize,
|
|
1166
|
+
limit: u64,
|
|
1167
|
+
) -> String {
|
|
1168
|
+
let first = start.saturating_add(1);
|
|
1169
|
+
let next = start.saturating_add(shown);
|
|
1170
|
+
format!(
|
|
1171
|
+
"(Partial: showing {first}-{next} of {total} {}. Call job_list again with status=\"{}\", limit={limit}, offset={next}.)",
|
|
1172
|
+
job_list_scope(status),
|
|
1173
|
+
job_list_status_name(status)
|
|
1174
|
+
)
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
fn job_list_scope(status: JobListStatus) -> &'static str {
|
|
1178
|
+
match status {
|
|
1179
|
+
JobListStatus::Running => "running jobs",
|
|
1180
|
+
JobListStatus::Finished => "finished records",
|
|
1181
|
+
JobListStatus::All => "jobs",
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
fn job_list_status_name(status: JobListStatus) -> &'static str {
|
|
1186
|
+
match status {
|
|
1187
|
+
JobListStatus::Running => "running",
|
|
1188
|
+
JobListStatus::Finished => "finished",
|
|
1189
|
+
JobListStatus::All => "all",
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
fn format_job_entry(record: &JobRecord) -> String {
|
|
1194
|
+
let status = match &record.status {
|
|
1195
|
+
JobStatus::Running => "running".to_string(),
|
|
1196
|
+
JobStatus::Exited(exit) if exit.was_killed() => "killed".to_string(),
|
|
1197
|
+
JobStatus::Exited(exit) => format!("exited {}", exit.exit_code),
|
|
1198
|
+
JobStatus::Interrupted => "interrupted".to_string(),
|
|
1199
|
+
};
|
|
1200
|
+
format!(
|
|
1201
|
+
"{} {status}; started {}\n {} — {}",
|
|
1202
|
+
record.id,
|
|
1203
|
+
record.meta.started_at,
|
|
1204
|
+
single_line(&record.meta.cwd),
|
|
1205
|
+
truncate_command(&record.meta.command)
|
|
1206
|
+
)
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
fn truncate_command(command: &str) -> String {
|
|
1210
|
+
let command = single_line(command);
|
|
1211
|
+
let mut characters = command.chars();
|
|
1212
|
+
let prefix = characters.by_ref().take(120).collect::<String>();
|
|
1213
|
+
if characters.next().is_some() {
|
|
1214
|
+
format!("{prefix}…")
|
|
1215
|
+
} else {
|
|
1216
|
+
prefix
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
fn single_line(value: &str) -> String {
|
|
1221
|
+
value
|
|
1222
|
+
.chars()
|
|
1223
|
+
.flat_map(|character| {
|
|
1224
|
+
if character.is_control() {
|
|
1225
|
+
character.escape_default().collect::<Vec<_>>()
|
|
1226
|
+
} else {
|
|
1227
|
+
vec![character]
|
|
1228
|
+
}
|
|
1229
|
+
})
|
|
1230
|
+
.collect()
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
fn compose_list(entries: &[String], terminal: &str) -> String {
|
|
1234
|
+
if entries.is_empty() {
|
|
1235
|
+
terminal.to_string()
|
|
1236
|
+
} else {
|
|
1237
|
+
format!("{}\n\n{terminal}", entries.join("\n\n"))
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
fn missing_job(job_id: &str) -> ToolResponse {
|
|
1242
|
+
ToolResponse::error(missing_job_text(job_id))
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
fn missing_job_text(job_id: &str) -> String {
|
|
1246
|
+
format!(
|
|
1247
|
+
"No such job: \"{job_id}\". It may never have existed, or its finished record was evicted by the job storage limit. List known jobs with job_list."
|
|
1248
|
+
)
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
fn global_terminal(terminal: String) -> ToolResponse {
|
|
1252
|
+
match global_token_budget() {
|
|
1253
|
+
Ok(budget) => terminal_response(terminal, budget),
|
|
1254
|
+
Err(error) => ToolResponse::error(error),
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
pub(crate) fn summaries(paths: &ControlPaths) -> Result<Vec<JobSummary>, JobRegistryError> {
|
|
1259
|
+
let mut records = store::scan_registry(&paths.jobs_dir)?.records;
|
|
1260
|
+
records.sort_by(
|
|
1261
|
+
|left, right| match (left.status.is_running(), right.status.is_running()) {
|
|
1262
|
+
(true, false) => std::cmp::Ordering::Less,
|
|
1263
|
+
(false, true) => std::cmp::Ordering::Greater,
|
|
1264
|
+
(true, true) => store::started_sort_key(right).cmp(&store::started_sort_key(left)),
|
|
1265
|
+
(false, false) => right
|
|
1266
|
+
.ended_sort_key
|
|
1267
|
+
.cmp(&left.ended_sort_key)
|
|
1268
|
+
.then_with(|| right.id.cmp(&left.id)),
|
|
1269
|
+
},
|
|
1270
|
+
);
|
|
1271
|
+
Ok(records
|
|
1272
|
+
.into_iter()
|
|
1273
|
+
.map(|record| {
|
|
1274
|
+
let source_key = format!(
|
|
1275
|
+
"{}:{}:{}",
|
|
1276
|
+
record.meta.origin.server_pid,
|
|
1277
|
+
record
|
|
1278
|
+
.meta
|
|
1279
|
+
.origin
|
|
1280
|
+
.server_started
|
|
1281
|
+
.as_deref()
|
|
1282
|
+
.unwrap_or("legacy"),
|
|
1283
|
+
record.meta.origin.server_cwd
|
|
1284
|
+
);
|
|
1285
|
+
let source = JobSourceSummary {
|
|
1286
|
+
tag: source_tag(&source_key),
|
|
1287
|
+
key: source_key,
|
|
1288
|
+
server_pid: record.meta.origin.server_pid,
|
|
1289
|
+
parent_executable: record.meta.origin.parent_executable,
|
|
1290
|
+
server_cwd: record.meta.origin.server_cwd,
|
|
1291
|
+
};
|
|
1292
|
+
JobSummary {
|
|
1293
|
+
id: record.id,
|
|
1294
|
+
command: record.meta.command,
|
|
1295
|
+
cwd: record.meta.cwd,
|
|
1296
|
+
started_at: record.meta.started_at,
|
|
1297
|
+
status: match record.status {
|
|
1298
|
+
JobStatus::Running => JobSummaryStatus::Running,
|
|
1299
|
+
JobStatus::Exited(exit) => JobSummaryStatus::Exited(exit.exit_code),
|
|
1300
|
+
JobStatus::Interrupted => JobSummaryStatus::Interrupted,
|
|
1301
|
+
},
|
|
1302
|
+
source,
|
|
1303
|
+
}
|
|
1304
|
+
})
|
|
1305
|
+
.collect())
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
/// Runs one admission-serialized history maintenance pass without starting a new job.
|
|
1309
|
+
pub(crate) fn reap_history(paths: &ControlPaths) -> Result<u64, String> {
|
|
1310
|
+
let _admission = admission::AdmissionGuard::acquire(paths)?;
|
|
1311
|
+
let limits = store::effective_limits(paths)?;
|
|
1312
|
+
store::reap(paths, limits.storage_limit_mib)
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
fn source_tag(source_key: &str) -> String {
|
|
1316
|
+
let hash = source_key
|
|
1317
|
+
.bytes()
|
|
1318
|
+
.fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
|
|
1319
|
+
(hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
|
|
1320
|
+
});
|
|
1321
|
+
format!("{:06x}", hash & 0x00ff_ffff)
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
pub(crate) fn running_summaries(paths: &ControlPaths) -> Result<Vec<JobSummary>, String> {
|
|
1325
|
+
Ok(summaries(paths)?
|
|
1326
|
+
.into_iter()
|
|
1327
|
+
.filter(|job| job.status == JobSummaryStatus::Running)
|
|
1328
|
+
.collect())
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
pub(crate) fn reap(paths: &ControlPaths) -> Result<u64, String> {
|
|
1332
|
+
let _admission = admission::AdmissionGuard::acquire(paths)?;
|
|
1333
|
+
let limits = store::effective_limits(paths)?;
|
|
1334
|
+
store::reap(paths, limits.storage_limit_mib)
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
#[cfg(unix)]
|
|
1338
|
+
pub(crate) fn run_bootstrap_entry() -> Result<(), String> {
|
|
1339
|
+
match host::run_bootstrap() {
|
|
1340
|
+
Ok(()) => Ok(()),
|
|
1341
|
+
Err(error) => {
|
|
1342
|
+
host::write_startup_error(&error);
|
|
1343
|
+
Err(error)
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
pub(crate) fn run_host_entry() -> Result<(), String> {
|
|
1349
|
+
host::run_job_host()
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
#[cfg(unix)]
|
|
1353
|
+
pub(crate) fn run_watchdog_entry(pid: u32, started: String) -> Result<(), String> {
|
|
1354
|
+
host::run_watchdog(pid, started)
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
#[cfg(test)]
|
|
1358
|
+
mod tests {
|
|
1359
|
+
use super::{OutputSnapshot, format_snapshot};
|
|
1360
|
+
use crate::budget::TokenBudget;
|
|
1361
|
+
|
|
1362
|
+
use crate::shell::jobs::model::{
|
|
1363
|
+
CaptureErrorRecord, ExitRecord, JobStatus, StoredLine, TerminationKind,
|
|
1364
|
+
};
|
|
1365
|
+
use std::path::PathBuf;
|
|
1366
|
+
|
|
1367
|
+
fn exited(code: i32, ended_order: u64) -> JobStatus {
|
|
1368
|
+
JobStatus::Exited(ExitRecord {
|
|
1369
|
+
exit_code: code,
|
|
1370
|
+
total_lines: 0,
|
|
1371
|
+
had_loss: false,
|
|
1372
|
+
ended_at: "2026-07-16T10:00:09Z".to_string(),
|
|
1373
|
+
ended_at_unix_nanos: ended_order,
|
|
1374
|
+
termination: TerminationKind::Exited,
|
|
1375
|
+
capture_error: None,
|
|
1376
|
+
output_truncation: None,
|
|
1377
|
+
})
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
#[test]
|
|
1381
|
+
fn direct_and_legacy_terminals_keep_their_capability_promises_separate() {
|
|
1382
|
+
let budget = TokenBudget {
|
|
1383
|
+
value: 8_500,
|
|
1384
|
+
variable: "FASTCTX_TOKEN_BUDGET",
|
|
1385
|
+
};
|
|
1386
|
+
let interrupted = format_snapshot(
|
|
1387
|
+
"j-000001",
|
|
1388
|
+
0,
|
|
1389
|
+
&OutputSnapshot {
|
|
1390
|
+
status: JobStatus::Interrupted,
|
|
1391
|
+
head: Vec::new(),
|
|
1392
|
+
tail: Vec::new(),
|
|
1393
|
+
unread_first: 4,
|
|
1394
|
+
unread_last: 3,
|
|
1395
|
+
all_unread_loaded: true,
|
|
1396
|
+
total_lines: 3,
|
|
1397
|
+
legacy_loss: false,
|
|
1398
|
+
capture_error: None,
|
|
1399
|
+
output_truncation: None,
|
|
1400
|
+
default_encoding: None,
|
|
1401
|
+
anchor: 3,
|
|
1402
|
+
direct_log: Some(PathBuf::from("/jobs/j-000001/output.log")),
|
|
1403
|
+
},
|
|
1404
|
+
None,
|
|
1405
|
+
budget,
|
|
1406
|
+
)
|
|
1407
|
+
.unwrap();
|
|
1408
|
+
assert!(interrupted.response.contains("Full log:"));
|
|
1409
|
+
assert!(interrupted.response.contains("output.log"));
|
|
1410
|
+
|
|
1411
|
+
let capture = format_snapshot(
|
|
1412
|
+
"j-000002",
|
|
1413
|
+
0,
|
|
1414
|
+
&OutputSnapshot {
|
|
1415
|
+
status: exited(17, 1),
|
|
1416
|
+
head: vec![StoredLine {
|
|
1417
|
+
seq: 1,
|
|
1418
|
+
bytes: b"kept".to_vec(),
|
|
1419
|
+
total_bytes: 4,
|
|
1420
|
+
stream_encoding: None,
|
|
1421
|
+
legacy_text: None,
|
|
1422
|
+
known_truncated: false,
|
|
1423
|
+
}],
|
|
1424
|
+
tail: Vec::new(),
|
|
1425
|
+
unread_first: 1,
|
|
1426
|
+
unread_last: 1,
|
|
1427
|
+
all_unread_loaded: true,
|
|
1428
|
+
total_lines: 2,
|
|
1429
|
+
legacy_loss: true,
|
|
1430
|
+
capture_error: Some(CaptureErrorRecord {
|
|
1431
|
+
after_seq: 1,
|
|
1432
|
+
reason: "disk unavailable".to_string(),
|
|
1433
|
+
}),
|
|
1434
|
+
output_truncation: None,
|
|
1435
|
+
default_encoding: None,
|
|
1436
|
+
anchor: 0,
|
|
1437
|
+
direct_log: None,
|
|
1438
|
+
},
|
|
1439
|
+
None,
|
|
1440
|
+
budget,
|
|
1441
|
+
)
|
|
1442
|
+
.unwrap();
|
|
1443
|
+
assert!(capture.response.contains("this legacy record stops here"));
|
|
1444
|
+
assert!(capture.response.contains("cannot be retrieved"));
|
|
1445
|
+
assert!(!capture.response.contains("Full log:"));
|
|
1446
|
+
assert!(!capture.response.contains("offset="));
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
#[test]
|
|
1450
|
+
fn a_capture_failure_on_a_direct_log_keeps_the_exit_status_and_points_at_the_log() {
|
|
1451
|
+
let budget = TokenBudget {
|
|
1452
|
+
value: 8_500,
|
|
1453
|
+
variable: "FASTCTX_TOKEN_BUDGET",
|
|
1454
|
+
};
|
|
1455
|
+
let rendered = format_snapshot(
|
|
1456
|
+
"j-000003",
|
|
1457
|
+
0,
|
|
1458
|
+
&OutputSnapshot {
|
|
1459
|
+
status: exited(17, 1),
|
|
1460
|
+
head: vec![StoredLine {
|
|
1461
|
+
seq: 1,
|
|
1462
|
+
bytes: b"output".to_vec(),
|
|
1463
|
+
total_bytes: 6,
|
|
1464
|
+
stream_encoding: None,
|
|
1465
|
+
legacy_text: None,
|
|
1466
|
+
known_truncated: false,
|
|
1467
|
+
}],
|
|
1468
|
+
tail: Vec::new(),
|
|
1469
|
+
unread_first: 1,
|
|
1470
|
+
unread_last: 1,
|
|
1471
|
+
all_unread_loaded: true,
|
|
1472
|
+
total_lines: 1,
|
|
1473
|
+
legacy_loss: false,
|
|
1474
|
+
capture_error: Some(CaptureErrorRecord {
|
|
1475
|
+
after_seq: 1,
|
|
1476
|
+
reason: "disk unavailable".to_string(),
|
|
1477
|
+
}),
|
|
1478
|
+
output_truncation: None,
|
|
1479
|
+
default_encoding: None,
|
|
1480
|
+
anchor: 0,
|
|
1481
|
+
direct_log: Some(PathBuf::from("/jobs/j-000003/output.log")),
|
|
1482
|
+
},
|
|
1483
|
+
None,
|
|
1484
|
+
budget,
|
|
1485
|
+
)
|
|
1486
|
+
.unwrap();
|
|
1487
|
+
assert!(
|
|
1488
|
+
rendered
|
|
1489
|
+
.response
|
|
1490
|
+
.contains("output capture failed after seq 1: disk unavailable"),
|
|
1491
|
+
"{}",
|
|
1492
|
+
rendered.response
|
|
1493
|
+
);
|
|
1494
|
+
assert!(
|
|
1495
|
+
rendered
|
|
1496
|
+
.response
|
|
1497
|
+
.contains("This does not kill the process; its exit status remains available"),
|
|
1498
|
+
"{}",
|
|
1499
|
+
rendered.response
|
|
1500
|
+
);
|
|
1501
|
+
assert!(
|
|
1502
|
+
rendered.response.contains("output.log") && rendered.response.contains("stops here.)"),
|
|
1503
|
+
"{}",
|
|
1504
|
+
rendered.response
|
|
1505
|
+
);
|
|
1506
|
+
assert!(
|
|
1507
|
+
rendered.response.contains("exited 17"),
|
|
1508
|
+
"{}",
|
|
1509
|
+
rendered.response
|
|
1510
|
+
);
|
|
1511
|
+
assert!(!rendered.response.contains("legacy record"));
|
|
1512
|
+
}
|
|
1513
|
+
}
|