@tishlang/tish-format 3.10.3 → 3.10.9

package/crates/tish_lsp/src/main.rs

@@ -447,7 +447,10 @@ impl LanguageServer for Backend {
         // files so an externally edited declaration (`.d.tish`) or source file refreshes the
         // workspace-symbol index without a server restart (#161). The client only delivers these
         // events for globs the server registers; a `.tish` glob does NOT cover `.d.tish` (it has a
-        // compound extension), so both are registered explicitly.
+        // compound extension), so both are registered explicitly. Also watch the manifests that
+        // drive `cargo:` resolution — `tish.rustDependencies` lives in package.json, and a path
+        // dep resolves through its Cargo.toml/Cargo.lock — so a dependency edit can invalidate
+        // `cargo_src_cache` (#714); without these globs no event could ever reach it.
         let watchers = |glob: &str| FileSystemWatcher {
             glob_pattern: GlobPattern::String(glob.to_string()),
             kind: None, // create | change | delete
@@ -456,7 +459,13 @@ impl LanguageServer for Backend {
             id: "tish-watch-d-tish".to_string(),
             method: "workspace/didChangeWatchedFiles".to_string(),
             register_options: serde_json::to_value(DidChangeWatchedFilesRegistrationOptions {
-                watchers: vec![watchers("**/*.tish"), watchers("**/*.d.tish")],
+                watchers: vec![
+                    watchers("**/*.tish"),
+                    watchers("**/*.d.tish"),
+                    watchers("**/package.json"),
+                    watchers("**/Cargo.toml"),
+                    watchers("**/Cargo.lock"),
+                ],
             })
             .ok(),
         };
@@ -485,6 +494,10 @@ impl LanguageServer for Backend {
             let mut roots = self.roots.write().unwrap();
             apply_workspace_folder_changes(&mut roots, &params.event);
         }
+        // The cargo-crate-root cache goes with the old folder set (#714): `cargo:` resolution is
+        // root-relative, entries are cheap to recompute on the next goto-definition/hover, and
+        // clearing here mirrors how the folder change already re-scopes the symbol index.
+        self.cargo_src_cache.write().unwrap().clear();
         self.client
             .log_message(MessageType::INFO, "tish-lsp: workspace folders updated")
             .await;
@@ -510,6 +523,25 @@ impl LanguageServer for Backend {
                 }
             }
         }
+        // #714: a manifest change re-routes `cargo:` resolution — `tish.rustDependencies` lives in
+        // package.json, and a path dep resolves through its Cargo.toml/Cargo.lock — so drop the
+        // whole cargo-crate-root cache when any watched manifest changes. Whole-cache rather than
+        // per-root pruning: a path dep's manifest may live outside every project root, entries are
+        // a few hundred bytes, and they are recomputed per (human-rate) goto-definition/hover.
+        let manifest_changed = params.changes.iter().any(|change| {
+            change
+                .uri
+                .to_file_path()
+                .ok()
+                .and_then(|p| {
+                    p.file_name()
+                        .map(|f| f == "package.json" || f == "Cargo.toml" || f == "Cargo.lock")
+                })
+                .unwrap_or(false)
+        });
+        if manifest_changed {
+            self.cargo_src_cache.write().unwrap().clear();
+        }
         // Re-lint any OPEN buffer whose declarations may now have changed. A `.d.tish` supplies
         // ambient `declare` bindings other buffers depend on, so a change there can flip
         // unresolved-name diagnostics; recompute open docs against the new on-disk state.
@@ -2558,6 +2590,63 @@ mod jsonrpc_integration_tests {
         std::fs::remove_dir_all(&dir).ok();
     }
 
+    /// Seed `cargo_src_cache` the way a successful goto-definition would.
+    fn seed_cargo_src_cache(service: &LspService<Backend>) {
+        service.inner().cargo_src_cache.write().unwrap().insert(
+            (PathBuf::from("/proj"), "cargo:serde".to_string()),
+            PathBuf::from("/old/crate/root"),
+        );
+    }
+
+    fn cargo_src_cache_len(service: &LspService<Backend>) -> usize {
+        service.inner().cargo_src_cache.read().unwrap().len()
+    }
+
+    fn did_change_watched_files_request(uri: &str) -> Request {
+        Request::build("workspace/didChangeWatchedFiles")
+            .params(serde_json::json!({ "changes": [ { "uri": uri, "type": 2 } ] }))
+            .finish()
+    }
+
+    // #714: a workspace-folder change must drop the cargo-crate-root cache along with the roots
+    // rewrite — previously nothing ever invalidated it, so a stale crate root was served for the
+    // rest of the session.
+    #[tokio::test]
+    async fn workspace_folder_change_clears_cargo_src_cache() {
+        let mut service = new_service();
+        initialize(&mut service).await;
+        seed_cargo_src_cache(&service);
+
+        let dir = crate::test_fs::unique_temp_dir("cargocache");
+        let uri = Url::from_file_path(&dir).unwrap().to_string();
+        let _ = call(&mut service, did_change_workspace_folders(&[&uri], &[])).await;
+        assert_eq!(cargo_src_cache_len(&service), 0, "folder change must clear the cargo src cache");
+
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    // #714: a watched manifest change (package.json / Cargo.toml / Cargo.lock — the files that
+    // drive `cargo:` resolution) must drop the cargo-crate-root cache; a plain `.tish` change
+    // must leave it alone.
+    #[tokio::test]
+    async fn manifest_change_clears_cargo_src_cache() {
+        let mut service = new_service();
+        initialize(&mut service).await;
+
+        // A `.tish` event leaves the cargo cache untouched…
+        seed_cargo_src_cache(&service);
+        let _ = call(&mut service, did_change_watched_files_request("file:///proj/a.tish")).await;
+        assert_eq!(cargo_src_cache_len(&service), 1, "a .tish change must not clear the cargo src cache");
+
+        // …while any manifest event drops it.
+        for name in ["package.json", "Cargo.toml", "Cargo.lock"] {
+            seed_cargo_src_cache(&service);
+            let req = did_change_watched_files_request(&format!("file:///proj/{name}"));
+            let _ = call(&mut service, req).await;
+            assert_eq!(cargo_src_cache_len(&service), 0, "a {name} change must clear the cargo src cache");
+        }
+    }
+
     // #162 protocol edge cases the client is allowed to send: adding the same folder twice must not
     // change the result, and removing a folder that was never added must not evict a tracked root.
     #[tokio::test]

package/crates/tish_runtime/src/fs_ext.rs

@@ -74,31 +74,40 @@ fn ms_since_epoch(t: std::io::Result<SystemTime>) -> f64 {
 }
 
 /// Build a Node-like `Stats` object from metadata (predicate methods + size + times + mode).
+///
+/// FIXED key order via `object_from_pairs`: building through an `ObjectMap` (AHashMap) iterated
+/// in random per-instance order made every call mint a fresh permanent chain in the global shape
+/// registry (~2.3KB leaked per `stat()` call, unbounded — a file watcher stat-walking a workspace
+/// leaked multi-GB/hour). Fixed-order construction dedupes to one cached shape chain, and skips
+/// the intermediate hashmap entirely.
 fn stats_object(md: &std::fs::Metadata) -> Value {
-    let mut m = ObjectMap::default();
     let is_file = md.is_file();
     let is_dir = md.is_dir();
     let is_symlink = md.file_type().is_symlink();
-    m.insert("isFile".into(), Value::native(move |_| Value::Bool(is_file)));
-    m.insert("isDirectory".into(), Value::native(move |_| Value::Bool(is_dir)));
-    m.insert("isSymbolicLink".into(), Value::native(move |_| Value::Bool(is_symlink)));
-    m.insert("isBlockDevice".into(), Value::native(|_| Value::Bool(false)));
-    m.insert("isCharacterDevice".into(), Value::native(|_| Value::Bool(false)));
-    m.insert("isFIFO".into(), Value::native(|_| Value::Bool(false)));
-    m.insert("isSocket".into(), Value::native(|_| Value::Bool(false)));
-    m.insert("size".into(), Value::Number(md.len() as f64));
-    m.insert("mtimeMs".into(), Value::Number(ms_since_epoch(md.modified())));
-    m.insert("atimeMs".into(), Value::Number(ms_since_epoch(md.accessed())));
-    m.insert("birthtimeMs".into(), Value::Number(ms_since_epoch(md.created())));
     #[cfg(unix)]
-    {
+    let (mode, uid, gid, ino) = {
         use std::os::unix::fs::MetadataExt;
-        m.insert("mode".into(), Value::Number(md.mode() as f64));
-        m.insert("uid".into(), Value::Number(md.uid() as f64));
-        m.insert("gid".into(), Value::Number(md.gid() as f64));
-        m.insert("ino".into(), Value::Number(md.ino() as f64));
-    }
-    Value::object(m)
+        (md.mode() as f64, md.uid() as f64, md.gid() as f64, md.ino() as f64)
+    };
+    #[cfg(not(unix))]
+    let (mode, uid, gid, ino) = (0.0_f64, 0.0_f64, 0.0_f64, 0.0_f64);
+    Value::object_from_pairs([
+        ("isFile".into(), Value::native(move |_| Value::Bool(is_file))),
+        ("isDirectory".into(), Value::native(move |_| Value::Bool(is_dir))),
+        ("isSymbolicLink".into(), Value::native(move |_| Value::Bool(is_symlink))),
+        ("isBlockDevice".into(), Value::native(|_| Value::Bool(false))),
+        ("isCharacterDevice".into(), Value::native(|_| Value::Bool(false))),
+        ("isFIFO".into(), Value::native(|_| Value::Bool(false))),
+        ("isSocket".into(), Value::native(|_| Value::Bool(false))),
+        ("size".into(), Value::Number(md.len() as f64)),
+        ("mtimeMs".into(), Value::Number(ms_since_epoch(md.modified()))),
+        ("atimeMs".into(), Value::Number(ms_since_epoch(md.accessed()))),
+        ("birthtimeMs".into(), Value::Number(ms_since_epoch(md.created()))),
+        ("mode".into(), Value::Number(mode)),
+        ("uid".into(), Value::Number(uid)),
+        ("gid".into(), Value::Number(gid)),
+        ("ino".into(), Value::Number(ino)),
+    ])
 }
 
 // ── cores ─────────────────────────────────────────────────────────────────────────────────

package/crates/tish_runtime/src/http_hyper.rs

@@ -491,6 +491,17 @@ fn handle_ws_upgrade(mut req: Request<hyper::body::Incoming>) -> Response<Full<B
     use tokio_tungstenite::tungstenite::handshake::derive_accept_key;
     use tokio_tungstenite::tungstenite::protocol::Role;
 
+    // Enforce the same connection cap as the tish:ws listener BEFORE the 101. This path runs
+    // pre-auth — the VM handler never sees the request — so without the check an unauthenticated
+    // client could register unbounded permanent connections (tasks + fd + buffers each) on any
+    // serve() app built with http-hyper + ws (#707).
+    if !crate::ws::has_ws_capacity() {
+        return simple_error_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "websocket connection limit reached",
+        );
+    }
+
     let accept = match req.headers().get(hyper::header::SEC_WEBSOCKET_KEY) {
         Some(k) => derive_accept_key(k.as_bytes()),
         None => return simple_error_response(StatusCode::BAD_REQUEST, "missing Sec-WebSocket-Key"),
@@ -501,8 +512,13 @@ fn handle_ws_upgrade(mut req: Request<hyper::body::Incoming>) -> Response<Full<B
         match on_upgrade.await {
             Ok(upgraded) => {
                 let io = TokioIo::new(upgraded);
-                let ws = tokio_tungstenite::WebSocketStream::from_raw_socket(io, Role::Server, None)
-                    .await;
+                // Shared config: 16 KiB buffers instead of tungstenite's eager 128 KiB default.
+                let ws = tokio_tungstenite::WebSocketStream::from_raw_socket(
+                    io,
+                    Role::Server,
+                    Some(crate::ws::ws_config()),
+                )
+                .await;
                 let id = crate::ws::register_ws_stream(ws);
                 crate::ws::enqueue_upgraded_conn(id);
             }

package/crates/tish_runtime/src/lib.rs

@@ -1773,6 +1773,10 @@ pub use tty::{
     tty_enter_alt_screen, tty_is_tty, tty_leave_alt_screen, tty_read, tty_read_line, tty_set_raw_mode, tty_size,
 };
 
+/// Shared reader-thread buffer + UTF-8 drain for the byte-stream modules (pty / process / net).
+#[cfg(any(feature = "pty", feature = "process", feature = "net"))]
+mod stream_buf;
+
 #[cfg(feature = "pty")]
 pub mod pty;
 #[cfg(feature = "pty")]

package/crates/tish_runtime/src/net.rs

@@ -18,24 +18,28 @@
 //!   - `probe(host, port, timeoutMs?) -> bool`        (can we connect?)
 //!   - `sleep(ms) -> null`                            (blocking backoff for the poll loops above)
 //!
-//! Each connection has a reader thread filling a UTF-8-boundary-drained buffer, exactly like
-//! pty.rs / process_spawn.rs. Global OnceLock<Mutex<HashMap>> registries; errors surface as
+//! Each connection has a reader thread filling a bounded, UTF-8-boundary-drained buffer shared
+//! with pty.rs / process_spawn.rs (see `stream_buf.rs` for the cap/backpressure and
+//! invalid-byte semantics). Global OnceLock<Mutex<HashMap>> registries; errors surface as
 //! null/false.
+//!
+//! Connection lifecycle: `close(id)` shuts the socket down (`Shutdown::Both`) so the reader
+//! thread — which holds a dup of the fd — unblocks and exits, freeing thread + fd + buffer
+//! together, and the peer sees a FIN. Entries whose peer disconnected and whose buffer was
+//! read to EOF (`read` returned `null`) are swept when the next connection registers, so
+//! read-to-EOF-and-forget cannot strand CLOSE_WAIT fds; a swept id keeps returning `null`
+//! from `read`, exactly as before.
 
 use std::collections::HashMap;
-use std::io::{ErrorKind, Read, Write};
+use std::io::{ErrorKind, Write};
 use std::net::{TcpListener, TcpStream, ToSocketAddrs};
 use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Arc, Condvar, Mutex, OnceLock};
+use std::sync::{Mutex, OnceLock};
 use std::time::{Duration, Instant};
 
 use tishlang_core::{ObjectMap, Value};
 
-struct StreamBuf {
-    data: Vec<u8>,
-    eof: bool,
-}
-type SharedBuf = Arc<(Mutex<StreamBuf>, Condvar)>;
+use crate::stream_buf::{drain_stream, is_drained, new_buf, retire_buf, spawn_reader, SharedBuf};
 
 struct Conn {
     writer: Mutex<TcpStream>,
@@ -82,95 +86,20 @@ fn arg_timeout(args: &[Value], i: usize) -> u64 {
     }
 }
 
-fn new_buf() -> SharedBuf {
-    Arc::new((
-        Mutex::new(StreamBuf {
-            data: Vec::new(),
-            eof: false,
-        }),
-        Condvar::new(),
-    ))
-}
-
-// Reader thread: block on the socket, append bytes, wake waiters; mark EOF on 0/err.
-fn spawn_reader(mut stream: TcpStream, buf: SharedBuf) {
-    std::thread::spawn(move || {
-        let mut tmp = [0u8; 8192];
-        loop {
-            match stream.read(&mut tmp) {
-                Ok(0) => {
-                    let (lock, cv) = &*buf;
-                    if let Ok(mut b) = lock.lock() {
-                        b.eof = true;
-                    }
-                    cv.notify_all();
-                    break;
-                }
-                Ok(n) => {
-                    let (lock, cv) = &*buf;
-                    if let Ok(mut b) = lock.lock() {
-                        b.data.extend_from_slice(&tmp[..n]);
-                    }
-                    cv.notify_all();
-                }
-                Err(_) => {
-                    let (lock, cv) = &*buf;
-                    if let Ok(mut b) = lock.lock() {
-                        b.eof = true;
-                    }
-                    cv.notify_all();
-                    break;
-                }
-            }
-        }
-    });
-}
-
-fn drain_stream(buf: &SharedBuf, timeout_ms: u64) -> Value {
-    let (lock, cv) = &**buf;
-    let mut b = match lock.lock() {
-        Ok(b) => b,
-        Err(_) => return Value::Null,
-    };
-    if b.data.is_empty() && !b.eof && timeout_ms > 0 {
-        let res = cv.wait_timeout_while(b, Duration::from_millis(timeout_ms), |b| {
-            b.data.is_empty() && !b.eof
-        });
-        b = match res {
-            Ok((g, _)) => g,
-            Err(e) => e.into_inner().0,
-        };
-    }
-    if b.data.is_empty() {
-        if b.eof {
-            return Value::Null;
-        }
-        return Value::String("".into());
-    }
-    let valid = match std::str::from_utf8(&b.data) {
-        Ok(_) => b.data.len(),
-        Err(e) => e.valid_up_to(),
-    };
-    if valid == 0 {
-        if b.eof {
-            let s = String::from_utf8_lossy(&b.data).into_owned();
-            b.data.clear();
-            return Value::String(s.into());
-        }
-        return Value::String("".into());
-    }
-    let out: Vec<u8> = b.data.drain(..valid).collect();
-    Value::String(String::from_utf8(out).unwrap_or_default().into())
-}
-
-// Register a connected stream (reader thread + writer clone) and return its id.
+// Register a connected stream (reader thread + writer clone) and return its id. Also sweeps
+// out entries whose peer disconnected and whose buffer was fully read (`read` already
+// returned `null` for them), so read-to-EOF-and-forget cannot strand CLOSE_WAIT fds — the
+// sweep never touches a conn with undrained data or a live reader, so an id that has not yet
+// returned `null` keeps its full read contract.
 fn register_conn(stream: TcpStream) -> Option<u64> {
     let reader = stream.try_clone().ok()?;
     let _ = reader.set_nonblocking(false);
     let buf = new_buf();
     spawn_reader(reader, buf.clone());
     let id = NEXT_ID.fetch_add(1, Ordering::SeqCst);
-    conns().lock().ok()?.insert(
+    let mut g = conns().lock().ok()?;
+    g.retain(|_, c| !is_drained(&c.buf));
+    g.insert(
         id,
         Conn {
             writer: Mutex::new(stream),
@@ -262,9 +191,12 @@ pub fn net_write(args: &[Value]) -> Value {
     }
 }
 
-/// `close(id)` → drop a connection (its reader thread ends on EOF) or a listener. Conn and
-/// listener ids share one counter, so an id names at most one of the two; closing a listener frees
-/// its port — the reserve-ephemeral-then-release pattern a TCP DAP `spawn` uses so the child can
+/// `close(id)` → close a connection or a listener. For a connection this actually shuts the
+/// socket down (`Shutdown::Both`, the `http.rs` idiom): the reader thread — which holds a dup
+/// of the fd — unblocks out of its `read()`, exits, and frees thread + fd + buffer together,
+/// and the peer sees a FIN instead of a silently half-alive socket. Conn and listener ids
+/// share one counter, so an id names at most one of the two; closing a listener frees its
+/// port — the reserve-ephemeral-then-release pattern a TCP DAP `spawn` uses so the child can
 /// bind it. Returns whether anything live was removed.
 pub fn net_close(args: &[Value]) -> Value {
     let id = match arg_u64(args, 0) {
@@ -272,10 +204,15 @@ pub fn net_close(args: &[Value]) -> Value {
         None => return Value::Bool(false),
     };
     let removed_conn = match conns().lock() {
-        Ok(mut g) => g.remove(&id).is_some(),
+        Ok(mut g) => g.remove(&id),
         Err(_) => return Value::Bool(false),
     };
-    if removed_conn {
+    if let Some(c) = removed_conn {
+        if let Ok(w) = c.writer.lock() {
+            let _ = w.shutdown(std::net::Shutdown::Both);
+        }
+        // Wake a reader parked at the buffer cap (shutdown only unblocks a read-blocked one).
+        retire_buf(&c.buf);
         return Value::Bool(true);
     }
     let removed_listener = match listeners().lock() {
@@ -468,6 +405,97 @@ mod tests {
         assert!(matches!(net_close(&[Value::Number(server)]), Value::Bool(true)));
     }
 
+    fn listen_pair() -> (f64, f64, f64) {
+        let l = net_listen(&[Value::Number(0.0)]);
+        let (lid, port) = match &l {
+            Value::Object(m) => {
+                let b = m.borrow();
+                let id = match b.strings.get("id") {
+                    Some(Value::Number(i)) => *i,
+                    _ => panic!("no id"),
+                };
+                let port = match b.strings.get("port") {
+                    Some(Value::Number(p)) => *p,
+                    _ => panic!("no port"),
+                };
+                (id, port)
+            }
+            other => panic!("listen failed: {:?}", other),
+        };
+        let client = match net_connect(&[Value::String("127.0.0.1".into()), Value::Number(port)]) {
+            Value::Number(n) => n,
+            other => panic!("connect failed: {:?}", other),
+        };
+        let server = match net_accept(&[Value::Number(lid), Value::Number(1000.0)]) {
+            Value::Number(n) => n,
+            other => panic!("accept failed: {:?}", other),
+        };
+        (lid, client, server)
+    }
+
+    #[test]
+    fn close_shuts_down_the_socket_so_the_peer_sees_eof() {
+        // Pre-fix, net_close only dropped the writer fd: the reader thread's dup kept the
+        // socket alive, no FIN was ever sent, and the peer's reads blocked forever.
+        let (lid, client, server) = listen_pair();
+        assert!(matches!(
+            net_close(&[Value::Number(server)]),
+            Value::Bool(true)
+        ));
+        let mut saw_eof = false;
+        for _ in 0..100 {
+            if let Value::Null = net_read(&[Value::Number(client), Value::Number(100.0)]) {
+                saw_eof = true;
+                break;
+            }
+        }
+        assert!(saw_eof, "peer never saw EOF after close — no FIN was sent");
+        let _ = net_close(&[Value::Number(client)]);
+        let _ = net_close(&[Value::Number(lid)]);
+    }
+
+    #[test]
+    fn conn_read_to_eof_is_swept_on_next_register() {
+        // The natural "read until null, forget the id" pattern: pre-fix the Conn entry (and
+        // its CLOSE_WAIT fd) survived forever.
+        let (lid, client, server) = listen_pair();
+        assert!(matches!(
+            net_write(&[Value::Number(server), Value::String("bye".into())]),
+            Value::Bool(true)
+        ));
+        assert!(matches!(
+            net_close(&[Value::Number(server)]),
+            Value::Bool(true)
+        ));
+        // Drain the client to EOF (null) — and never close it.
+        let mut got = String::new();
+        let mut saw_eof = false;
+        for _ in 0..100 {
+            match net_read(&[Value::Number(client), Value::Number(100.0)]) {
+                Value::String(s) => got.push_str(&s),
+                Value::Null => {
+                    saw_eof = true;
+                    break;
+                }
+                _ => {}
+            }
+        }
+        assert!(saw_eof, "client never reached EOF");
+        assert!(got.contains("bye"), "payload lost before EOF: {:?}", got);
+        // Registering the next connection sweeps the drained entry.
+        let (lid2, client2, server2) = listen_pair();
+        assert!(
+            !conns().lock().unwrap().contains_key(&(client as u64)),
+            "drained-at-EOF conn was not swept"
+        );
+        // The read-to-EOF-then-read-again contract: the swept id still reads as null.
+        assert!(matches!(net_read(&[Value::Number(client)]), Value::Null));
+        let _ = net_close(&[Value::Number(client2)]);
+        let _ = net_close(&[Value::Number(server2)]);
+        let _ = net_close(&[Value::Number(lid)]);
+        let _ = net_close(&[Value::Number(lid2)]);
+    }
+
     #[test]
     fn probe_and_unknown() {
         // Nothing listening on this port -> probe false.