@tishlang/tish-format 1.0.12 → 2.0.1

package/crates/tish_core/src/vmref.rs

@@ -0,0 +1,183 @@
+//! Shared-mutable reference used by the Tish runtime for `Value::Array`,
+//! `Value::Object`, and `Value::RegExp` payloads.
+//!
+//! ## Why this exists
+//!
+//! Tish's `Value` uses interior mutability for arrays, objects, and regex
+//! state. Historically that was `Rc<RefCell<T>>`, which is fast but
+//! `!Send` — so `Value` couldn't move across threads, which in turn meant
+//! `serve(port, handler)` had to serialise every request through one
+//! VM dispatcher thread.
+//!
+//! `VmRef<T>` lets the build system pick the right trade-off **per
+//! compile target**:
+//!
+//! | feature `send-values`     | `VmRef<T>`              | `NativeFn`                       | targets                                          |
+//! |---------------------------|-------------------------|----------------------------------|--------------------------------------------------|
+//! | **off** *(default)*       | `Rc<RefCell<T>>`        | `Rc<dyn Fn + 'static>`           | wasm32, wasi, interpreter, cranelift/llvm VMs    |
+//! | **on**                    | `Arc<Mutex<T>>`         | `Arc<dyn Fn + Send + Sync>`      | Rust native with `http` enabled (server workloads) |
+//!
+//! The *API* is identical in both configurations (`borrow` / `borrow_mut`
+//! / `ptr_eq` / `Clone`), so every existing call site in the workspace
+//! compiles unchanged. What flips is only the underlying primitive.
+//!
+//! ## Why this matters for performance
+//!
+//! * **wasm / wasi / cranelift / llvm / interpreter**: still pure
+//!   `Rc<RefCell<T>>`. Zero atomic ops, no mutex churn, behaviour
+//!   bit-identical to the pre-migration baseline.
+//! * **Rust native, non-server**: same — `send-values` only activates
+//!   when something in the dependency graph (usually `http`) needs it.
+//! * **Rust native with server**: `Arc<Mutex<T>>` pays ~3–5 ns per
+//!   `borrow` in the uncontended case (single atomic CAS). On Tish's
+//!   hot paths — roughly 6–12 borrows per request — that's ~30–60 ns of
+//!   overhead. In exchange we get `N×` handler scaling across cores,
+//!   which recovers orders of magnitude more throughput than it costs.
+//!
+//! ## API surface
+//!
+//! ```ignore
+//! let cell = VmRef::new(42);
+//! *cell.borrow() + 1;          // read
+//! *cell.borrow_mut() = 99;     // write
+//! VmRef::ptr_eq(&a, &b);       // identity
+//! let clone = cell.clone();    // shared ownership
+//! ```
+//!
+//! Returned guard types (`VmReadGuard<'_, T>`, `VmWriteGuard<'_, T>`) are
+//! type aliases that pick `Ref`/`RefMut` or `MutexGuard` depending on the
+//! feature. They both `Deref` (and, for write guards, `DerefMut`) to `T`
+//! just like the underlying types.
+
+use std::fmt;
+
+// --------------------------------------------------------------------------
+// Single-threaded backing store (default): Rc<RefCell<T>>
+// --------------------------------------------------------------------------
+#[cfg(not(feature = "send-values"))]
+mod imp {
+    use std::cell::RefCell;
+    use std::rc::Rc;
+
+    #[derive(Default)]
+    pub struct VmRef<T: ?Sized>(pub(super) Rc<RefCell<T>>);
+
+    /// Read guard alias. On the single-threaded path this is a true
+    /// `Ref<'_, T>`, so multiple readers can coexist.
+    pub type ReadGuard<'a, T> = std::cell::Ref<'a, T>;
+    /// Write guard alias. Exclusive, `DerefMut`.
+    pub type WriteGuard<'a, T> = std::cell::RefMut<'a, T>;
+
+    impl<T> VmRef<T> {
+        #[inline]
+        pub fn new(value: T) -> Self {
+            VmRef(Rc::new(RefCell::new(value)))
+        }
+    }
+
+    impl<T: ?Sized> VmRef<T> {
+        #[inline]
+        pub fn borrow(&self) -> ReadGuard<'_, T> {
+            self.0.borrow()
+        }
+
+        #[inline]
+        pub fn borrow_mut(&self) -> WriteGuard<'_, T> {
+            self.0.borrow_mut()
+        }
+
+        #[inline]
+        pub fn ptr_eq(a: &Self, b: &Self) -> bool {
+            Rc::ptr_eq(&a.0, &b.0)
+        }
+
+        #[inline]
+        pub fn strong_count(this: &Self) -> usize {
+            Rc::strong_count(&this.0)
+        }
+    }
+
+    impl<T: ?Sized> Clone for VmRef<T> {
+        #[inline]
+        fn clone(&self) -> Self {
+            VmRef(Rc::clone(&self.0))
+        }
+    }
+}
+
+// --------------------------------------------------------------------------
+// Thread-safe backing store (opt-in): Arc<Mutex<T>>
+// --------------------------------------------------------------------------
+#[cfg(feature = "send-values")]
+mod imp {
+    use parking_lot::Mutex;
+    use std::sync::Arc;
+
+    #[derive(Default)]
+    pub struct VmRef<T: ?Sized>(pub(super) Arc<Mutex<T>>);
+
+    /// Read guard alias. On the multi-threaded path both readers and
+    /// writers share a single `MutexGuard` (exclusive access).
+    pub type ReadGuard<'a, T> = parking_lot::MutexGuard<'a, T>;
+    /// Write guard alias.
+    pub type WriteGuard<'a, T> = parking_lot::MutexGuard<'a, T>;
+
+    impl<T> VmRef<T> {
+        #[inline]
+        pub fn new(value: T) -> Self {
+            VmRef(Arc::new(Mutex::new(value)))
+        }
+    }
+
+    impl<T: ?Sized> VmRef<T> {
+        /// Acquire the inner mutex. `parking_lot::Mutex` is used (not
+        /// `std::sync::Mutex`): its uncontended lock is a single atomic with
+        /// no pthread syscall — a profile of object/array-heavy code showed
+        /// `pthread_mutex_lock/unlock` as a top cost under send-values, since
+        /// every property/element access locks. It also has no poisoning, so
+        /// there is no `Result` to swallow (a handler panic aborts the thread
+        /// regardless).
+        #[inline]
+        pub fn borrow(&self) -> ReadGuard<'_, T> {
+            self.0.lock()
+        }
+
+        #[inline]
+        pub fn borrow_mut(&self) -> WriteGuard<'_, T> {
+            self.0.lock()
+        }
+
+        #[inline]
+        pub fn ptr_eq(a: &Self, b: &Self) -> bool {
+            Arc::ptr_eq(&a.0, &b.0)
+        }
+
+        #[inline]
+        pub fn strong_count(this: &Self) -> usize {
+            Arc::strong_count(&this.0)
+        }
+    }
+
+    impl<T: ?Sized> Clone for VmRef<T> {
+        #[inline]
+        fn clone(&self) -> Self {
+            VmRef(Arc::clone(&self.0))
+        }
+    }
+}
+
+pub use imp::{ReadGuard as VmReadGuard, VmRef, WriteGuard as VmWriteGuard};
+
+impl<T: fmt::Debug> fmt::Debug for VmRef<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Match `RefCell`'s debug format so snapshot-test output stays
+        // stable across the migration.
+        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            let guard = self.borrow();
+            format!("{:?}", &*guard)
+        })) {
+            Ok(s) => write!(f, "RefCell {{ value: {} }}", s),
+            Err(_) => write!(f, "RefCell {{ value: <borrowed> }}"),
+        }
+    }
+}

package/crates/tish_cranelift/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "tishlang_cranelift"
+version = "0.1.0"
+edition = "2021"
+description = "Bytecode to native via Cranelift"
+
+license-file = { workspace = true }
+repository = { workspace = true }
+[dependencies]
+tishlang_build_utils = { path = "../tish_build_utils", version = ">=0.1" }
+cranelift = "0.130"
+cranelift-codegen = "0.130"
+cranelift-frontend = "0.130"
+cranelift-module = "0.130"
+cranelift-native = "0.130"
+cranelift-object = "0.130"
+target-lexicon = "0.13"
+tishlang_bytecode = { path = "../tish_bytecode", version = ">=0.1" }
+tishlang_core = { path = "../tish_core", version = ">=0.1" }

package/crates/tish_cranelift/src/lib.rs

@@ -0,0 +1,43 @@
+//! Standalone native binary: embedded bytecode + VM (Cranelift used as object builder).
+//!
+//! Produces an executable that runs **`tishlang_vm`** on serialized bytecode embedded in
+//! the binary — not lowering of Tish opcodes to CLIF/machine code (see module docs in
+//! `lower.rs`). For Rust transpile + `tishlang_runtime`, use `--native-backend rust`.
+
+mod link;
+mod lower;
+
+pub use link::link_to_binary;
+
+use std::path::Path;
+
+use tishlang_bytecode::Chunk;
+
+/// Error from Cranelift compilation.
+#[derive(Debug)]
+pub struct CraneliftError {
+    pub message: String,
+}
+
+impl std::fmt::Display for CraneliftError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl std::error::Error for CraneliftError {}
+
+/// Build a native binary that embeds `chunk` and runs it with the bytecode VM.
+/// `features` are passed to `tishlang_cranelift_runtime` (e.g. fs, process, http).
+pub fn compile_chunk_to_native(
+    chunk: &Chunk,
+    output_path: &Path,
+    features: &[String],
+) -> Result<(), CraneliftError> {
+    let object_path = output_path.with_extension("o");
+    lower::lower_and_emit(chunk, &object_path)?;
+    link::link_to_binary(&object_path, output_path, features)?;
+    // Clean up .o file
+    let _ = std::fs::remove_file(&object_path);
+    Ok(())
+}

package/crates/tish_cranelift/src/link.rs

@@ -0,0 +1,130 @@
+//! Link object file with runtime to produce final binary.
+//!
+//! Uses Cargo to build a small binary that links our .o and runs the chunk.
+
+use std::fs;
+use std::path::Path;
+
+use crate::CraneliftError;
+
+pub fn link_to_binary(
+    object_path: &Path,
+    output_path: &Path,
+    features: &[String],
+) -> Result<(), CraneliftError> {
+    let workspace_root =
+        tishlang_build_utils::find_workspace_root().map_err(|e| CraneliftError { message: e })?;
+    let out_name = output_path
+        .file_stem()
+        .and_then(|s| s.to_str())
+        .unwrap_or("tish_out");
+    let build_dir = tishlang_build_utils::create_build_dir("tishlang_cranelift_build", out_name)
+        .map_err(|e| CraneliftError { message: e })?;
+
+    let object_path_str = object_path
+        .canonicalize()
+        .map_err(|e| CraneliftError {
+            message: format!("Cannot canonicalize object path: {}", e),
+        })?
+        .display()
+        .to_string()
+        .replace('\\', "/");
+
+    // tishlang_cranelift_runtime crate lives in crates/tish_cranelift_runtime
+    let runtime_path = workspace_root
+        .join("crates")
+        .join("tish_cranelift_runtime")
+        .canonicalize()
+        .map_err(|e| CraneliftError {
+            message: format!("Cannot find tishlang_cranelift_runtime: {}", e),
+        })?
+        .display()
+        .to_string()
+        .replace('\\', "/");
+
+    let features_str = if features.is_empty() {
+        String::new()
+    } else {
+        format!(
+            ", features = [{}]",
+            features
+                .iter()
+                .map(|f| format!("{:?}", f))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+    };
+    // UNIQUE package name per program. With the shared target dir, a fixed package name
+    // (`tishlang_cranelift_out`) made cargo cross-contaminate builds — each program links its own
+    // per-program object via build.rs, so program B would reuse program A's cached build script and
+    // link A's (now-deleted) `.o`. A distinct package name keeps them separate while the heavy
+    // dependency (`tishlang_cranelift_runtime`, same path + features for all) still compiles once.
+    let pkg_name = format!("clout_{}", out_name);
+    let cargo_toml_fixed = format!(
+        r#"[package]
+name = "{}"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "{}"
+path = "src/main.rs"
+
+[dependencies]
+tishlang_cranelift_runtime = {{ path = {:?}{} }}
+"#,
+        pkg_name, out_name, runtime_path, features_str
+    );
+
+    let main_rs = r#"
+extern "C" {
+    static tish_chunk_data: [u8; 1];
+    static tish_chunk_len: u64;
+}
+
+fn main() {
+    let len = unsafe { tish_chunk_len } as usize;
+    let ptr = unsafe { tish_chunk_data.as_ptr() };
+    let exit_code = tishlang_cranelift_runtime::tish_run_chunk(ptr, len);
+    std::process::exit(exit_code);
+}
+"#;
+
+    let build_rs = format!(
+        r#"
+fn main() {{
+    println!("cargo:rustc-link-arg={}");
+}}
+"#,
+        object_path_str
+    );
+
+    fs::write(build_dir.join("Cargo.toml"), cargo_toml_fixed).map_err(|e| CraneliftError {
+        message: format!("Cannot write Cargo.toml: {}", e),
+    })?;
+    fs::write(build_dir.join("src/main.rs"), main_rs).map_err(|e| CraneliftError {
+        message: format!("Cannot write main.rs: {}", e),
+    })?;
+    fs::write(build_dir.join("build.rs"), build_rs).map_err(|e| CraneliftError {
+        message: format!("Cannot write build.rs: {}", e),
+    })?;
+
+    // Build into a SHARED target dir (one per host), not the per-program `build_dir/target`.
+    // The heavy deps (cranelift_codegen + the embedded VM, ~several GB) then compile ONCE and
+    // are reused by every cranelift build; only each program's tiny main + object is rebuilt.
+    // Without this, every program left its own multi-GB `target/` behind and a full-suite sweep
+    // filled the disk (see docs/full-backend-parity-plan.md A3). Concurrent builds are serialized
+    // by `run_cargo_build`'s nested-cargo mutex and cargo's own target lock, so sharing is safe.
+    let shared_target = std::env::temp_dir().join("tishlang_cranelift_target");
+    tishlang_build_utils::run_cargo_build(&build_dir, Some(&shared_target), None)
+        .map_err(|e| CraneliftError { message: e })?;
+
+    let binary_dir = shared_target.join("release");
+    let binary = tishlang_build_utils::find_release_binary(&binary_dir, out_name)
+        .map_err(|e| CraneliftError { message: e })?;
+    let target = tishlang_build_utils::resolve_output_path(output_path, out_name);
+    tishlang_build_utils::copy_binary_to_output(&binary, &target)
+        .map_err(|e| CraneliftError { message: e })?;
+
+    Ok(())
+}

package/crates/tish_cranelift/src/lower.rs

@@ -0,0 +1,85 @@
+//! Embed serialized bytecode in an object file for the standalone native binary.
+//!
+//! **This is not AOT compilation of Tish into Cranelift IR.** The chunk is stored as
+//! read-only data (`tish_chunk_data`, `tish_chunk_len`). The link step produces an
+//! executable that **deserializes the chunk and runs `tishlang_vm`** — same VM as
+//! `tish run --backend vm`. Cranelift is only the object-file emitter for that blob.
+
+use std::path::Path;
+
+use cranelift::codegen::settings;
+use cranelift::codegen::settings::Configurable;
+use cranelift_module::{DataDescription, Linkage, Module};
+use cranelift_object::{ObjectBuilder, ObjectModule};
+
+use tishlang_bytecode::{serialize, Chunk};
+
+use crate::CraneliftError;
+
+pub fn lower_and_emit(chunk: &Chunk, object_path: &Path) -> Result<(), CraneliftError> {
+    let mut settings_builder = settings::builder();
+    settings_builder
+        .set("opt_level", "speed")
+        .map_err(|_| CraneliftError {
+            message: "Failed to set opt_level".to_string(),
+        })?;
+    let flags = settings::Flags::new(settings_builder);
+
+    let isa_builder = cranelift_native::builder().map_err(|e| CraneliftError {
+        message: format!("Failed to build ISA: {}", e),
+    })?;
+    let isa = isa_builder.finish(flags).map_err(|e| CraneliftError {
+        message: format!("Failed to finish ISA: {}", e),
+    })?;
+
+    let object_builder = ObjectBuilder::new(
+        isa,
+        "tishlang_cranelift",
+        cranelift_module::default_libcall_names(),
+    )
+    .map_err(|e| CraneliftError {
+        message: format!("Failed to create ObjectBuilder: {}", e),
+    })?;
+    let mut module = ObjectModule::new(object_builder);
+
+    // Serialize chunk and emit as data - link step will build a Rust binary that reads it
+    let chunk_data = serialize(chunk);
+    let chunk_len = chunk_data.len() as u64;
+    let data_id = module
+        .declare_data("tish_chunk_data", Linkage::Export, false, false)
+        .map_err(|e| CraneliftError {
+            message: format!("Failed to declare chunk data: {}", e),
+        })?;
+    let mut data_desc = DataDescription::new();
+    data_desc.define(chunk_data.into_boxed_slice());
+    module
+        .define_data(data_id, &data_desc)
+        .map_err(|e| CraneliftError {
+            message: format!("Failed to define chunk data: {}", e),
+        })?;
+
+    let len_data = chunk_len.to_le_bytes();
+    let len_id = module
+        .declare_data("tish_chunk_len", Linkage::Export, false, false)
+        .map_err(|e| CraneliftError {
+            message: format!("Failed to declare chunk len: {}", e),
+        })?;
+    let mut len_desc = DataDescription::new();
+    len_desc.define(len_data.to_vec().into_boxed_slice());
+    module
+        .define_data(len_id, &len_desc)
+        .map_err(|e| CraneliftError {
+            message: format!("Failed to define chunk len: {}", e),
+        })?;
+
+    let object_product = module.finish();
+    let bytes = object_product.emit().map_err(|e| CraneliftError {
+        message: format!("Failed to emit object: {}", e),
+    })?;
+
+    std::fs::write(object_path, bytes).map_err(|e| CraneliftError {
+        message: format!("Failed to write object file: {}", e),
+    })?;
+
+    Ok(())
+}

package/crates/tish_cranelift_runtime/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tishlang_cranelift_runtime"
+version = "0.1.0"
+edition = "2021"
+description = "Runtime for Cranelift-compiled Tish bytecode"
+
+license-file = { workspace = true }
+repository = { workspace = true }
+[features]
+default = []
+fs = ["tishlang_vm/fs"]
+process = ["tishlang_vm/process"]
+http = ["tishlang_vm/http"]
+promise = ["tishlang_vm/promise"]
+timers = ["tishlang_vm/timers"]
+ws = ["tishlang_vm/ws"]
+tty = ["tishlang_vm/tty"]
+regex = ["tishlang_vm/regex"]
+
+[lib]
+crate-type = ["staticlib", "rlib"]
+
+[dependencies]
+tishlang_bytecode = { path = "../tish_bytecode", version = ">=0.1" }
+tishlang_vm = { path = "../tish_vm", version = ">=0.1" }
+tishlang_core = { path = "../tish_core", version = ">=0.1" }

package/crates/tish_cranelift_runtime/src/lib.rs

@@ -0,0 +1,45 @@
+//! Runtime linked into the `tish build --native-backend cranelift` executable.
+//!
+//! **`tish_run_chunk`** deserializes embedded bytecode and runs **`tishlang_vm`** — the same
+//! execution engine as `tish run --backend vm`. The crate name is historical; this is not
+//! running CLIF-emitted machine code for each Tish opcode.
+
+use tishlang_bytecode::deserialize;
+use tishlang_vm::Vm;
+
+/// Serialization format:
+/// - u64: code len
+/// - bytes: code
+/// - u64: constants count
+/// - for each constant: u8 tag + payload
+/// - u64: names count
+/// - for each name: u64 len + bytes
+///
+/// Rust-callable wrapper. Run serialized chunk data. Returns exit code (0 on success).
+pub fn tish_run_chunk(ptr: *const u8, len: usize) -> i32 {
+    tish_run_chunk_impl(ptr, len)
+}
+
+#[no_mangle]
+extern "C" fn tish_run_chunk_impl(ptr: *const u8, len: usize) -> i32 {
+    if ptr.is_null() || len < 8 {
+        return 1;
+    }
+    let slice = unsafe { std::slice::from_raw_parts(ptr, len) };
+    match deserialize(slice) {
+        Ok(chunk) => {
+            let mut vm = Vm::new();
+            match vm.run(&chunk) {
+                Ok(_) => 0,
+                Err(e) => {
+                    eprintln!("Runtime error: {}", e);
+                    1
+                }
+            }
+        }
+        Err(e) => {
+            eprintln!("Deserialization error: {}", e);
+            1
+        }
+    }
+}

package/crates/tish_eval/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "tishlang_eval"
+version = "0.1.0"
+edition = "2021"
+description = "Tish tree-walk interpreter"
+
+license-file = { workspace = true }
+repository = { workspace = true }
+[features]
+default = []
+# setTimeout / setInterval / clear* (standalone or with `import { … } from "tish:timers"`).
+timers = []
+http = [
+    "timers",
+    "tokio",
+    "reqwest",
+    "futures",
+    "tiny_http",
+    "tishlang_core/regex",
+    "dep:tishlang_runtime",
+    "tishlang_runtime/http",
+    # Interpreter + http means the runtime's `NativeFn` is `Arc<... + Send>`,
+    # so the interpreter's `CoreFn` variant must use the same shape.
+    "tishlang_core/send-values",
+    "tishlang_builtins/send-values",
+]
+fs = []
+process = []
+tty = ["dep:tishlang_runtime", "tishlang_runtime/tty"]
+regex = ["dep:fancy-regex", "tishlang_core/regex"]
+tokio = ["dep:tokio"]
+ws = ["dep:tishlang_runtime", "tishlang_runtime/ws"]
+
+[dependencies]
+ahash = "0.8.12"
+indexmap = "2"
+rand = "0.10.1"
+# On-demand native stack growth for deep (non-tail) recursion in the tree-walker,
+# mirroring the bytecode VM. Plain dep (stacker is a no-op shim on wasm32, where
+# tish_vm already depends on it for the wasi backend).
+stacker = "0.1"
+tishlang_ast = { path = "../tish_ast", version = ">=0.1" }
+tishlang_builtins = { path = "../tish_builtins", version = ">=0.1" }
+tishlang_parser = { path = "../tish_parser", version = ">=0.1" }
+tishlang_core = { path = "../tish_core", version = ">=0.1" }
+reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "json", "stream"], optional = true }
+futures = { version = "0.3.32", optional = true }
+tiny_http = { version = "0.12.0", optional = true }
+fancy-regex = { version = "0.17.0", optional = true }
+tokio = { version = "1.50.0", features = ["rt-multi-thread", "time", "sync"], optional = true }
+tishlang_runtime = { path = "../tish_runtime", version = ">=0.1", optional = true }