rubyx-py 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9296be89cdf78ab0e25656b6643c5a7c407157d7e2767c8707fdf79a5369bdc8
+  data.tar.gz: 1f60ada7b7c16f048b5f8cd259526d7e5534b70fd7d53a6efa33d8ac0cb7c6c4
+SHA512:
+  metadata.gz: 60ffe57e3abbb175e595c9c26bf0a8128fa872d358dfa9a121d267b6e54dcd00ad4ac413071540ed0c050b6d322eeac93cdbe191a94d81f7049e315398e910df
+  data.tar.gz: 8095f4af8147e579aadc3bd928886379070292ab01fbee68c69259caeefeaeaefb87970f501fba47a51ebe2192f5ed685c1455c88ee5f761a7cc02493b36acc6

data/Cargo.toml

@@ -0,0 +1,19 @@
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+description = "Ruby-Python bridge powered by Rust"
+license = "MIT"
+
+[workspace]
+members = ["ext/rubyx"]
+resolver = "2"
+
+
+[workspace.dependencies]
+magnus = { version = "0.8", features = ["embed"] }
+libloading = "0.9"
+libc = "1.0.0-alpha.3"
+thiserror = "2.0"
+rb-sys = { version = "0.9", features = ["link-ruby"] }
+crossbeam-channel = "0.5"
+serial_test = "3.4.0"

data/README.md

@@ -0,0 +1,469 @@
+<div align="center">
+
+# Rubyx-py
+
+**Call Python from Ruby. No microservices, no REST APIs, no serialization overhead.**
+
+Powered by Rust for safety and performance. Built for Rails.
+
+[![Gem Version](https://badge.fury.io/rb/rubyx.svg)](https://badge.fury.io/rb/rubyx)
+[![CI](https://github.com/yinho999/rubyx/actions/workflows/ci.yml/badge.svg)](https://github.com/yinho999/rubyx/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Ruby](https://img.shields.io/badge/Ruby-%3E%3D%203.0-red.svg)](https://www.ruby-lang.org)
+[![Rust](https://img.shields.io/badge/Rust-powered-orange.svg)](https://www.rust-lang.org)
+
+</div>
+
+---
+
+```ruby
+np = Rubyx.import('numpy')
+np.array([1, 2, 3]).mean().to_ruby # => 2.0
+```
+
+```ruby
+Rubyx.eval("sum(items)", items: [1, 2, 3, 4]) # => 10
+```
+
+```ruby
+# Stream LLM tokens in real-time
+Rubyx.stream(llm.generate("Tell me about Ruby")).each { |token| print token }
+```
+
+```ruby
+# Non-blocking — Ruby stays free while Python works
+future = Rubyx.async_await("model.predict(data)", data: [1, 2, 3])
+do_other_work()
+result = future.value # get result when ready
+```
+
+### Built with non-blocking in mind
+
+- **`Rubyx.stream`** / **`Rubyx.nb_stream`** — release Ruby's GVL during iteration, other threads and Fibers keep
+  running
+- **`Rubyx.async_await`** — spawns Python on background threads, returns a `Future` immediately
+- **`Rubyx.await`** — blocks only when you choose to
+
+Ideal for LLM streaming, ML inference, data pipelines, and high-concurrency Rails apps.
+
+## Install
+
+```ruby
+# Gemfile
+gem 'rubyx-py'
+```
+
+Python is auto-managed by [uv](https://github.com/astral-sh/uv). No manual install needed.
+
+## Rails Setup Simple Example
+
+```bash
+rails generate rubyx:install
+```
+
+Creates `config/initializers/rubyx.rb`, `pyproject.toml`, and `app/python/`.
+
+### Configuration
+
+```ruby
+# config/initializers/rubyx.rb
+Rubyx::Rails.configure do |config|
+  config.pyproject_path = Rails.root.join('pyproject.toml')
+  config.auto_init = true
+  config.python_paths = [Rails.root.join('app/python').to_s]
+end
+```
+
+### Python dependencies
+
+```toml
+# pyproject.toml
+[project]
+name = "my-app"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = []
+```
+
+### 1. Sync — call a Python function
+
+```python
+# app/python/hello.py
+def greet(name):
+    return f"Hello, {name}!"
+```
+
+```ruby
+class GreetingsController < ApplicationController
+  def show
+    hello = Rubyx.import('hello')
+    render json: { message: hello.greet(params[:name]).to_ruby }
+  end
+end
+```
+
+### 2. Streaming — iterate a Python generator
+
+```python
+# app/python/count.py
+def count_up(n):
+    for i in range(n):
+        yield f"Step {i + 1}"
+```
+
+```ruby
+class CountController < ApplicationController
+  include ActionController::Live
+
+  def stream
+    response.headers['Content-Type'] = 'text/event-stream'
+
+    counter = Rubyx.import('count')
+    Rubyx.stream(counter.count_up(5)).each do |msg|
+      response.stream.write("data: #{msg}\n\n")
+    end
+  ensure
+    response.stream.close
+  end
+end
+```
+
+### 3. Async — non-blocking Python calls
+
+```python
+# app/python/tasks.py
+import asyncio
+
+
+async def delayed_greet(name, seconds=1):
+    await asyncio.sleep(seconds)
+    return f"Hello, {name}! (after {seconds}s)"
+```
+
+```ruby
+class TasksController < ApplicationController
+  def show
+    tasks = Rubyx.import('tasks')
+
+    # Non-blocking — returns a Future immediately
+    future = Rubyx.async_await(tasks.delayed_greet(params[:name], seconds: 2))
+    do_other_work()
+    render json: { message: future.value.to_ruby }
+  end
+end
+```
+
+## Rails Setup Advanced LLM Example
+
+```bash
+rails generate rubyx:install
+```
+
+Creates `config/initializers/rubyx.rb`, `pyproject.toml`, and `app/python/`.
+
+### Configuration
+
+```ruby
+# config/initializers/rubyx.rb
+Rubyx::Rails.configure do |config|
+  config.pyproject_path = Rails.root.join('pyproject.toml')
+  config.auto_init = true
+  config.python_paths = [Rails.root.join('app/python').to_s]
+end
+```
+
+### Python dependencies
+
+```toml
+# pyproject.toml
+[project]
+name = "my-app"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = ["numpy", "pandas", "transformers"]
+```
+
+### Write Python code
+
+```python
+# app/python/services/text_processing.py
+class TextAnalyzer:
+    def __init__(self, text):
+        self.text = text
+        self._words = text.split()
+
+    def summary(self):
+        return {
+            "word_count": len(self._words),
+            "unique_words": len(set(self._words)),
+            "avg_word_length": round(
+                sum(len(w) for w in self._words) / max(len(self._words), 1), 2
+            ),
+        }
+```
+
+### Call it from Rails
+
+```ruby
+
+class AnalysisController < ApplicationController
+  def analyze
+    tp = Rubyx.import('services.text_processing')
+    analyzer = tp.TextAnalyzer(params[:text])
+    render json: analyzer.summary.to_ruby
+  end
+end
+```
+
+### SSE streaming (LLM-style)
+
+```python
+# app/python/services/llm.py
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+import threading
+
+_model, _tokenizer = None, None
+
+
+def load_model(name="Qwen/Qwen2.5-0.5B-Instruct"):
+    global _model, _tokenizer
+    _tokenizer = AutoTokenizer.from_pretrained(name)
+    _model = AutoModelForCausalLM.from_pretrained(name, torch_dtype="auto")
+
+
+def stream_generate(prompt, max_tokens=256):
+    messages = [{"role": "user", "content": prompt}]
+    text = _tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    inputs = _tokenizer([text], return_tensors="pt").to(_model.device)
+    streamer = TextIteratorStreamer(_tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    thread = threading.Thread(target=_model.generate,
+                              kwargs={**inputs, "max_new_tokens": max_tokens, "streamer": streamer})
+    thread.start()
+    for token in streamer:
+        if token:
+            yield token
+    thread.join()
+```
+
+```ruby
+# config/initializers/rubyx.rb
+Rubyx::Rails.configure do |config|
+  config.pyproject_path = Rails.root.join('pyproject.toml')
+  config.auto_init = true
+  config.python_paths = [Rails.root.join('app/python').to_s]
+end
+
+# Load models once at boot — must be inside after_initialize
+# so Python is already initialized by the Railtie
+Rails.application.config.after_initialize do
+  llm = Rubyx.import('services.llm')
+  llm.load_model("Qwen/Qwen2.5-0.5B-Instruct")
+end
+```
+
+```ruby
+class ChatController < ApplicationController
+  include ActionController::Live
+
+  def stream
+    llm = Rubyx.import('services.llm')
+    response.headers['Content-Type'] = 'text/event-stream'
+
+    Rubyx.stream(llm.stream_generate(params[:prompt])).each do |token|
+      token_str = token.to_s.gsub("\n", "\\n")
+      response.stream.write("data: #{token_str}\n\n")
+    end
+    response.stream.write("data: [DONE]\n\n")
+  ensure
+    response.stream.close
+  end
+end
+```
+
+### Rake tasks
+
+```bash
+rake rubyx:init         # Initialize Python environment
+rake rubyx:status       # Check environment status
+rake rubyx:packages     # List installed Python packages
+rake rubyx:clear_cache  # Clear cached environments
+```
+
+## Standalone Setup
+
+```ruby
+require 'rubyx'
+
+Rubyx.uv_init <<~TOML
+  [project]
+  name = "my-script"
+  version = "0.1.0"
+  requires-python = ">=3.12"
+  dependencies = ["numpy"]
+TOML
+
+np = Rubyx.import('numpy')
+np.array([1, 2, 3, 4, 5]).std().to_ruby # => 1.4142135623730951
+```
+
+## Eval with Globals
+
+Pass Ruby data directly into Python:
+
+```ruby
+Rubyx.eval("x ** 2 + y ** 2", x: 3, y: 4).to_ruby # => 25
+Rubyx.eval("f'Hello, {name}!'", name: "World").to_ruby # => "Hello, World!"
+Rubyx.eval("max(items)", items: [3, 1, 4, 1, 5]).to_ruby # => 5
+```
+
+Supports: Integer, Float, String, Symbol, Bool, nil, Array, Hash, and RubyxObject.
+
+## Python Objects
+
+Python objects are wrapped as `RubyxObject`:
+
+```ruby
+os = Rubyx.import('os')
+os.getcwd().to_ruby # => "/home/user"
+os.path.exists("/tmp").to_ruby # => true
+
+# Subscript access
+d = Rubyx.eval("{'a': 1, 'b': 2}")
+d['a'].to_ruby # => 1
+d['c'] = 3
+
+# Enumerable
+py_list = Rubyx.eval("[1, 2, 3, 4, 5]")
+py_list.map { |x| x.to_ruby * 2 } # => [2, 4, 6, 8, 10]
+py_list.select { |x| x.to_ruby > 3 } # filtered RubyxObjects
+
+# Introspection
+py_list.truthy? # => true
+py_list.callable? # => false
+py_list.py_type # => "list"
+```
+
+## Context
+
+Persistent state across multiple eval calls:
+
+```ruby
+ctx = Rubyx.context
+
+ctx.eval("import math")
+ctx.eval("data = [1, 2, 3, 4, 5]")
+ctx.eval("avg = sum(data) / len(data)")
+ctx.eval("avg").to_ruby # => 3.0
+
+# Inject Ruby data into context
+ctx.eval("total = base + offset", base: 100, offset: 42)
+ctx.eval("total").to_ruby # => 142
+```
+
+## Streaming
+
+```ruby
+gen = Rubyx.eval("(x ** 2 for x in range(5))")
+Rubyx.stream(gen).each { |val| puts val } # 0, 1, 4, 9, 16
+
+# Non-blocking (releases GVL for other Ruby threads)
+Rubyx.nb_stream(gen).each { |val| process(val) }
+```
+
+## Async / Await
+
+```ruby
+ctx = Rubyx.context
+ctx.eval("import asyncio")
+ctx.eval("async def fetch(url): ...")
+
+# Blocking
+result = ctx.await("fetch(url)", url: "https://example.com")
+
+# Non-blocking (returns Future)
+future = ctx.async_await("fetch(url)", url: "https://example.com")
+do_other_stuff()
+result = future.value # blocks only when needed
+future.ready? # check without blocking
+```
+
+## Error Handling
+
+Python exceptions map to Ruby classes:
+
+```ruby
+
+begin
+  Rubyx.eval('{}["missing"]')
+rescue Rubyx::KeyError => e
+  puts e.message # includes Python traceback
+end
+```
+
+| Python                                | Ruby                          |
+|---------------------------------------|-------------------------------|
+| `KeyError`                            | `Rubyx::KeyError`             |
+| `IndexError`                          | `Rubyx::IndexError`           |
+| `ValueError`                          | `Rubyx::ValueError`           |
+| `TypeError`                           | `Rubyx::TypeError`            |
+| `AttributeError`                      | `Rubyx::AttributeError`       |
+| `ImportError` / `ModuleNotFoundError` | `Rubyx::ImportError`          |
+| `SyntaxError`                         | `SyntaxError` (Ruby built-in) |
+| Everything else                       | `Rubyx::PythonError`          |
+
+All inherit from `Rubyx::Error` (`StandardError`).
+
+## Local Python Files
+
+```python
+# app/python/services/analyzer.py
+class Analyzer:
+    def __init__(self, data):
+        self.data = data
+
+    def summary(self):
+        return {"count": len(self.data), "sum": sum(self.data)}
+```
+
+```ruby
+svc = Rubyx.import('services.analyzer')
+svc.Analyzer([1, 2, 3]).summary.to_ruby # => {"count" => 3, "sum" => 6}
+```
+
+## API Reference
+
+| Method                               | Description                     |
+|--------------------------------------|---------------------------------|
+| `Rubyx.uv_init(toml, **opts)`        | Setup Python env and initialize |
+| `Rubyx.import(name)`                 | Import a Python module          |
+| `Rubyx.eval(code, **globals)`        | Evaluate Python code            |
+| `Rubyx.await(code, **globals)`       | Run async code (blocking)       |
+| `Rubyx.async_await(code, **globals)` | Run async code (returns Future) |
+| `Rubyx.stream(iterable)`             | Stream a Python generator       |
+| `Rubyx.nb_stream(iterable)`          | Non-blocking stream (GVL-aware) |
+| `Rubyx.context`                      | Create isolated Python context  |
+| `Rubyx.initialized?`                 | Check if Python is ready        |
+
+| RubyxObject              |                               |
+|--------------------------|-------------------------------|
+| `.to_ruby`               | Convert to native Ruby type   |
+| `.to_s` / `.inspect`     | String / repr                 |
+| `.method_missing`        | Delegates to Python           |
+| `[]` / `[]=` / `.delete` | Subscript access              |
+| `.each`                  | Iterate (includes Enumerable) |
+| `.truthy?` / `.falsy?`   | Python truthiness             |
+| `.callable?`             | Check if callable             |
+| `.py_type`               | Python type name              |
+
+## Requirements
+
+- Ruby >= 3.0
+- Rust (for building from source)
+- Python >= 3.12 (auto-managed by uv)
+
+Precompiled gems available for Linux and macOS (x86_64 and ARM64).
+
+## License
+
+MIT

data/ext/rubyx/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "rubyx"
+version.workspace = true
+edition.workspace = true
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { workspace = true }
+libloading = { workspace = true }
+libc = { workspace = true }
+thiserror = { workspace = true }
+rb-sys = { workspace = true }
+crossbeam-channel = { workspace = true }
+
+[dev-dependencies]
+serial_test = { workspace = true }

data/ext/rubyx/extconf.rb

@@ -0,0 +1,22 @@
+require 'mkmf'
+
+unless respond_to?(:dummy_makefile)
+  def dummy_makefile(dir)
+    ["SHELL = /bin/sh",
+     "ECHO = @echo",
+     "Q = @",
+     "TOUCH = touch",
+     "COPY = cp",
+     "RM_RF = rm -rf",
+     "MAKEDIRS = mkdir -p",
+     "INSTALL_PROG = install -m 0755",
+     "all install static install-so install-rb:",
+     "pre-install-rb: install-rb",
+     "clean distclean realclean:",
+     "\t@-$(RM_RF) mkmf.log"]
+  end
+end
+
+require 'rb_sys/mkmf'
+
+create_rust_makefile('rubyx/rubyx')