@synsci/cli-darwin-x64 1.1.49

package/bin/skills/tinker/references/reinforcement-learning.md

@@ -0,0 +1,212 @@
+# Reinforcement Learning
+
+## Quick Start
+
+```bash
+python -m tinker_cookbook.recipes.rl_basic
+```
+
+Fine-tunes Llama-3.1-8B on GSM8K with reward:
+```
+1[answer correct] + 0.1 * (1[format correct] - 1)
+```
+
+## Basic RL Config
+
+```python
+import chz
+import asyncio
+from tinker_cookbook.rl import train
+from tinker_cookbook import model_info
+from tinker_cookbook.recipes.math_rl.math_env import Gsm8kDatasetBuilder
+
+def build_config_blueprint() -> chz.Blueprint[train.Config]:
+    model_name = "meta-llama/Llama-3.1-8B"
+    renderer_name = model_info.get_recommended_renderer_name(model_name)
+
+    builder = Gsm8kDatasetBuilder(
+        batch_size=128,
+        group_size=16,
+        renderer_name=renderer_name,
+        model_name_for_tokenizer=model_name,
+    )
+
+    return chz.Blueprint(train.Config).apply({
+        "model_name": model_name,
+        "log_path": "/tmp/rl_basic",
+        "dataset_builder": builder,
+        "learning_rate": 4e-5,
+        "max_tokens": 256,
+    })
+
+if __name__ == "__main__":
+    blueprint = build_config_blueprint()
+    blueprint.make_from_argv(sys.argv[1:])
+    asyncio.run(train.main(blueprint.make()))
+```
+
+## Key Metrics
+
+- `ac_tokens_per_turn` - Tokens per completion
+- `env/all/correct` - Accuracy
+- `env/all/format` - Format compliance
+- `env/all/reward/total` - Mean total reward
+- `entropy` - Per-token entropy
+- `kl_sample_train_v1/v2` - KL divergence (sampler vs learner)
+
+## Custom RL Loop
+
+```python
+import tinker
+from tinker import types
+from tinker.types.tensor_data import TensorData
+import torch
+from tinker_cookbook import model_info, renderers
+from tinker_cookbook.tokenizer_utils import get_tokenizer
+
+@chz.chz
+class Config:
+    model_name: str = "meta-llama/Llama-3.1-8B"
+    batch_size: int = 128
+    group_size: int = 16
+    learning_rate: float = 4e-5
+    max_tokens: int = 256
+
+def main(config: Config):
+    service_client = tinker.ServiceClient()
+    training_client = service_client.create_lora_training_client(
+        base_model=config.model_name, rank=32
+    )
+    tokenizer = training_client.get_tokenizer()
+    renderer = renderers.get_renderer(
+        model_info.get_recommended_renderer_name(config.model_name),
+        tokenizer
+    )
+
+    sampling_params = types.SamplingParams(
+        max_tokens=config.max_tokens,
+        stop=renderer.get_stop_sequences(),
+    )
+    adam_params = types.AdamParams(learning_rate=config.learning_rate)
+
+    for batch_idx, batch_rows in enumerate(dataset):
+        # Save weights for sampling
+        sampling_path = training_client.save_weights_for_sampler(name=f"{batch_idx:06d}").result().path
+        sampling_client = service_client.create_sampling_client(model_path=sampling_path)
+
+        datums = []
+        for question, answer in batch_rows:
+            convo = [{"role": "user", "content": question}]
+            model_input = renderer.build_generation_prompt(convo)
+            prompt_tokens = model_input.to_ints()
+
+            # Sample group_size responses
+            result = sampling_client.sample(
+                prompt=model_input,
+                num_samples=config.group_size,
+                sampling_params=sampling_params,
+            ).result()
+
+            rewards = []
+            for seq in result.sequences:
+                parsed, _ = renderer.parse_response(seq.tokens)
+                reward = compute_reward(parsed["content"], answer)
+                rewards.append(reward)
+
+            # GRPO-style advantage centering
+            mean_reward = sum(rewards) / len(rewards)
+            advantages = [r - mean_reward for r in rewards]
+
+            if all(a == 0 for a in advantages):
+                continue
+
+            for seq, advantage in zip(result.sequences, advantages):
+                tokens = prompt_tokens + seq.tokens
+                ob_len = len(prompt_tokens) - 1
+
+                datum = types.Datum(
+                    model_input=types.ModelInput.from_ints(tokens=tokens[:-1]),
+                    loss_fn_inputs={
+                        "target_tokens": TensorData.from_torch(torch.tensor(tokens[1:])),
+                        "logprobs": TensorData.from_torch(torch.tensor([0.0]*ob_len + list(seq.logprobs))),
+                        "advantages": TensorData.from_torch(torch.tensor([0.0]*ob_len + [advantage]*(len(tokens)-1-ob_len))),
+                    },
+                )
+                datums.append(datum)
+
+        # Training step
+        fwd_bwd = training_client.forward_backward(datums, loss_fn="importance_sampling")
+        optim = training_client.optim_step(adam_params)
+        fwd_bwd.result()
+        optim.result()
+```
+
+## Hyperparameters
+
+### Batch and Group Sizes
+
+- `batch_size`: Number of unique problems
+- `group_size`: Rollouts per problem (for variance reduction)
+
+Scale: `LR ∝ √batch_size`
+
+### Multiple Updates (num_substeps)
+
+```python
+# Default: 1 update per batch
+num_substeps = 1
+
+# Multiple updates: split batch into mini-batches
+num_substeps = 4  # Batch must be divisible
+```
+
+Use with PPO objective. Start with 2-4.
+
+### Streaming Minibatch Training
+
+Overlaps sampling and training for throughput:
+
+```python
+StreamMinibatchConfig(
+    groups_per_batch=128,
+    num_minibatches=8,
+)
+```
+
+### Async Off-Policy Training
+
+For long rollouts:
+
+```python
+AsyncConfig(
+    max_steps_off_policy=3,  # Max age of trajectories
+    groups_per_batch=64,
+)
+```
+
+## Monitoring
+
+### KL Divergence
+
+Monitor `kl_sample_train_v1/v2`:
+- Should stay below 0.01 for stable training
+- High KL indicates numerical instability
+
+### Reward Curves
+
+```python
+import pandas
+df = pandas.read_json("/tmp/rl-loop/metrics.jsonl", lines=True)
+plt.plot(df["reward/total"])
+```
+
+## Loss Functions for RL
+
+| Loss | Description |
+|------|-------------|
+| `importance_sampling` | Policy gradient with importance weighting |
+| `ppo` | Proximal Policy Optimization with clipping |
+| `cispo` | Clipped Importance Sampling PO |
+| `dro` | Direct Reward Optimization |
+
+See [Loss Functions](loss-functions.md) for details.

package/bin/skills/tinker/references/rendering.md

@@ -0,0 +1,243 @@
+# Rendering to Tokens
+
+Renderers convert messages ↔ tokens for training and inference.
+
+## Getting a Renderer
+
+```python
+from tinker_cookbook.model_info import get_recommended_renderer_name
+from tinker_cookbook.renderers import get_renderer
+from tinker_cookbook.tokenizer_utils import get_tokenizer
+
+model_name = "meta-llama/Llama-3.1-8B"
+renderer_name = get_recommended_renderer_name(model_name)
+
+tokenizer = get_tokenizer(model_name)
+renderer = get_renderer(name=renderer_name, tokenizer=tokenizer)
+```
+
+**Renderer names:** `qwen3`, `qwen3_disable_thinking`, `qwen3_instruct`, `qwen3_vl`, `qwen3_vl_instruct`, `llama3`, `deepseekv3`, `deepseekv3_thinking`, `kimi_k2`, `gpt_oss_no_sysprompt`, `gpt_oss_low_reasoning`, `gpt_oss_medium_reasoning`, `gpt_oss_high_reasoning`, `role_colon`
+
+## HuggingFace Compatibility
+
+Default renderers produce **identical tokens** to HuggingFace's `apply_chat_template`:
+
+| Renderer | HF Equivalent |
+|----------|---------------|
+| `qwen3` | `apply_chat_template(..., enable_thinking=True)` |
+| `qwen3_disable_thinking` | `apply_chat_template(..., enable_thinking=False)` |
+| `llama3` | `apply_chat_template(...)` * |
+| `deepseekv3` | `apply_chat_template(...)` |
+
+\* Llama3 omits "Cutting Knowledge Date..." preamble
+
+## Core Methods
+
+### build_supervised_example
+
+For training with loss weights:
+
+```python
+from tinker_cookbook.renderers import TrainOnWhat
+
+messages = [
+    {"role": "user", "content": "What is 2+2?"},
+    {"role": "assistant", "content": "4"},
+]
+
+model_input, weights = renderer.build_supervised_example(
+    messages,
+    train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES,
+)
+# model_input: ModelInput (list of chunks)
+# weights: per-token weights (0.0 = prompt, 1.0 = train)
+```
+
+### build_generation_prompt
+
+For inference:
+
+```python
+messages = [
+    {"role": "system", "content": "You are helpful."},
+    {"role": "user", "content": "What is 2+2?"},
+]
+
+prompt = renderer.build_generation_prompt(messages)
+# Returns ModelInput ready for sampling
+```
+
+### get_stop_sequences
+
+```python
+stop_sequences = renderer.get_stop_sequences()
+
+sampling_params = SamplingParams(
+    max_tokens=100,
+    stop=stop_sequences,
+)
+```
+
+### parse_response
+
+```python
+output_tokens = result.sequences[0].tokens
+message, success = renderer.parse_response(output_tokens)
+# {"role": "assistant", "content": "..."}
+```
+
+## TrainOnWhat Enum
+
+```python
+from tinker_cookbook.renderers import TrainOnWhat
+
+# Train on ALL assistant messages
+TrainOnWhat.ALL_ASSISTANT_MESSAGES
+
+# Train only on LAST assistant message
+TrainOnWhat.LAST_ASSISTANT_MESSAGE
+```
+
+**ALL_ASSISTANT_MESSAGES:**
+```python
+messages = [
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hi!"},  # weight=1
+    {"role": "user", "content": "How are you?"},
+    {"role": "assistant", "content": "Good!"},  # weight=1
+]
+```
+
+**LAST_ASSISTANT_MESSAGE:**
+```python
+messages = [
+    {"role": "user", "content": "Let me think..."},
+    {"role": "assistant", "content": "..."},  # weight=0
+    {"role": "user", "content": "Answer?"},
+    {"role": "assistant", "content": "42"},  # weight=1
+]
+```
+
+Use `LAST` for classification, reward modeling, preference learning.
+
+## Message Formats
+
+### Text-Only
+
+```python
+messages = [
+    {"role": "system", "content": "You are helpful."},
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hi!"},
+]
+```
+
+### Vision (Multi-Modal)
+
+```python
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "image": image_bytes},
+            {"type": "text", "text": "What's in this image?"},
+        ]
+    },
+    {"role": "assistant", "content": "A cat."}
+]
+```
+
+## Using with conversation_to_datum
+
+```python
+from tinker_cookbook.supervised.data import conversation_to_datum
+
+datum = conversation_to_datum(
+    messages=messages,
+    renderer=renderer,
+    max_length=2048,
+    train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES,
+)
+# Returns Datum ready for training
+```
+
+## Format Examples
+
+### ChatML
+
+```
+<|im_start|>system
+You are helpful.<|im_end|>
+<|im_start|>user
+What is 2+2?<|im_end|>
+<|im_start|>assistant
+4<|im_end|>
+```
+
+### Llama 3
+
+```
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+You are helpful.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+4<|eot_id|>
+```
+
+**Important:** Don't construct formats manually—use renderers!
+
+## Vision Renderers
+
+For VLMs (Qwen3-VL):
+
+```python
+from tinker_cookbook.image_processing_utils import get_image_processor
+
+model_name = "Qwen/Qwen3-VL-235B-A22B-Instruct"
+tokenizer = get_tokenizer(model_name)
+image_processor = get_image_processor(model_name)
+
+renderer = renderers.Qwen3VLInstructRenderer(tokenizer, image_processor)
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "image": "https://example.com/image.png"},
+            {"type": "text", "text": "What is this?"},
+        ]
+    }
+]
+
+prompt = renderer.build_generation_prompt(messages)
+```
+
+## In Dataset Builders
+
+`ChatDatasetBuilder` creates renderer automatically:
+
+```python
+@chz.chz
+class MyDatasetBuilder(ChatDatasetBuilder):
+    common_config: ChatDatasetBuilderCommonConfig
+
+    def __call__(self):
+        def map_fn(row):
+            return conversation_to_datum(
+                messages=messages,
+                renderer=self.renderer,  # Auto-created from common_config
+                max_length=self.common_config.max_length,
+                train_on_what=self.common_config.train_on_what,
+            )
+        # ...
+```
+
+## Troubleshooting
+
+**Wrong format:** Use `get_recommended_renderer_name(model_name)`
+
+**High loss:** Check weights (0.0 for prompts, 1.0 for completions)
+
+**Generation doesn't stop:** Use `renderer.get_stop_sequences()` in SamplingParams

package/bin/skills/tinker/references/supervised-learning.md

@@ -0,0 +1,232 @@
+# Supervised Learning
+
+## Quick Start with Cookbook
+
+```bash
+python -m tinker_cookbook.recipes.sl_basic
+```
+
+## Blueprint Pattern (Recommended)
+
+```python
+import chz
+import sys
+import asyncio
+from tinker_cookbook.supervised import train
+from tinker_cookbook.supervised.types import ChatDatasetBuilderCommonConfig
+from tinker_cookbook.supervised.data import FromConversationFileBuilder
+from tinker_cookbook.renderers import TrainOnWhat
+from tinker_cookbook.model_info import get_recommended_renderer_name
+
+def build_config_blueprint() -> chz.Blueprint[train.Config]:
+    model_name = "meta-llama/Llama-3.1-8B"
+    renderer_name = get_recommended_renderer_name(model_name)
+
+    common_config = ChatDatasetBuilderCommonConfig(
+        model_name_for_tokenizer=model_name,
+        renderer_name=renderer_name,
+        max_length=2048,
+        batch_size=128,
+        train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES,
+    )
+
+    dataset_builder = FromConversationFileBuilder(
+        common_config=common_config,
+        file_path="data.jsonl",
+    )
+
+    return chz.Blueprint(train.Config).apply({
+        "log_path": "/tmp/training",
+        "model_name": model_name,
+        "dataset_builder": dataset_builder,
+        "learning_rate": 2e-4,
+        "lr_schedule": "cosine",
+        "num_epochs": 3,
+        "lora_rank": 32,
+    })
+
+if __name__ == "__main__":
+    blueprint = build_config_blueprint()
+    blueprint.make_from_argv(sys.argv[1:])
+    asyncio.run(train.main(blueprint.make()))
+```
+
+## HuggingFace Dataset Builder
+
+```python
+from tinker_cookbook.supervised.types import ChatDatasetBuilder
+from tinker_cookbook.supervised.data import SupervisedDatasetFromHFDataset, conversation_to_datum
+import datasets
+
+@chz.chz
+class MyDatasetBuilder(ChatDatasetBuilder):
+    common_config: ChatDatasetBuilderCommonConfig
+
+    def __call__(self):
+        hf_dataset = datasets.load_dataset("HuggingFaceH4/no_robots", split="train")
+        split = hf_dataset.train_test_split(test_size=0.1, seed=42)
+
+        def map_fn(row):
+            messages = [
+                {"role": "user", "content": row["prompt"]},
+                {"role": "assistant", "content": row["completion"]},
+            ]
+            return conversation_to_datum(
+                messages=messages,
+                renderer=self.renderer,
+                max_length=self.common_config.max_length,
+                train_on_what=self.common_config.train_on_what,
+            )
+
+        train_dataset = SupervisedDatasetFromHFDataset(
+            hf_dataset=split["train"],
+            batch_size=self.common_config.batch_size,
+            map_fn=map_fn,
+        )
+        test_dataset = SupervisedDatasetFromHFDataset(
+            hf_dataset=split["test"],
+            batch_size=self.common_config.batch_size,
+            map_fn=map_fn,
+        )
+        return train_dataset, test_dataset
+```
+
+## Streaming Large Datasets
+
+For datasets >1M examples:
+
+```python
+from tinker_cookbook.supervised.data import StreamingSupervisedDatasetFromHFDataset
+
+@chz.chz
+class StreamingDatasetBuilder(ChatDatasetBuilder):
+    common_config: ChatDatasetBuilderCommonConfig
+    max_prompts: int = 100000
+
+    def __call__(self):
+        ds = datasets.load_dataset(
+            "open-thoughts/OpenThoughts3-1.2M",
+            split="train",
+            streaming=True,  # Important!
+        )
+
+        def map_fn(row):
+            messages = [
+                {"role": "user", "content": row["question"]},
+                {"role": "assistant", "content": row["response"]},
+            ]
+            return conversation_to_datum(
+                messages=messages,
+                renderer=self.renderer,
+                max_length=self.common_config.max_length,
+                train_on_what=self.common_config.train_on_what,
+            )
+
+        train_dataset = StreamingSupervisedDatasetFromHFDataset(
+            hf_dataset=ds,
+            batch_size=self.common_config.batch_size,
+            length=self.max_prompts,  # Required
+            map_fn=map_fn,
+            buffer_size=10000,
+        )
+        return train_dataset, train_dataset.take(1000)
+```
+
+## File-Based Dataset
+
+JSONL format:
+```json
+{"messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+```
+
+```python
+from tinker_cookbook.supervised.data import FromConversationFileBuilder
+
+dataset_builder = FromConversationFileBuilder(
+    common_config=common_config,
+    file_path="/path/to/data.jsonl",
+)
+```
+
+## TrainOnWhat Enum
+
+```python
+from tinker_cookbook.renderers import TrainOnWhat
+
+TrainOnWhat.ALL_ASSISTANT_MESSAGES  # Train on all assistant turns
+TrainOnWhat.LAST_ASSISTANT_MESSAGE  # Train only on final response
+```
+
+Use `LAST_ASSISTANT_MESSAGE` for:
+- Classification (only final answer matters)
+- Chain-of-thought where intermediate steps shouldn't be trained
+- Preference learning
+
+## Custom Dataset
+
+```python
+from tinker_cookbook.supervised.types import SupervisedDataset
+from tinker.types import Datum, ModelInput, TensorData
+import numpy as np
+
+class CustomDataset(SupervisedDataset):
+    def __init__(self, config):
+        self.config = config
+        self.tokenizer = get_tokenizer(config.model_name)
+        self.renderer = get_renderer(config.renderer_name, self.tokenizer)
+        self.data = self._load_data()
+
+    def __len__(self):
+        return len(self.data) // self.config.batch_size
+
+    def __iter__(self):
+        for item in self.data:
+            messages = self._preprocess(item)
+            example = self.renderer.build_supervised_example(
+                messages=messages,
+                train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES,
+            )
+            yield Datum(
+                model_input=ModelInput([example.chunk]),
+                loss_fn_inputs={
+                    "target_tokens": TensorData.from_numpy(np.array(example.target_tokens, dtype=np.int64)),
+                    "weights": TensorData.from_numpy(np.array(example.weights, dtype=np.float32)),
+                },
+            )
+```
+
+## Hyperparameters
+
+### Learning Rate
+
+```python
+from tinker_cookbook.hyperparam_utils import get_lr
+
+model_name = "meta-llama/Llama-3.2-1B"
+recommended_lr = get_lr(model_name)
+```
+
+Formula: `LR = lr_base * M_LoRA * (2000/H_m)^P_m`
+- `lr_base = 5e-5`
+- `M_LoRA = 10` (multiplier for LoRA)
+- `P_m = 0.0775` (Qwen) or `0.781` (Llama)
+
+### Batch Size
+
+- Smaller batch sizes (128) generally better for fine-tuning
+- Scale LR with `LR ∝ √batch_size`
+- Aim for at least 100 steps of training
+
+## Output Files
+
+After training, check `log_path` for:
+- `metrics.jsonl` - Training metrics
+- `checkpoints.jsonl` - Saved checkpoints
+- `config.json` - Training configuration
+
+```python
+import pandas
+df = pandas.read_json("/tmp/training/metrics.jsonl", lines=True)
+plt.plot(df['train_mean_nll'], label='train_loss')
+plt.plot(df['test/nll'].dropna(), label='test_loss')
+```