@synsci/cli-darwin-x64 1.1.55 → 1.1.57

package/bin/skills/tinker/SKILL.md

@@ -76,6 +76,7 @@ Use this for standard SFT with JSONL or HuggingFace datasets.
 ### Implementation
 
 ```python
+import json
 import chz
 import asyncio
 from tinker_cookbook.supervised import train
@@ -84,9 +85,12 @@ from tinker_cookbook.supervised.data import FromConversationFileBuilder
 from tinker_cookbook.renderers import TrainOnWhat
 from tinker_cookbook.model_info import get_recommended_renderer_name
 from tinker_cookbook.hyperparam_utils import get_lr
+from tinker_cookbook.tokenizer_utils import get_tokenizer
 
 model_name = "Qwen/Qwen3-30B-A3B"
 renderer_name = get_recommended_renderer_name(model_name)
+num_epochs = 3
+data_file = "training_data.jsonl"
 
 common_config = ChatDatasetBuilderCommonConfig(
     model_name_for_tokenizer=model_name,
@@ -98,7 +102,7 @@ common_config = ChatDatasetBuilderCommonConfig(
 
 dataset_builder = FromConversationFileBuilder(
     common_config=common_config,
-    file_path="training_data.jsonl",
+    file_path=data_file,
 )
 
 blueprint = chz.Blueprint(train.Config).apply({
@@ -107,12 +111,23 @@ blueprint = chz.Blueprint(train.Config).apply({
     "dataset_builder": dataset_builder,
     "learning_rate": get_lr(model_name),
     "lr_schedule": "linear",
-    "num_epochs": 3,
+    "num_epochs": num_epochs,
     "lora_rank": 32,
 })
 
 config = blueprint.make()
 asyncio.run(train.main(config))
+
+# --- Exact usage reporting (auto-captured by CLI) ---
+tokenizer = get_tokenizer(model_name)
+total_tokens = 0
+with open(data_file) as f:
+    for line in f:
+        row = json.loads(line)
+        text = " ".join(m.get("content", "") for m in row.get("messages", []))
+        total_tokens += len(tokenizer.encode(text))
+total_tokens *= num_epochs
+print(f'\n[SYNSCI_USAGE] {json.dumps({"service": "tinker", "event_type": "training", "model": model_name, "tokens_used": total_tokens})}')
 ```
 
 ### Data Format
@@ -145,6 +160,7 @@ Use this for training with reward functions — math reasoning, format complianc
 ### Implementation (Cookbook)
 
 ```python
+import json
 import asyncio
 import chz
 from tinker_cookbook.rl import train
@@ -153,10 +169,13 @@ from tinker_cookbook import model_info
 
 model_name = "meta-llama/Llama-3.1-8B"
 renderer_name = model_info.get_recommended_renderer_name(model_name)
+batch_size = 128
+group_size = 16
+max_tokens = 256
 
 builder = Gsm8kDatasetBuilder(
-    batch_size=128,
-    group_size=16,
+    batch_size=batch_size,
+    group_size=group_size,
     renderer_name=renderer_name,
     model_name_for_tokenizer=model_name,
 )
@@ -166,11 +185,27 @@ blueprint = chz.Blueprint(train.Config).apply({
     "log_path": "/tmp/rl-run",
     "dataset_builder": builder,
     "learning_rate": 4e-5,
-    "max_tokens": 256,
+    "max_tokens": max_tokens,
 })
 
 config = blueprint.make()
 asyncio.run(train.main(config))
+
+# --- Exact usage reporting (auto-captured by CLI) ---
+# For RL: estimate from batch_size × group_size × max_tokens × num_batches
+# The exact count comes from the training loop — check /tmp/rl-run for logs
+import glob, os
+log_files = sorted(glob.glob("/tmp/rl-run/metrics*.json"))
+total_tokens = 0
+for lf in log_files:
+    with open(lf) as f:
+        for line in f:
+            m = json.loads(line)
+            total_tokens += m.get("num_tokens", 0)
+if total_tokens == 0:
+    # Fallback estimate: batch_size × group_size × max_tokens × num_batches
+    total_tokens = batch_size * group_size * max_tokens * 100
+print(f'\n[SYNSCI_USAGE] {json.dumps({"service": "tinker", "event_type": "training", "model": model_name, "tokens_used": total_tokens})}')
 ```
 
 ### Custom RL with Low-Level API
@@ -178,16 +213,20 @@ asyncio.run(train.main(config))
 For full control over sampling, reward computation, and advantage centering:
 
 ```python
+import json
 import tinker
 from tinker import types
 from tinker.types.tensor_data import TensorData
 import torch
 
+model_name = "meta-llama/Llama-3.1-8B"
 service_client = tinker.ServiceClient()
 training_client = service_client.create_lora_training_client(
-    base_model="meta-llama/Llama-3.1-8B", rank=32
+    base_model=model_name, rank=32
 )
 
+total_tokens = 0  # Track exact tokens for billing
+
 for batch_idx, batch_rows in enumerate(dataset):
     path = training_client.save_weights_for_sampler(name=f"{batch_idx:06d}").result().path
     sampling_client = service_client.create_sampling_client(model_path=path)
@@ -220,9 +259,15 @@ for batch_idx, batch_rows in enumerate(dataset):
             )
             datums.append(datum)
 
+    # Track exact token count from datums
+    total_tokens += sum(d.model_input.length() for d in datums)
+
     fwd_bwd = training_client.forward_backward(datums, loss_fn="importance_sampling")
     optim = training_client.optim_step(types.AdamParams(learning_rate=4e-5))
     fwd_bwd.result(); optim.result()
+
+# --- Exact usage reporting (auto-captured by CLI) ---
+print(f'\n[SYNSCI_USAGE] {json.dumps({"service": "tinker", "event_type": "training", "model": model_name, "tokens_used": total_tokens})}')
 ```
 
 ### Available RL Loss Functions
@@ -342,30 +387,30 @@ See [Evaluations](references/evaluations.md) for custom evaluators and LLM-as-ju
 
 ### Pre-Training Cost Estimation
 
-**ALWAYS estimate cost before starting Tinker training.** Load the `tinker-training-cost` skill and use its pricing tables or the bundled calculator:
+**ALWAYS estimate cost before starting Tinker training.** Load the `tinker-training-cost` skill and use its pricing tables or calculate manually:
 
-```bash
-python scripts/calculate_cost.py training_data.jsonl --model Qwen3-8B --epochs 3 --json
+```
+Training Cost = (total_tokens × epochs × train_price_per_million) / 1,000,000
 ```
 
 Present the cost estimate to the user for approval before starting training.
 
-### Post-Training Usage Reporting
+### Automatic Usage Reporting (Ground Truth)
 
-After training completes, report usage for billing:
+**CRITICAL**: All training scripts MUST print a `[SYNSCI_USAGE]` line at the end. The CLI automatically captures this and reports exact billing to the dashboard.
 
-```typescript
-// Called automatically by the CLI after Tinker training
-await SynSci.reportUsage({
-  service: "tinker",
-  event_type: "training",
-  model: "Qwen3-8B",
-  tokens_used: 5000000,
-  metadata: { run_id: "my-run", epochs: 3, dataset: "training_data.jsonl" },
-})
+```python
+# Add this at the END of every training script:
+import json
+print(f'\n[SYNSCI_USAGE] {json.dumps({"service": "tinker", "event_type": "training", "model": model_name, "tokens_used": total_tokens})}')
 ```
 
-The CLI tracks all Tinker usage and reports it to the Synthetic Sciences dashboard for billing.
+How token counting works per workflow:
+- **Cookbook SFT**: Tokenize dataset with `get_tokenizer(model_name)`, multiply by `num_epochs`
+- **Cookbook RL**: Parse training logs for `num_tokens`, or estimate from `batch_size × group_size × max_tokens × batches`
+- **Low-level API**: Sum `datum.model_input.length()` across all `forward_backward()` calls
+
+The CLI bash tool scans output for `[SYNSCI_USAGE]` markers and auto-reports to the dashboard — no manual reporting needed.
 
 ## Common Issues
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@synsci/cli-darwin-x64",
-  "version": "1.1.55",
+  "version": "1.1.57",
   "os": [
     "darwin"
   ],