@myvillage/cli 1.66.0 → 1.69.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myvillage/cli",
-  "version": "1.66.0",
+  "version": "1.69.0",
   "description": "MyVillageOS CLI for community developers",
   "type": "module",
   "bin": {
@@ -30,6 +30,7 @@
     "@ai-sdk/anthropic": "^3.0.78",
     "@ai-sdk/mcp": "^1.0.42",
     "@ai-sdk/openai": "^3.0.33",
+    "@tensorflow/tfjs": "^4.22.0",
     "ai": "^6.0.0",
     "axios": "^1.6.2",
     "chalk": "^5.3.0",

package/src/agent-runtime/builtin-tools.js

@@ -0,0 +1,77 @@
+/**
+ * Native built-in tools for the agent loop.
+ *
+ * Unlike MCP tools (loaded from tools.yaml via mcp-client.js), these run
+ * IN-PROCESS in the local daemon — no server, no subprocess, no socket.
+ * They're merged into the tool set passed to the LLM alongside MCP tools.
+ *
+ * `dataset_append` is the agent-facing "collect" input to the Studio Lab
+ * flywheel. It wraps appendRows() — the SAME execution engine the
+ * `myvillage dataset append` command uses — so agents and humans write
+ * identically. Raw data stays local; nothing leaves the machine.
+ *
+ * Tool contract mirrors @myvillage-studio/dataset-schema DATASET_APPEND_TOOL.
+ */
+import { tool } from 'ai';
+import { z } from 'zod';
+import { appendRows, datasetExists } from '../utils/dataset-workspace.js';
+
+/**
+ * Build the native built-in tools for an agent.
+ * @param {string} agentName
+ * @param {object} config  parsed agent.config.yaml
+ * @returns {Record<string, object>} AI SDK tools keyed by tool name
+ */
+export function getBuiltinTools(agentName, config) {
+  return {
+    dataset_append: tool({
+      description:
+        'Append labeled training example(s) to a LOCAL dataset on this machine. ' +
+        'The dataset must already exist (created via the Studio Lab or ' +
+        '`myvillage dataset create`). For classification provide `label`; for ' +
+        'regression provide a numeric `target`. Data stays local; provenance ' +
+        'is recorded automatically.',
+      inputSchema: z.object({
+        datasetId: z.string().describe('Slug of the local dataset to append to'),
+        rows: z
+          .array(
+            z.object({
+              features: z
+                .union([z.string(), z.array(z.number())])
+                .describe('Text content, or a numeric vector for tabular data'),
+              label: z.string().optional().describe('Class label (classification datasets)'),
+              target: z.number().optional().describe('Numeric target (regression datasets)'),
+              split: z.enum(['train', 'val']).optional(),
+            }),
+          )
+          .min(1)
+          .describe('One or more labeled examples to append'),
+      }),
+      execute: async ({ datasetId, rows }) => {
+        if (!datasetExists(datasetId)) {
+          return {
+            error:
+              `Dataset "${datasetId}" not found locally. Create it first ` +
+              '(Studio Lab or `myvillage dataset create`).',
+          };
+        }
+        try {
+          const provenance = { source: `agent:${agentName}` };
+          const agentId = config?.man?.village_agent_id;
+          if (agentId) provenance.agentId = agentId;
+          const result = appendRows(datasetId, rows, provenance);
+          return {
+            ok: true,
+            datasetId: result.datasetId,
+            appended: result.appended,
+            totalRows: result.stats.rowCount,
+          };
+        } catch (e) {
+          // Return the error to the model so it can self-correct rather than
+          // crashing the step.
+          return { error: e.message };
+        }
+      },
+    }),
+  };
+}

package/src/agent-runtime/loop.js

@@ -9,6 +9,7 @@ import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } fr
 import { join } from 'path';
 import { homedir } from 'os';
 import { getMCPTools, cleanupMCPClients } from './mcp-client.js';
+import { getBuiltinTools } from './builtin-tools.js';
 import { gatherContext } from './context.js';
 import { isWithinActiveHours, getNextCheckInMs } from './scheduler.js';
 import { parse as parseYaml } from 'yaml';
@@ -78,6 +79,8 @@ export async function agentLoop(agentName, { signal }) {
   try {
     const result = await getMCPTools(agentDir, config);
     tools = result.tools;
+    // Merge native built-in tools (in-process; no MCP server). e.g. dataset_append.
+    Object.assign(tools, getBuiltinTools(agentName, config));
     const servers = result.servers;
     const totalTools = Object.keys(tools).length;
     const failed = servers.filter(s => s.status === 'failed');
@@ -142,6 +145,7 @@ export async function agentLoop(agentName, { signal }) {
         await cleanupMCPClients();
         const refreshed = await getMCPTools(agentDir, config);
         tools = refreshed.tools;
+        Object.assign(tools, getBuiltinTools(agentName, config));
         logActivity(agentDir, { type: 'mcp_reconnected', reason: 'token rotated' });
       }
     } catch (err) {
@@ -571,6 +575,7 @@ const ACTION_TOOLS = new Set([
   'community_event_cancel',
   'moment_create',
   'pulse_create',
+  'dataset_append',
   'agent_join_community',
   'agent_leave_community',
   'wallet_send',

package/src/commands/dataset.js

@@ -0,0 +1,198 @@
+/**
+ * `myvillage dataset *` — manage local training datasets for the Studio Lab.
+ *
+ * All data stays under ~/.myvillage/datasets/ (local-only). These commands
+ * are the human-facing surface over src/utils/dataset-workspace.js; the same
+ * appendRows() execution function will back the dataset_append MCP tool so
+ * agents and the CLI write identically.
+ */
+import { existsSync, readFileSync } from 'fs';
+import { brand, success, error, info, header, villageSpinner } from '../utils/brand.js';
+import {
+  MODALITIES,
+  TASK_TYPES,
+  createDataset,
+  listDatasets,
+  readManifest,
+  datasetExists,
+  appendRows,
+  readRows,
+  freezeVersion,
+} from '../utils/dataset-workspace.js';
+
+// ── create ─────────────────────────────────────────────
+
+export async function datasetCreateCommand(options) {
+  try {
+    const labels = options.labels
+      ? options.labels.split(',').map((s) => s.trim()).filter(Boolean)
+      : [];
+    const manifest = createDataset({
+      name: options.name,
+      modality: options.modality,
+      taskType: options.task || 'classification',
+      labels,
+      description: options.description,
+    });
+    if (options.json) {
+      console.log(JSON.stringify(manifest, null, 2));
+      return;
+    }
+    success(`Created dataset "${manifest.name}" (${manifest.slug})`);
+    info(`  modality: ${manifest.modality} · task: ${manifest.taskType}`);
+    if (manifest.labels.length) info(`  labels: ${manifest.labels.join(', ')}`);
+    info(`  append rows with:  myvillage dataset append --dataset ${manifest.slug} ...`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── list ───────────────────────────────────────────────
+
+export async function datasetListCommand(options) {
+  try {
+    const datasets = listDatasets();
+    if (options.json) {
+      console.log(JSON.stringify(datasets, null, 2));
+      return;
+    }
+    if (datasets.length === 0) {
+      info('No local datasets yet. Create one with: myvillage dataset create --name "..." --modality text');
+      return;
+    }
+    header('Local datasets');
+    for (const d of datasets) {
+      const ver = d.currentVersion ? `v${d.currentVersion}` : 'unfrozen';
+      console.log(
+        `  ${brand.gold(d.slug.padEnd(28))} ${String(d.modality).padEnd(8)} ${String(d.taskType).padEnd(15)} ${String(d.stats.rowCount).padStart(6)} rows  ${ver}`,
+      );
+    }
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── append ─────────────────────────────────────────────
+
+export async function datasetAppendCommand(options) {
+  try {
+    const slug = options.dataset;
+    if (!slug) throw new Error('--dataset <slug> is required.');
+    if (!datasetExists(slug)) throw new Error(`Dataset "${slug}" not found.`);
+
+    let rows;
+    if (options.file) {
+      rows = readRowsFromFile(options.file);
+    } else if (options.text !== undefined || options.features !== undefined) {
+      const row = { split: options.split || 'train' };
+      row.features = options.features !== undefined ? parseFeatures(options.features) : options.text;
+      if (options.label !== undefined) row.label = options.label;
+      if (options.target !== undefined) row.target = Number(options.target);
+      rows = [row];
+    } else {
+      throw new Error('Provide either --file <jsonl> or --text/--features (with --label or --target).');
+    }
+
+    const spinner = options.json ? null : villageSpinner('Appending rows...').start();
+    const result = appendRows(slug, rows, { source: 'cli' });
+    if (spinner) spinner.stop();
+
+    if (options.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    success(`Appended ${result.appended} row(s) to ${slug}`);
+    info(`  total: ${result.stats.rowCount} (train ${result.stats.bySplit.train} / val ${result.stats.bySplit.val})`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── show ───────────────────────────────────────────────
+
+export async function datasetShowCommand(slug, options) {
+  try {
+    const manifest = readManifest(slug);
+    if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+    const limit = options.limit ? parseInt(options.limit, 10) : 5;
+    const sample = readRows(slug, { limit });
+    if (options.json) {
+      console.log(JSON.stringify({ manifest, sample }, null, 2));
+      return;
+    }
+    header(`${manifest.name} (${manifest.slug})`);
+    info(`  modality: ${manifest.modality} · task: ${manifest.taskType}`);
+    if (manifest.labels.length) info(`  labels: ${manifest.labels.join(', ')}`);
+    info(`  rows: ${manifest.stats.rowCount} (train ${manifest.stats.bySplit.train} / val ${manifest.stats.bySplit.val})`);
+    info(`  versions: ${manifest.versions.length ? manifest.versions.map((v) => `v${v.version}`).join(', ') : '(none frozen)'}`);
+    if (sample.length) {
+      console.log(brand.cream(`\n  sample (${sample.length}):`));
+      for (const r of sample) {
+        const feat = typeof r.features === 'string' ? truncate(r.features, 50) : JSON.stringify(r.features);
+        const tgt = r.label != null ? `label=${r.label}` : `target=${r.target}`;
+        console.log(`    [${r.split}] ${tgt}  ${brand.teal(feat)}`);
+      }
+    }
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── freeze ─────────────────────────────────────────────
+
+export async function datasetFreezeCommand(slug, options) {
+  try {
+    const meta = freezeVersion(slug, options.changelog);
+    if (options.json) {
+      console.log(JSON.stringify(meta, null, 2));
+      return;
+    }
+    success(`Froze ${slug} as v${meta.version} (${meta.rowCount} rows)`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── helpers ────────────────────────────────────────────
+
+function readRowsFromFile(path) {
+  if (!existsSync(path)) throw new Error(`File not found: ${path}`);
+  const rows = [];
+  for (const line of readFileSync(path, 'utf-8').split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      rows.push(JSON.parse(line));
+    } catch {
+      throw new Error(`Invalid JSONL line in ${path}: ${truncate(line, 60)}`);
+    }
+  }
+  if (rows.length === 0) throw new Error(`No rows found in ${path}`);
+  return rows;
+}
+
+function parseFeatures(raw) {
+  // tabular features may be passed as a comma/JSON numeric vector
+  const trimmed = raw.trim();
+  if (trimmed.startsWith('[')) {
+    try {
+      return JSON.parse(trimmed);
+    } catch {
+      /* fall through */
+    }
+  }
+  if (/^-?\d.*,/.test(trimmed)) {
+    return trimmed.split(',').map((n) => Number(n.trim()));
+  }
+  return raw;
+}
+
+function truncate(s, n) {
+  return s.length > n ? s.slice(0, n - 1) + '…' : s;
+}

package/src/commands/model.js

@@ -0,0 +1,143 @@
+/**
+ * `myvillage model *` — train and use local models for the Studio Lab.
+ *
+ * Training runs in-process on TF.js (CPU). This is the single training engine;
+ * the Studio drives these same commands and streams the progress they emit
+ * (with --json, one progress line per epoch, then a final result object).
+ */
+import { brand, success, error, info, header, villageSpinner } from '../utils/brand.js';
+import { readManifest as readDatasetManifest, datasetExists } from '../utils/dataset-workspace.js';
+import { trainModel, predictWithModel, TRAINABLE_MODALITIES } from '../utils/model-trainer.js';
+import { saveModel, listModels, readModelManifest, modelExists, loadModel } from '../utils/model-workspace.js';
+
+// ── train ──────────────────────────────────────────────
+
+export async function modelTrainCommand(options) {
+  try {
+    const datasetSlug = options.dataset;
+    if (!datasetSlug) throw new Error('--dataset <slug> is required.');
+    if (!datasetExists(datasetSlug)) throw new Error(`Dataset "${datasetSlug}" not found.`);
+    const ds = readDatasetManifest(datasetSlug);
+    if (!TRAINABLE_MODALITIES.includes(ds.modality)) {
+      throw new Error(
+        `Training "${ds.modality}" datasets is coming soon — v1 supports ${TRAINABLE_MODALITIES.join(' and ')}.`,
+      );
+    }
+    const name = options.name || `${ds.name} model`;
+    const epochs = options.epochs ? parseInt(options.epochs, 10) : 40;
+
+    // Progress: stream JSON lines when --json (Studio parses these); else spinner.
+    const spinner = options.json ? null : villageSpinner('Training…').start();
+    const onProgress = (p) => {
+      if (options.json) {
+        process.stdout.write(JSON.stringify({ type: 'progress', ...p }) + '\n');
+      } else {
+        const metric =
+          p.valAccuracy != null ? `val acc ${(p.valAccuracy * 100).toFixed(1)}%`
+          : p.valMae != null ? `val MAE ${p.valMae}`
+          : `loss ${p.loss}`;
+        spinner.text = `Training… epoch ${p.epoch}/${p.totalEpochs} · ${metric}`;
+      }
+    };
+
+    const { model, record } = await trainModel(datasetSlug, ds, { epochs, onProgress });
+    const manifest = await saveModel(name, {
+      model,
+      record,
+      datasetSlug,
+      datasetVersion: ds.currentVersion || 0,
+    });
+    if (spinner) spinner.stop();
+
+    if (options.json) {
+      process.stdout.write(JSON.stringify({ type: 'result', model: manifest }) + '\n');
+      return;
+    }
+    success(`Trained "${manifest.name}" (${manifest.slug}) from ${record.examplesUsed} examples`);
+    const m = record.metrics;
+    if (m.accuracy != null) info(`  accuracy: ${(m.accuracy * 100).toFixed(1)}% over ${m.classes} classes`);
+    if (m.mae != null) info(`  MAE: ${m.mae} · R²: ${m.r2}`);
+    info(`  test it:  myvillage model predict ${manifest.slug} --input "..."`);
+  } catch (e) {
+    if (options.json) process.stdout.write(JSON.stringify({ type: 'error', error: e.message }) + '\n');
+    else error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── list ───────────────────────────────────────────────
+
+export async function modelListCommand(options) {
+  try {
+    const models = listModels();
+    if (options.json) {
+      console.log(JSON.stringify(models, null, 2));
+      return;
+    }
+    if (models.length === 0) {
+      info('No models yet. Train one with: myvillage model train --dataset <slug>');
+      return;
+    }
+    header('Local models');
+    for (const m of models) {
+      const score =
+        m.metrics?.accuracy != null ? `${(m.metrics.accuracy * 100).toFixed(0)}% acc`
+        : m.metrics?.mae != null ? `MAE ${m.metrics.mae}`
+        : '—';
+      console.log(`  ${brand.gold(m.slug.padEnd(28))} ${String(m.taskType).padEnd(15)} ${String(score).padStart(10)}  ← ${m.datasetSlug}`);
+    }
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── show ───────────────────────────────────────────────
+
+export async function modelShowCommand(slug, options) {
+  try {
+    const m = readModelManifest(slug);
+    if (!m) throw new Error(`Model "${slug}" not found.`);
+    if (options.json) {
+      console.log(JSON.stringify(m, null, 2));
+      return;
+    }
+    header(`${m.name} (${m.slug})`);
+    info(`  task: ${m.taskType} · modality: ${m.modality} · trained on: ${m.datasetSlug} (v${m.datasetVersion})`);
+    info(`  examples: ${m.examplesUsed} · format: ${m.format}`);
+    if (m.metrics?.accuracy != null) info(`  accuracy: ${(m.metrics.accuracy * 100).toFixed(1)}% over ${m.metrics.classes} classes`);
+    if (m.metrics?.mae != null) info(`  MAE: ${m.metrics.mae} · R²: ${m.metrics.r2}`);
+    if (m.labels?.length) info(`  labels: ${m.labels.join(', ')}`);
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── predict ────────────────────────────────────────────
+
+export async function modelPredictCommand(slug, options) {
+  try {
+    if (!modelExists(slug)) throw new Error(`Model "${slug}" not found.`);
+    if (options.input === undefined) throw new Error('--input <value> is required.');
+    const record = readModelManifest(slug);
+    const model = await loadModel(slug);
+    const result = predictWithModel(model, record, options.input);
+    if (options.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    if (result.label != null) {
+      success(`Prediction: ${result.label}`);
+      const ranked = Object.entries(result.scores).sort((a, b) => b[1] - a[1]);
+      for (const [label, score] of ranked) info(`  ${label}: ${(score * 100).toFixed(1)}%`);
+    } else {
+      success(`Prediction: ${result.score}`);
+    }
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}

package/src/index.js

@@ -118,6 +118,19 @@ import {
   soulprintPublishCommand,
   soulprintModelsCommand,
 } from './commands/soulprint.js';
+import {
+  datasetCreateCommand,
+  datasetListCommand,
+  datasetAppendCommand,
+  datasetShowCommand,
+  datasetFreezeCommand,
+} from './commands/dataset.js';
+import {
+  modelTrainCommand,
+  modelListCommand,
+  modelShowCommand,
+  modelPredictCommand,
+} from './commands/model.js';
 
 const require = createRequire(import.meta.url);
 const pkg = require('../package.json');
@@ -833,5 +846,91 @@ export function run() {
     .option('--public', 'Make the model publicly available')
     .action(soulprintPublishCommand);
 
+  // ── Studio Lab: local training datasets ────────────────
+  // Raw data stays under ~/.myvillage/datasets/ (local-only). appendRows()
+  // in dataset-workspace.js is the single execution engine the dataset_append
+  // MCP tool will also wrap.
+
+  const datasetCmd = program
+    .command('dataset')
+    .description('Manage local training datasets for the Studio Lab (data stays on your machine)');
+
+  datasetCmd
+    .command('create')
+    .description('Create a new local dataset')
+    .requiredOption('--name <name>', 'Human-readable dataset name')
+    .requiredOption('--modality <modality>', 'text | image | audio | tabular')
+    .option('--task <task>', 'classification | regression', 'classification')
+    .option('--labels <list>', 'Comma-separated class labels (classification)')
+    .option('--description <text>', 'Optional description')
+    .option('--json', 'Output raw JSON')
+    .action(datasetCreateCommand);
+
+  datasetCmd
+    .command('list')
+    .description('List local datasets')
+    .option('--json', 'Output raw JSON')
+    .action(datasetListCommand);
+
+  datasetCmd
+    .command('append')
+    .description('Append labeled example(s) to a dataset')
+    .requiredOption('--dataset <slug>', 'Target dataset slug')
+    .option('--file <path>', 'JSONL file of rows ({features,label|target,split})')
+    .option('--text <text>', 'Single-row text features')
+    .option('--features <value>', 'Single-row features (string, or numeric vector for tabular)')
+    .option('--label <label>', 'Class label (classification)')
+    .option('--target <number>', 'Numeric target (regression)')
+    .option('--split <split>', 'train | val', 'train')
+    .option('--json', 'Output raw JSON')
+    .action(datasetAppendCommand);
+
+  datasetCmd
+    .command('show <slug>')
+    .description('Show a dataset manifest and a sample of rows')
+    .option('--limit <n>', 'Number of sample rows to show', '5')
+    .option('--json', 'Output raw JSON')
+    .action(datasetShowCommand);
+
+  datasetCmd
+    .command('freeze <slug>')
+    .description('Freeze the working set as the next immutable version')
+    .option('--changelog <text>', 'What changed in this version')
+    .option('--json', 'Output raw JSON')
+    .action(datasetFreezeCommand);
+
+  // ── Studio Lab: model training (local, TF.js CPU engine) ───────────────
+  const modelCmd = program
+    .command('model')
+    .description('Train and use local models for the Studio Lab (runs on your machine)');
+
+  modelCmd
+    .command('train')
+    .description('Train a model from a local dataset (tabular or text in v1)')
+    .requiredOption('--dataset <slug>', 'Dataset slug to train on')
+    .option('--name <name>', 'Name for the resulting model')
+    .option('--epochs <n>', 'Training epochs (default 40)')
+    .option('--json', 'Stream progress as JSON lines, then a final result object')
+    .action(modelTrainCommand);
+
+  modelCmd
+    .command('list')
+    .description('List local trained models')
+    .option('--json', 'Output raw JSON')
+    .action(modelListCommand);
+
+  modelCmd
+    .command('show <slug>')
+    .description('Show a model: task, metrics, training dataset')
+    .option('--json', 'Output raw JSON')
+    .action(modelShowCommand);
+
+  modelCmd
+    .command('predict <slug>')
+    .description('Run a prediction with a trained model')
+    .requiredOption('--input <value>', 'Input: text, or a comma-separated numeric vector for tabular')
+    .option('--json', 'Output raw JSON')
+    .action(modelPredictCommand);
+
   program.parse();
 }

package/src/utils/dataset-workspace.js

@@ -0,0 +1,284 @@
+/**
+ * Local dataset store for the Studio "Lab" (Data & Models) pillar.
+ *
+ * Raw training data is LOCAL-ONLY — it lives under ~/.myvillage/datasets/
+ * and never leaves the machine. Only metadata syncs to the backend (later).
+ *
+ * This module is the single execution engine for dataset writes: both the
+ * `myvillage dataset *` commands and the future `dataset_append` MCP server
+ * call `appendRows()` here, so the behavior can never fork.
+ *
+ * Shapes mirror @myvillage-studio/dataset-schema (the canonical source).
+ *
+ * Layout:
+ *   ~/.myvillage/datasets/<slug>/
+ *     manifest.json     # name, modality, task, labels, versions, stats
+ *     current.jsonl     # working set (mutable until frozen)
+ *     v1.jsonl, v2...    # frozen snapshots
+ *     assets/           # local image/audio files (referenced by rows)
+ */
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  readdirSync,
+  copyFileSync,
+} from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+import { getConfigDir } from './config.js';
+
+const DATASETS_DIR_NAME = 'datasets';
+export const DATASET_SCHEMA_VERSION = '1.0.0';
+export const MODALITIES = ['text', 'image', 'audio', 'tabular'];
+export const TASK_TYPES = ['classification', 'regression'];
+export const SPLITS = ['train', 'val'];
+
+// ── Directory helpers ──────────────────────────────────
+
+export function getDatasetsRootDir() {
+  const dir = join(getConfigDir(), DATASETS_DIR_NAME);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export function getDatasetDir(slug) {
+  return join(getDatasetsRootDir(), slug);
+}
+
+export function getAssetsDir(slug) {
+  const dir = join(getDatasetDir(slug), 'assets');
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function getManifestPath(slug) {
+  return join(getDatasetDir(slug), 'manifest.json');
+}
+
+export function getCurrentDataFile(slug) {
+  return join(getDatasetDir(slug), 'current.jsonl');
+}
+
+export function getVersionFile(slug, version) {
+  return join(getDatasetDir(slug), `v${version}.jsonl`);
+}
+
+export function datasetExists(slug) {
+  return existsSync(getManifestPath(slug));
+}
+
+// ── Manifest read/write ────────────────────────────────
+
+export function readManifest(slug) {
+  const p = getManifestPath(slug);
+  if (!existsSync(p)) return null;
+  try {
+    return JSON.parse(readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeManifest(slug, manifest) {
+  manifest.updatedAt = nowISO();
+  writeFileSync(getManifestPath(slug), JSON.stringify(manifest, null, 2));
+  return manifest;
+}
+
+export function listDatasets() {
+  const root = getDatasetsRootDir();
+  return readdirSync(root, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => readManifest(e.name))
+    .filter(Boolean);
+}
+
+// ── Create ─────────────────────────────────────────────
+
+export function slugify(name) {
+  return String(name)
+    .toLowerCase()
+    .trim()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 64);
+}
+
+function nowISO() {
+  return new Date().toISOString();
+}
+
+function emptyStats() {
+  return { rowCount: 0, bySplit: { train: 0, val: 0 }, byLabel: {} };
+}
+
+/**
+ * Create a new local dataset. Returns the manifest.
+ * @param {{name:string, modality:string, taskType?:string, labels?:string[], description?:string}} opts
+ */
+export function createDataset({ name, modality, taskType = 'classification', labels = [], description }) {
+  if (!name) throw new Error('A dataset name is required.');
+  if (!MODALITIES.includes(modality)) {
+    throw new Error(`modality must be one of: ${MODALITIES.join(', ')}`);
+  }
+  if (!TASK_TYPES.includes(taskType)) {
+    throw new Error(`taskType must be one of: ${TASK_TYPES.join(', ')}`);
+  }
+  const slug = slugify(name);
+  if (!slug) throw new Error('Could not derive a slug from the name.');
+  if (datasetExists(slug)) {
+    throw new Error(`A dataset with slug "${slug}" already exists.`);
+  }
+  mkdirSync(getDatasetDir(slug), { recursive: true });
+  // touch the working set
+  writeFileSync(getCurrentDataFile(slug), '');
+  const ts = nowISO();
+  const manifest = {
+    schemaVersion: DATASET_SCHEMA_VERSION,
+    id: randomUUID(),
+    slug,
+    name,
+    description: description || undefined,
+    modality,
+    taskType,
+    labels: [...labels],
+    splitConfig: { trainRatio: 0.8 },
+    currentVersion: 0,
+    versions: [],
+    stats: emptyStats(),
+    createdAt: ts,
+    updatedAt: ts,
+  };
+  return writeManifest(slug, manifest);
+}
+
+// ── Append (THE execution engine — CLI + future MCP both call this) ─────
+
+/**
+ * Append rows to a dataset's working set, stamping provenance, validating
+ * against the dataset's modality/task, and updating manifest stats.
+ *
+ * @param {string} slug
+ * @param {Array<{features:any, label?:string, target?:number, split?:string}>} rows
+ * @param {object} provenance  partial provenance ({ source, agentId, villagerId, villageId })
+ * @returns {{datasetId:string, appended:number, stats:object}}
+ */
+export function appendRows(slug, rows, provenance = {}) {
+  const manifest = readManifest(slug);
+  if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+  if (!Array.isArray(rows) || rows.length === 0) {
+    throw new Error('No rows to append.');
+  }
+  const source = provenance.source || 'cli';
+  const lines = [];
+  let appended = 0;
+
+  for (const raw of rows) {
+    const row = normalizeRow(raw, manifest);
+    row.id = randomUUID();
+    row.provenance = {
+      source,
+      ...(provenance.agentId ? { agentId: provenance.agentId } : {}),
+      ...(provenance.villagerId ? { villagerId: provenance.villagerId } : {}),
+      ...(provenance.villageId ? { villageId: provenance.villageId } : {}),
+      ts: nowISO(),
+    };
+    lines.push(JSON.stringify(row));
+
+    // stats
+    manifest.stats.rowCount += 1;
+    manifest.stats.bySplit[row.split] += 1;
+    if (row.label != null) {
+      manifest.stats.byLabel[row.label] = (manifest.stats.byLabel[row.label] || 0) + 1;
+      if (!manifest.labels.includes(row.label)) manifest.labels.push(row.label);
+    }
+    appended += 1;
+  }
+
+  appendFileSync(getCurrentDataFile(slug), lines.join('\n') + '\n');
+  writeManifest(slug, manifest);
+  return { datasetId: slug, appended, stats: manifest.stats };
+}
+
+function normalizeRow(raw, manifest) {
+  if (raw == null || typeof raw !== 'object') {
+    throw new Error('Each row must be an object.');
+  }
+  const split = raw.split || 'train';
+  if (!SPLITS.includes(split)) {
+    throw new Error(`split must be one of: ${SPLITS.join(', ')}`);
+  }
+  if (raw.features === undefined || raw.features === null) {
+    throw new Error('Each row needs a "features" value.');
+  }
+  const row = { features: raw.features, split };
+  if (manifest.taskType === 'classification') {
+    if (raw.label == null || String(raw.label).length === 0) {
+      throw new Error('Classification rows require a "label".');
+    }
+    row.label = String(raw.label);
+  } else {
+    if (raw.target == null || Number.isNaN(Number(raw.target))) {
+      throw new Error('Regression rows require a numeric "target".');
+    }
+    row.target = Number(raw.target);
+  }
+  return row;
+}
+
+// ── Read ───────────────────────────────────────────────
+
+/**
+ * Read rows from a dataset. Defaults to the working set (current.jsonl).
+ * @param {string} slug
+ * @param {{version?:number, split?:string, limit?:number}} opts
+ */
+export function readRows(slug, { version, split, limit } = {}) {
+  if (!datasetExists(slug)) throw new Error(`Dataset "${slug}" not found.`);
+  const file = version ? getVersionFile(slug, version) : getCurrentDataFile(slug);
+  if (!existsSync(file)) return [];
+  const out = [];
+  for (const line of readFileSync(file, 'utf-8').split('\n')) {
+    if (!line.trim()) continue;
+    let row;
+    try {
+      row = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (split && row.split !== split) continue;
+    out.push(row);
+    if (limit && out.length >= limit) break;
+  }
+  return out;
+}
+
+// ── Freeze a version ───────────────────────────────────
+
+/**
+ * Snapshot the working set as the next immutable version.
+ * @returns {object} the new DatasetVersionMeta
+ */
+export function freezeVersion(slug, changelog) {
+  const manifest = readManifest(slug);
+  if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+  if (manifest.stats.rowCount === 0) {
+    throw new Error('Cannot freeze an empty dataset.');
+  }
+  const version = (manifest.currentVersion || 0) + 1;
+  copyFileSync(getCurrentDataFile(slug), getVersionFile(slug, version));
+  const meta = {
+    version,
+    file: `v${version}.jsonl`,
+    rowCount: manifest.stats.rowCount,
+    frozenAt: nowISO(),
+    ...(changelog ? { changelog } : {}),
+  };
+  manifest.currentVersion = version;
+  manifest.versions.push(meta);
+  writeManifest(slug, manifest);
+  return meta;
+}

package/src/utils/model-trainer.js

@@ -0,0 +1,249 @@
+/**
+ * Local model trainer for the Studio "Lab" pillar.
+ *
+ * Runs IN NODE on TensorFlow.js (CPU backend) — no GPU, no native binary, no
+ * downloads — so it works headless from `myvillage model train` AND when the
+ * Studio shells out to that same command. This is the single training engine;
+ * the Studio is just a front-door that streams the progress it emits.
+ *
+ * v1 modalities (zero downloaded backbones):
+ *   - tabular: numeric feature vectors → standardized → dense head
+ *   - text:    hashing vectorizer (pure JS) → dense head
+ * image/audio need a bundled vision/audio backbone and land in a later slice.
+ *
+ * Transfer-learning-style split: featurize once, then train a small head. The
+ * featurization params travel in the model record so predict() needs no data.
+ */
+import * as tf from '@tensorflow/tfjs';
+import { readRows } from './dataset-workspace.js';
+
+const TEXT_DIM = 1024;
+export const TRAINABLE_MODALITIES = ['tabular', 'text'];
+
+// ── Featurization ──────────────────────────────────────
+
+function tokenize(s) {
+  return String(s).toLowerCase().match(/[a-z0-9]+/g) || [];
+}
+
+// FNV-1a → bucket. Stable across runs/machines (no Math.random).
+function hashToken(t) {
+  let h = 2166136261 >>> 0;
+  for (let i = 0; i < t.length; i++) {
+    h ^= t.charCodeAt(i);
+    h = Math.imul(h, 16777619) >>> 0;
+  }
+  return h >>> 0;
+}
+
+function vectorizeText(s) {
+  const v = new Array(TEXT_DIM).fill(0);
+  for (const tok of tokenize(s)) v[hashToken(tok) % TEXT_DIM] += 1;
+  let norm = 0;
+  for (let i = 0; i < TEXT_DIM; i++) norm += v[i] * v[i];
+  norm = Math.sqrt(norm);
+  if (norm > 0) for (let i = 0; i < TEXT_DIM; i++) v[i] /= norm;
+  return v;
+}
+
+function toNumericVector(features) {
+  if (Array.isArray(features)) return features.map(Number);
+  if (typeof features === 'string') {
+    return features.split(',').map((n) => Number(n.trim()));
+  }
+  throw new Error('Tabular features must be a numeric vector.');
+}
+
+/** Build the feature matrix + the featurization params to persist. */
+function featurize(rows, modality) {
+  if (modality === 'text') {
+    return { X: rows.map((r) => vectorizeText(r.features)), featurization: { modality, dim: TEXT_DIM } };
+  }
+  // tabular: standardize each column (store mean/std for predict)
+  const raw = rows.map((r) => toNumericVector(r.features));
+  const dim = raw[0].length;
+  if (raw.some((v) => v.length !== dim)) {
+    throw new Error('All tabular rows must have the same number of features.');
+  }
+  const mean = new Array(dim).fill(0);
+  const std = new Array(dim).fill(0);
+  for (const v of raw) for (let j = 0; j < dim; j++) mean[j] += v[j];
+  for (let j = 0; j < dim; j++) mean[j] /= raw.length;
+  for (const v of raw) for (let j = 0; j < dim; j++) std[j] += (v[j] - mean[j]) ** 2;
+  for (let j = 0; j < dim; j++) std[j] = Math.sqrt(std[j] / raw.length) || 1;
+  const X = raw.map((v) => v.map((x, j) => (x - mean[j]) / std[j]));
+  return { X, featurization: { modality, dim, mean, std } };
+}
+
+/** Featurize a single input the same way (for predict). */
+export function featurizeOne(input, featurization) {
+  if (featurization.modality === 'text') return vectorizeText(input);
+  const v = toNumericVector(input);
+  return v.map((x, j) => (x - featurization.mean[j]) / (featurization.std[j] || 1));
+}
+
+// ── Model ──────────────────────────────────────────────
+
+function buildModel(inputDim, task, numClasses) {
+  const model = tf.sequential();
+  model.add(tf.layers.dense({ inputShape: [inputDim], units: 32, activation: 'relu' }));
+  model.add(tf.layers.dropout({ rate: 0.2 }));
+  if (task === 'classification') {
+    model.add(tf.layers.dense({ units: numClasses, activation: 'softmax' }));
+    // one-hot targets + categoricalCrossentropy (avoids int32/sparse dtype traps)
+    model.compile({ optimizer: tf.train.adam(0.01), loss: 'categoricalCrossentropy', metrics: ['accuracy'] });
+  } else {
+    model.add(tf.layers.dense({ units: 1 }));
+    model.compile({ optimizer: tf.train.adam(0.01), loss: 'meanSquaredError', metrics: ['mae'] });
+  }
+  return model;
+}
+
+// ── Train ──────────────────────────────────────────────
+
+/**
+ * Train a model from a local dataset's working set.
+ * @returns {{ model: tf.LayersModel, record: object }} record holds everything
+ *          predict() needs (featurization, labels/target stats, metrics).
+ */
+export async function trainModel(slug, manifest, { epochs = 40, onProgress } = {}) {
+  await tf.setBackend('cpu');
+  await tf.ready();
+
+  if (!TRAINABLE_MODALITIES.includes(manifest.modality)) {
+    throw new Error(
+      `Training for "${manifest.modality}" datasets is coming soon — v1 supports ${TRAINABLE_MODALITIES.join(' and ')}.`,
+    );
+  }
+  const rows = readRows(slug, {});
+  if (rows.length < 4) throw new Error('Need at least 4 examples to train (more is better).');
+
+  const task = manifest.taskType;
+  const { X, featurization } = featurize(rows, manifest.modality);
+  const inputDim = X[0].length;
+
+  // Build targets: one-hot rows (classification) or standardized scalars (regression)
+  let labels = [], targetMean = null, targetStd = null, numClasses = 0;
+  let Y;
+  if (task === 'classification') {
+    labels = [...new Set(rows.map((r) => String(r.label)))].sort();
+    if (labels.length < 2) throw new Error('Classification needs at least 2 distinct labels.');
+    const idx = new Map(labels.map((l, i) => [l, i]));
+    numClasses = labels.length;
+    Y = rows.map((r) => {
+      const o = new Array(numClasses).fill(0);
+      o[idx.get(String(r.label))] = 1;
+      return o;
+    });
+  } else {
+    const t = rows.map((r) => Number(r.target));
+    targetMean = t.reduce((a, b) => a + b, 0) / t.length;
+    targetStd = Math.sqrt(t.reduce((a, b) => a + (b - targetMean) ** 2, 0) / t.length) || 1;
+    Y = t.map((v) => (v - targetMean) / targetStd);
+  }
+
+  // train/val split in plain JS (explicit val rows, else hold out the last ~20%)
+  const explicitVal = rows.map((r, i) => (r.split === 'val' ? i : -1)).filter((i) => i >= 0);
+  let valSet;
+  if (explicitVal.length >= 2 && explicitVal.length < rows.length) {
+    valSet = new Set(explicitVal);
+  } else {
+    const n = Math.max(1, Math.floor(rows.length * 0.2));
+    valSet = new Set(rows.map((_, i) => i).slice(rows.length - n));
+  }
+  const trI = [], vaI = [];
+  rows.forEach((_, i) => (valSet.has(i) ? vaI : trI).push(i));
+  if (trI.length === 0) { trI.push(...vaI); vaI.length = 0; }
+
+  const pick = (arr, idxs) => idxs.map((i) => arr[i]);
+  const mkY = (vals) => (task === 'classification' ? tf.tensor2d(vals) : tf.tensor1d(vals));
+  const trainXs = tf.tensor2d(pick(X, trI));
+  const trainYs = mkY(pick(Y, trI));
+  const valXs = vaI.length ? tf.tensor2d(pick(X, vaI)) : null;
+  const valYs = vaI.length ? mkY(pick(Y, vaI)) : null;
+
+  const model = buildModel(inputDim, task, numClasses);
+  const fitConfig = {
+    epochs,
+    shuffle: true,
+    callbacks: {
+      onEpochEnd: (epoch, logs) => {
+        onProgress?.({
+          epoch: epoch + 1,
+          totalEpochs: epochs,
+          loss: round(logs.loss),
+          valLoss: round(logs.val_loss),
+          accuracy: round(logs.acc ?? logs.accuracy),
+          valAccuracy: round(logs.val_acc ?? logs.val_accuracy),
+          mae: round(logs.mae),
+          valMae: round(logs.val_mae),
+        });
+      },
+    },
+  };
+  if (valXs) fitConfig.validationData = [valXs, valYs];
+  await model.fit(trainXs, trainYs, fitConfig);
+
+  // Metrics over the eval set (val if we have one, else the training set).
+  const evalXs = valXs || trainXs;
+  const evalYs = valYs || trainYs;
+  const metrics = computeMetrics(model, evalXs, evalYs, task, labels, targetMean, targetStd);
+  trainXs.dispose(); trainYs.dispose();
+  if (valXs) { valXs.dispose(); valYs.dispose(); }
+
+  const record = {
+    modality: manifest.modality,
+    taskType: task,
+    inputDim,
+    featurization,
+    labels: labels ?? [],
+    targetMean: targetMean ?? null,
+    targetStd: targetStd ?? null,
+    metrics,
+    hyperparams: { epochs, optimizer: 'adam', learningRate: 0.01 },
+    examplesUsed: rows.length,
+  };
+  return { model, record };
+}
+
+function computeMetrics(model, xs, ys, task, labels, targetMean, targetStd) {
+  return tf.tidy(() => {
+    const pred = model.predict(xs);
+    if (task === 'classification') {
+      const yhat = pred.argMax(1);
+      const ytrue = ys.argMax(1); // ys is one-hot
+      const acc = yhat.equal(ytrue).mean().arraySync();
+      return { accuracy: round(acc), classes: labels.length };
+    }
+    // regression: de-standardize, compute MAE + R²
+    const yhat = pred.flatten().mul(targetStd).add(targetMean);
+    const ytrue = ys.mul(targetStd).add(targetMean);
+    const mae = yhat.sub(ytrue).abs().mean().arraySync();
+    const ssRes = yhat.sub(ytrue).square().sum().arraySync();
+    const yMean = ytrue.mean().arraySync();
+    const ssTot = ytrue.sub(yMean).square().sum().arraySync();
+    return { mae: round(mae), r2: round(ssTot > 0 ? 1 - ssRes / ssTot : 0) };
+  });
+}
+
+/** Run a prediction with an already-loaded model + its record. */
+export function predictWithModel(model, record, input) {
+  return tf.tidy(() => {
+    const feats = featurizeOne(input, record.featurization);
+    const out = model.predict(tf.tensor2d([feats]));
+    if (record.taskType === 'classification') {
+      const scores = Array.from(out.dataSync());
+      let best = 0;
+      for (let i = 1; i < scores.length; i++) if (scores[i] > scores[best]) best = i;
+      const byLabel = {};
+      record.labels.forEach((l, i) => (byLabel[l] = round(scores[i])));
+      return { label: record.labels[best], scores: byLabel };
+    }
+    const z = out.dataSync()[0];
+    return { score: round(z * record.targetStd + record.targetMean) };
+  });
+}
+
+function round(n) {
+  return n == null || Number.isNaN(n) ? undefined : Math.round(n * 1e4) / 1e4;
+}

package/src/utils/model-workspace.js

@@ -0,0 +1,113 @@
+/**
+ * Local trained-model store for the Studio "Lab" pillar.
+ *
+ * Models live under ~/.myvillage/models/<slug>/ — local-only, like datasets.
+ *   manifest.json   # TrainedModelRecord (modality, task, metrics, featurization…)
+ *   model.json      # tfjs topology + weight specs
+ *   weights.b64     # base64 weight data
+ *
+ * Save/load go through TF.js's in-memory IO handlers (no tfjs-node needed),
+ * so this stays pure-JS and matches the trainer's CPU runtime.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+import * as tf from '@tensorflow/tfjs';
+import { getConfigDir } from './config.js';
+import { slugify } from './dataset-workspace.js';
+
+const MODELS_DIR_NAME = 'models';
+export const MODEL_SCHEMA_VERSION = '1.0.0';
+
+export function getModelsRootDir() {
+  const dir = join(getConfigDir(), MODELS_DIR_NAME);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export function getModelDir(slug) {
+  return join(getModelsRootDir(), slug);
+}
+
+function manifestPath(slug) {
+  return join(getModelDir(slug), 'manifest.json');
+}
+
+export function modelExists(slug) {
+  return existsSync(manifestPath(slug));
+}
+
+export function readModelManifest(slug) {
+  const p = manifestPath(slug);
+  if (!existsSync(p)) return null;
+  try {
+    return JSON.parse(readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+export function listModels() {
+  const root = getModelsRootDir();
+  return readdirSync(root, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => readModelManifest(e.name))
+    .filter(Boolean);
+}
+
+export function deleteModel(slug) {
+  rmSync(getModelDir(slug), { recursive: true, force: true });
+}
+
+/**
+ * Persist a trained model + its record.
+ * @param {string} name human-readable model name (slug derived from it)
+ * @param {object} opts { model: tf.LayersModel, record, datasetSlug, datasetVersion }
+ * @returns {object} the saved manifest
+ */
+export async function saveModel(name, { model, record, datasetSlug, datasetVersion }) {
+  const slug = slugify(name);
+  if (!slug) throw new Error('Could not derive a slug from the model name.');
+  const dir = getModelDir(slug);
+  mkdirSync(dir, { recursive: true });
+
+  let modelTopology, weightSpecs, weightB64;
+  await model.save(
+    tf.io.withSaveHandler(async (artifacts) => {
+      modelTopology = artifacts.modelTopology;
+      weightSpecs = artifacts.weightSpecs;
+      weightB64 = Buffer.from(artifacts.weightData).toString('base64');
+      return { modelArtifactsInfo: { dateSaved: new Date(), modelTopologyType: 'JSON' } };
+    }),
+  );
+  writeFileSync(join(dir, 'model.json'), JSON.stringify({ modelTopology, weightSpecs }));
+  writeFileSync(join(dir, 'weights.b64'), weightB64);
+
+  const ts = new Date().toISOString();
+  const manifest = {
+    schemaVersion: MODEL_SCHEMA_VERSION,
+    id: randomUUID(),
+    slug,
+    name,
+    datasetSlug,
+    datasetVersion: datasetVersion ?? 0,
+    format: 'tfjs',
+    artifacts: ['model.json', 'weights.b64'],
+    ...record,
+    status: 'ready',
+    createdAt: ts,
+    updatedAt: ts,
+  };
+  writeFileSync(manifestPath(slug), JSON.stringify(manifest, null, 2));
+  return manifest;
+}
+
+/** Load a saved tf model back into memory (for predict). */
+export async function loadModel(slug) {
+  const dir = getModelDir(slug);
+  if (!existsSync(join(dir, 'model.json'))) throw new Error(`Model "${slug}" not found.`);
+  const { modelTopology, weightSpecs } = JSON.parse(readFileSync(join(dir, 'model.json'), 'utf-8'));
+  const buf = Buffer.from(readFileSync(join(dir, 'weights.b64'), 'utf-8'), 'base64');
+  const weightData = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+  return tf.loadLayersModel(tf.io.fromMemory({ modelTopology, weightSpecs, weightData }));
+}