@myvillage/cli 1.66.0 → 1.68.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myvillage/cli",
-  "version": "1.66.0",
+  "version": "1.68.0",
   "description": "MyVillageOS CLI for community developers",
   "type": "module",
   "bin": {

package/src/agent-runtime/builtin-tools.js

@@ -0,0 +1,77 @@
+/**
+ * Native built-in tools for the agent loop.
+ *
+ * Unlike MCP tools (loaded from tools.yaml via mcp-client.js), these run
+ * IN-PROCESS in the local daemon — no server, no subprocess, no socket.
+ * They're merged into the tool set passed to the LLM alongside MCP tools.
+ *
+ * `dataset_append` is the agent-facing "collect" input to the Studio Lab
+ * flywheel. It wraps appendRows() — the SAME execution engine the
+ * `myvillage dataset append` command uses — so agents and humans write
+ * identically. Raw data stays local; nothing leaves the machine.
+ *
+ * Tool contract mirrors @myvillage-studio/dataset-schema DATASET_APPEND_TOOL.
+ */
+import { tool } from 'ai';
+import { z } from 'zod';
+import { appendRows, datasetExists } from '../utils/dataset-workspace.js';
+
+/**
+ * Build the native built-in tools for an agent.
+ * @param {string} agentName
+ * @param {object} config  parsed agent.config.yaml
+ * @returns {Record<string, object>} AI SDK tools keyed by tool name
+ */
+export function getBuiltinTools(agentName, config) {
+  return {
+    dataset_append: tool({
+      description:
+        'Append labeled training example(s) to a LOCAL dataset on this machine. ' +
+        'The dataset must already exist (created via the Studio Lab or ' +
+        '`myvillage dataset create`). For classification provide `label`; for ' +
+        'regression provide a numeric `target`. Data stays local; provenance ' +
+        'is recorded automatically.',
+      inputSchema: z.object({
+        datasetId: z.string().describe('Slug of the local dataset to append to'),
+        rows: z
+          .array(
+            z.object({
+              features: z
+                .union([z.string(), z.array(z.number())])
+                .describe('Text content, or a numeric vector for tabular data'),
+              label: z.string().optional().describe('Class label (classification datasets)'),
+              target: z.number().optional().describe('Numeric target (regression datasets)'),
+              split: z.enum(['train', 'val']).optional(),
+            }),
+          )
+          .min(1)
+          .describe('One or more labeled examples to append'),
+      }),
+      execute: async ({ datasetId, rows }) => {
+        if (!datasetExists(datasetId)) {
+          return {
+            error:
+              `Dataset "${datasetId}" not found locally. Create it first ` +
+              '(Studio Lab or `myvillage dataset create`).',
+          };
+        }
+        try {
+          const provenance = { source: `agent:${agentName}` };
+          const agentId = config?.man?.village_agent_id;
+          if (agentId) provenance.agentId = agentId;
+          const result = appendRows(datasetId, rows, provenance);
+          return {
+            ok: true,
+            datasetId: result.datasetId,
+            appended: result.appended,
+            totalRows: result.stats.rowCount,
+          };
+        } catch (e) {
+          // Return the error to the model so it can self-correct rather than
+          // crashing the step.
+          return { error: e.message };
+        }
+      },
+    }),
+  };
+}

package/src/agent-runtime/loop.js

@@ -9,6 +9,7 @@ import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } fr
 import { join } from 'path';
 import { homedir } from 'os';
 import { getMCPTools, cleanupMCPClients } from './mcp-client.js';
+import { getBuiltinTools } from './builtin-tools.js';
 import { gatherContext } from './context.js';
 import { isWithinActiveHours, getNextCheckInMs } from './scheduler.js';
 import { parse as parseYaml } from 'yaml';
@@ -78,6 +79,8 @@ export async function agentLoop(agentName, { signal }) {
   try {
     const result = await getMCPTools(agentDir, config);
     tools = result.tools;
+    // Merge native built-in tools (in-process; no MCP server). e.g. dataset_append.
+    Object.assign(tools, getBuiltinTools(agentName, config));
     const servers = result.servers;
     const totalTools = Object.keys(tools).length;
     const failed = servers.filter(s => s.status === 'failed');
@@ -142,6 +145,7 @@ export async function agentLoop(agentName, { signal }) {
         await cleanupMCPClients();
         const refreshed = await getMCPTools(agentDir, config);
         tools = refreshed.tools;
+        Object.assign(tools, getBuiltinTools(agentName, config));
         logActivity(agentDir, { type: 'mcp_reconnected', reason: 'token rotated' });
       }
     } catch (err) {
@@ -571,6 +575,7 @@ const ACTION_TOOLS = new Set([
   'community_event_cancel',
   'moment_create',
   'pulse_create',
+  'dataset_append',
   'agent_join_community',
   'agent_leave_community',
   'wallet_send',

package/src/commands/dataset.js

@@ -0,0 +1,198 @@
+/**
+ * `myvillage dataset *` — manage local training datasets for the Studio Lab.
+ *
+ * All data stays under ~/.myvillage/datasets/ (local-only). These commands
+ * are the human-facing surface over src/utils/dataset-workspace.js; the same
+ * appendRows() execution function will back the dataset_append MCP tool so
+ * agents and the CLI write identically.
+ */
+import { existsSync, readFileSync } from 'fs';
+import { brand, success, error, info, header, villageSpinner } from '../utils/brand.js';
+import {
+  MODALITIES,
+  TASK_TYPES,
+  createDataset,
+  listDatasets,
+  readManifest,
+  datasetExists,
+  appendRows,
+  readRows,
+  freezeVersion,
+} from '../utils/dataset-workspace.js';
+
+// ── create ─────────────────────────────────────────────
+
+export async function datasetCreateCommand(options) {
+  try {
+    const labels = options.labels
+      ? options.labels.split(',').map((s) => s.trim()).filter(Boolean)
+      : [];
+    const manifest = createDataset({
+      name: options.name,
+      modality: options.modality,
+      taskType: options.task || 'classification',
+      labels,
+      description: options.description,
+    });
+    if (options.json) {
+      console.log(JSON.stringify(manifest, null, 2));
+      return;
+    }
+    success(`Created dataset "${manifest.name}" (${manifest.slug})`);
+    info(`  modality: ${manifest.modality} · task: ${manifest.taskType}`);
+    if (manifest.labels.length) info(`  labels: ${manifest.labels.join(', ')}`);
+    info(`  append rows with:  myvillage dataset append --dataset ${manifest.slug} ...`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── list ───────────────────────────────────────────────
+
+export async function datasetListCommand(options) {
+  try {
+    const datasets = listDatasets();
+    if (options.json) {
+      console.log(JSON.stringify(datasets, null, 2));
+      return;
+    }
+    if (datasets.length === 0) {
+      info('No local datasets yet. Create one with: myvillage dataset create --name "..." --modality text');
+      return;
+    }
+    header('Local datasets');
+    for (const d of datasets) {
+      const ver = d.currentVersion ? `v${d.currentVersion}` : 'unfrozen';
+      console.log(
+        `  ${brand.gold(d.slug.padEnd(28))} ${String(d.modality).padEnd(8)} ${String(d.taskType).padEnd(15)} ${String(d.stats.rowCount).padStart(6)} rows  ${ver}`,
+      );
+    }
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── append ─────────────────────────────────────────────
+
+export async function datasetAppendCommand(options) {
+  try {
+    const slug = options.dataset;
+    if (!slug) throw new Error('--dataset <slug> is required.');
+    if (!datasetExists(slug)) throw new Error(`Dataset "${slug}" not found.`);
+
+    let rows;
+    if (options.file) {
+      rows = readRowsFromFile(options.file);
+    } else if (options.text !== undefined || options.features !== undefined) {
+      const row = { split: options.split || 'train' };
+      row.features = options.features !== undefined ? parseFeatures(options.features) : options.text;
+      if (options.label !== undefined) row.label = options.label;
+      if (options.target !== undefined) row.target = Number(options.target);
+      rows = [row];
+    } else {
+      throw new Error('Provide either --file <jsonl> or --text/--features (with --label or --target).');
+    }
+
+    const spinner = options.json ? null : villageSpinner('Appending rows...').start();
+    const result = appendRows(slug, rows, { source: 'cli' });
+    if (spinner) spinner.stop();
+
+    if (options.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    success(`Appended ${result.appended} row(s) to ${slug}`);
+    info(`  total: ${result.stats.rowCount} (train ${result.stats.bySplit.train} / val ${result.stats.bySplit.val})`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── show ───────────────────────────────────────────────
+
+export async function datasetShowCommand(slug, options) {
+  try {
+    const manifest = readManifest(slug);
+    if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+    const limit = options.limit ? parseInt(options.limit, 10) : 5;
+    const sample = readRows(slug, { limit });
+    if (options.json) {
+      console.log(JSON.stringify({ manifest, sample }, null, 2));
+      return;
+    }
+    header(`${manifest.name} (${manifest.slug})`);
+    info(`  modality: ${manifest.modality} · task: ${manifest.taskType}`);
+    if (manifest.labels.length) info(`  labels: ${manifest.labels.join(', ')}`);
+    info(`  rows: ${manifest.stats.rowCount} (train ${manifest.stats.bySplit.train} / val ${manifest.stats.bySplit.val})`);
+    info(`  versions: ${manifest.versions.length ? manifest.versions.map((v) => `v${v.version}`).join(', ') : '(none frozen)'}`);
+    if (sample.length) {
+      console.log(brand.cream(`\n  sample (${sample.length}):`));
+      for (const r of sample) {
+        const feat = typeof r.features === 'string' ? truncate(r.features, 50) : JSON.stringify(r.features);
+        const tgt = r.label != null ? `label=${r.label}` : `target=${r.target}`;
+        console.log(`    [${r.split}] ${tgt}  ${brand.teal(feat)}`);
+      }
+    }
+    console.log('');
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── freeze ─────────────────────────────────────────────
+
+export async function datasetFreezeCommand(slug, options) {
+  try {
+    const meta = freezeVersion(slug, options.changelog);
+    if (options.json) {
+      console.log(JSON.stringify(meta, null, 2));
+      return;
+    }
+    success(`Froze ${slug} as v${meta.version} (${meta.rowCount} rows)`);
+  } catch (e) {
+    error(e.message);
+    process.exitCode = 1;
+  }
+}
+
+// ── helpers ────────────────────────────────────────────
+
+function readRowsFromFile(path) {
+  if (!existsSync(path)) throw new Error(`File not found: ${path}`);
+  const rows = [];
+  for (const line of readFileSync(path, 'utf-8').split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      rows.push(JSON.parse(line));
+    } catch {
+      throw new Error(`Invalid JSONL line in ${path}: ${truncate(line, 60)}`);
+    }
+  }
+  if (rows.length === 0) throw new Error(`No rows found in ${path}`);
+  return rows;
+}
+
+function parseFeatures(raw) {
+  // tabular features may be passed as a comma/JSON numeric vector
+  const trimmed = raw.trim();
+  if (trimmed.startsWith('[')) {
+    try {
+      return JSON.parse(trimmed);
+    } catch {
+      /* fall through */
+    }
+  }
+  if (/^-?\d.*,/.test(trimmed)) {
+    return trimmed.split(',').map((n) => Number(n.trim()));
+  }
+  return raw;
+}
+
+function truncate(s, n) {
+  return s.length > n ? s.slice(0, n - 1) + '…' : s;
+}

package/src/index.js

@@ -118,6 +118,13 @@ import {
   soulprintPublishCommand,
   soulprintModelsCommand,
 } from './commands/soulprint.js';
+import {
+  datasetCreateCommand,
+  datasetListCommand,
+  datasetAppendCommand,
+  datasetShowCommand,
+  datasetFreezeCommand,
+} from './commands/dataset.js';
 
 const require = createRequire(import.meta.url);
 const pkg = require('../package.json');
@@ -833,5 +840,58 @@ export function run() {
     .option('--public', 'Make the model publicly available')
     .action(soulprintPublishCommand);
 
+  // ── Studio Lab: local training datasets ────────────────
+  // Raw data stays under ~/.myvillage/datasets/ (local-only). appendRows()
+  // in dataset-workspace.js is the single execution engine the dataset_append
+  // MCP tool will also wrap.
+
+  const datasetCmd = program
+    .command('dataset')
+    .description('Manage local training datasets for the Studio Lab (data stays on your machine)');
+
+  datasetCmd
+    .command('create')
+    .description('Create a new local dataset')
+    .requiredOption('--name <name>', 'Human-readable dataset name')
+    .requiredOption('--modality <modality>', 'text | image | audio | tabular')
+    .option('--task <task>', 'classification | regression', 'classification')
+    .option('--labels <list>', 'Comma-separated class labels (classification)')
+    .option('--description <text>', 'Optional description')
+    .option('--json', 'Output raw JSON')
+    .action(datasetCreateCommand);
+
+  datasetCmd
+    .command('list')
+    .description('List local datasets')
+    .option('--json', 'Output raw JSON')
+    .action(datasetListCommand);
+
+  datasetCmd
+    .command('append')
+    .description('Append labeled example(s) to a dataset')
+    .requiredOption('--dataset <slug>', 'Target dataset slug')
+    .option('--file <path>', 'JSONL file of rows ({features,label|target,split})')
+    .option('--text <text>', 'Single-row text features')
+    .option('--features <value>', 'Single-row features (string, or numeric vector for tabular)')
+    .option('--label <label>', 'Class label (classification)')
+    .option('--target <number>', 'Numeric target (regression)')
+    .option('--split <split>', 'train | val', 'train')
+    .option('--json', 'Output raw JSON')
+    .action(datasetAppendCommand);
+
+  datasetCmd
+    .command('show <slug>')
+    .description('Show a dataset manifest and a sample of rows')
+    .option('--limit <n>', 'Number of sample rows to show', '5')
+    .option('--json', 'Output raw JSON')
+    .action(datasetShowCommand);
+
+  datasetCmd
+    .command('freeze <slug>')
+    .description('Freeze the working set as the next immutable version')
+    .option('--changelog <text>', 'What changed in this version')
+    .option('--json', 'Output raw JSON')
+    .action(datasetFreezeCommand);
+
   program.parse();
 }

package/src/utils/dataset-workspace.js

@@ -0,0 +1,284 @@
+/**
+ * Local dataset store for the Studio "Lab" (Data & Models) pillar.
+ *
+ * Raw training data is LOCAL-ONLY — it lives under ~/.myvillage/datasets/
+ * and never leaves the machine. Only metadata syncs to the backend (later).
+ *
+ * This module is the single execution engine for dataset writes: both the
+ * `myvillage dataset *` commands and the future `dataset_append` MCP server
+ * call `appendRows()` here, so the behavior can never fork.
+ *
+ * Shapes mirror @myvillage-studio/dataset-schema (the canonical source).
+ *
+ * Layout:
+ *   ~/.myvillage/datasets/<slug>/
+ *     manifest.json     # name, modality, task, labels, versions, stats
+ *     current.jsonl     # working set (mutable until frozen)
+ *     v1.jsonl, v2...    # frozen snapshots
+ *     assets/           # local image/audio files (referenced by rows)
+ */
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  readdirSync,
+  copyFileSync,
+} from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+import { getConfigDir } from './config.js';
+
+const DATASETS_DIR_NAME = 'datasets';
+export const DATASET_SCHEMA_VERSION = '1.0.0';
+export const MODALITIES = ['text', 'image', 'audio', 'tabular'];
+export const TASK_TYPES = ['classification', 'regression'];
+export const SPLITS = ['train', 'val'];
+
+// ── Directory helpers ──────────────────────────────────
+
+export function getDatasetsRootDir() {
+  const dir = join(getConfigDir(), DATASETS_DIR_NAME);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export function getDatasetDir(slug) {
+  return join(getDatasetsRootDir(), slug);
+}
+
+export function getAssetsDir(slug) {
+  const dir = join(getDatasetDir(slug), 'assets');
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function getManifestPath(slug) {
+  return join(getDatasetDir(slug), 'manifest.json');
+}
+
+export function getCurrentDataFile(slug) {
+  return join(getDatasetDir(slug), 'current.jsonl');
+}
+
+export function getVersionFile(slug, version) {
+  return join(getDatasetDir(slug), `v${version}.jsonl`);
+}
+
+export function datasetExists(slug) {
+  return existsSync(getManifestPath(slug));
+}
+
+// ── Manifest read/write ────────────────────────────────
+
+export function readManifest(slug) {
+  const p = getManifestPath(slug);
+  if (!existsSync(p)) return null;
+  try {
+    return JSON.parse(readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeManifest(slug, manifest) {
+  manifest.updatedAt = nowISO();
+  writeFileSync(getManifestPath(slug), JSON.stringify(manifest, null, 2));
+  return manifest;
+}
+
+export function listDatasets() {
+  const root = getDatasetsRootDir();
+  return readdirSync(root, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => readManifest(e.name))
+    .filter(Boolean);
+}
+
+// ── Create ─────────────────────────────────────────────
+
+export function slugify(name) {
+  return String(name)
+    .toLowerCase()
+    .trim()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 64);
+}
+
+function nowISO() {
+  return new Date().toISOString();
+}
+
+function emptyStats() {
+  return { rowCount: 0, bySplit: { train: 0, val: 0 }, byLabel: {} };
+}
+
+/**
+ * Create a new local dataset. Returns the manifest.
+ * @param {{name:string, modality:string, taskType?:string, labels?:string[], description?:string}} opts
+ */
+export function createDataset({ name, modality, taskType = 'classification', labels = [], description }) {
+  if (!name) throw new Error('A dataset name is required.');
+  if (!MODALITIES.includes(modality)) {
+    throw new Error(`modality must be one of: ${MODALITIES.join(', ')}`);
+  }
+  if (!TASK_TYPES.includes(taskType)) {
+    throw new Error(`taskType must be one of: ${TASK_TYPES.join(', ')}`);
+  }
+  const slug = slugify(name);
+  if (!slug) throw new Error('Could not derive a slug from the name.');
+  if (datasetExists(slug)) {
+    throw new Error(`A dataset with slug "${slug}" already exists.`);
+  }
+  mkdirSync(getDatasetDir(slug), { recursive: true });
+  // touch the working set
+  writeFileSync(getCurrentDataFile(slug), '');
+  const ts = nowISO();
+  const manifest = {
+    schemaVersion: DATASET_SCHEMA_VERSION,
+    id: randomUUID(),
+    slug,
+    name,
+    description: description || undefined,
+    modality,
+    taskType,
+    labels: [...labels],
+    splitConfig: { trainRatio: 0.8 },
+    currentVersion: 0,
+    versions: [],
+    stats: emptyStats(),
+    createdAt: ts,
+    updatedAt: ts,
+  };
+  return writeManifest(slug, manifest);
+}
+
+// ── Append (THE execution engine — CLI + future MCP both call this) ─────
+
+/**
+ * Append rows to a dataset's working set, stamping provenance, validating
+ * against the dataset's modality/task, and updating manifest stats.
+ *
+ * @param {string} slug
+ * @param {Array<{features:any, label?:string, target?:number, split?:string}>} rows
+ * @param {object} provenance  partial provenance ({ source, agentId, villagerId, villageId })
+ * @returns {{datasetId:string, appended:number, stats:object}}
+ */
+export function appendRows(slug, rows, provenance = {}) {
+  const manifest = readManifest(slug);
+  if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+  if (!Array.isArray(rows) || rows.length === 0) {
+    throw new Error('No rows to append.');
+  }
+  const source = provenance.source || 'cli';
+  const lines = [];
+  let appended = 0;
+
+  for (const raw of rows) {
+    const row = normalizeRow(raw, manifest);
+    row.id = randomUUID();
+    row.provenance = {
+      source,
+      ...(provenance.agentId ? { agentId: provenance.agentId } : {}),
+      ...(provenance.villagerId ? { villagerId: provenance.villagerId } : {}),
+      ...(provenance.villageId ? { villageId: provenance.villageId } : {}),
+      ts: nowISO(),
+    };
+    lines.push(JSON.stringify(row));
+
+    // stats
+    manifest.stats.rowCount += 1;
+    manifest.stats.bySplit[row.split] += 1;
+    if (row.label != null) {
+      manifest.stats.byLabel[row.label] = (manifest.stats.byLabel[row.label] || 0) + 1;
+      if (!manifest.labels.includes(row.label)) manifest.labels.push(row.label);
+    }
+    appended += 1;
+  }
+
+  appendFileSync(getCurrentDataFile(slug), lines.join('\n') + '\n');
+  writeManifest(slug, manifest);
+  return { datasetId: slug, appended, stats: manifest.stats };
+}
+
+function normalizeRow(raw, manifest) {
+  if (raw == null || typeof raw !== 'object') {
+    throw new Error('Each row must be an object.');
+  }
+  const split = raw.split || 'train';
+  if (!SPLITS.includes(split)) {
+    throw new Error(`split must be one of: ${SPLITS.join(', ')}`);
+  }
+  if (raw.features === undefined || raw.features === null) {
+    throw new Error('Each row needs a "features" value.');
+  }
+  const row = { features: raw.features, split };
+  if (manifest.taskType === 'classification') {
+    if (raw.label == null || String(raw.label).length === 0) {
+      throw new Error('Classification rows require a "label".');
+    }
+    row.label = String(raw.label);
+  } else {
+    if (raw.target == null || Number.isNaN(Number(raw.target))) {
+      throw new Error('Regression rows require a numeric "target".');
+    }
+    row.target = Number(raw.target);
+  }
+  return row;
+}
+
+// ── Read ───────────────────────────────────────────────
+
+/**
+ * Read rows from a dataset. Defaults to the working set (current.jsonl).
+ * @param {string} slug
+ * @param {{version?:number, split?:string, limit?:number}} opts
+ */
+export function readRows(slug, { version, split, limit } = {}) {
+  if (!datasetExists(slug)) throw new Error(`Dataset "${slug}" not found.`);
+  const file = version ? getVersionFile(slug, version) : getCurrentDataFile(slug);
+  if (!existsSync(file)) return [];
+  const out = [];
+  for (const line of readFileSync(file, 'utf-8').split('\n')) {
+    if (!line.trim()) continue;
+    let row;
+    try {
+      row = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (split && row.split !== split) continue;
+    out.push(row);
+    if (limit && out.length >= limit) break;
+  }
+  return out;
+}
+
+// ── Freeze a version ───────────────────────────────────
+
+/**
+ * Snapshot the working set as the next immutable version.
+ * @returns {object} the new DatasetVersionMeta
+ */
+export function freezeVersion(slug, changelog) {
+  const manifest = readManifest(slug);
+  if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
+  if (manifest.stats.rowCount === 0) {
+    throw new Error('Cannot freeze an empty dataset.');
+  }
+  const version = (manifest.currentVersion || 0) + 1;
+  copyFileSync(getCurrentDataFile(slug), getVersionFile(slug, version));
+  const meta = {
+    version,
+    file: `v${version}.jsonl`,
+    rowCount: manifest.stats.rowCount,
+    frozenAt: nowISO(),
+    ...(changelog ? { changelog } : {}),
+  };
+  manifest.currentVersion = version;
+  manifest.versions.push(meta);
+  writeManifest(slug, manifest);
+  return meta;
+}