@myvillage/cli 2.3.0 → 2.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myvillage/cli",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "MyVillageOS CLI for community developers",
   "type": "module",
   "bin": {

package/src/commands/dataset.js

@@ -103,7 +103,7 @@ export async function datasetAppendCommand(options) {
       rows = readRowsFromFile(options.file);
     } else if (options.text !== undefined || options.features !== undefined) {
       const row = { split: options.split || 'train' };
-      row.features = options.features !== undefined ? parseFeatures(options.features) : options.text;
+      row.features = options.features !== undefined ? parseFeatures(options.features, manifest.modality) : options.text;
       if (options.label !== undefined) row.label = options.label;
       if (options.target !== undefined) row.target = Number(options.target);
       rows = [row];
@@ -111,15 +111,27 @@ export async function datasetAppendCommand(options) {
       throw new Error('Provide --image <path> (image datasets), --file <jsonl>, or --text/--features (with --label or --target).');
     }
 
+    if (options.replace && !options.file) {
+      throw new Error('--replace only works with --file (it swaps in the whole working set).');
+    }
+    const featureNames = options.featureNames !== undefined ? parseNameList(options.featureNames) : undefined;
+
     const spinner = options.json ? null : villageSpinner('Appending rows...').start();
-    const result = appendRows(slug, rows, { source: 'cli' });
-    if (spinner) spinner.stop();
+    let result;
+    try {
+      result = appendRows(slug, rows, { source: options.source || 'cli' }, {
+        replace: Boolean(options.replace),
+        featureNames,
+      });
+    } finally {
+      if (spinner) spinner.stop();
+    }
 
     if (options.json) {
       console.log(JSON.stringify(result, null, 2));
       return;
     }
-    success(`Appended ${result.appended} row(s) to ${slug}`);
+    success(`${result.replaced ? 'Replaced the working set with' : 'Appended'} ${result.appended} row(s) ${result.replaced ? 'in' : 'to'} ${slug}`);
     info(`  total: ${result.stats.rowCount} (train ${result.stats.bySplit.train} / val ${result.stats.bySplit.val})`);
   } catch (e) {
     error(e.message);
@@ -133,8 +145,13 @@ export async function datasetShowCommand(slug, options) {
   try {
     const manifest = readManifest(slug);
     if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
-    const limit = options.limit ? parseInt(options.limit, 10) : 5;
-    const sample = readRows(slug, { limit });
+    // --limit 0 returns every row (used by Studio's export).
+    const limit = options.limit !== undefined ? parseInt(options.limit, 10) : 5;
+    const version = options.frozen !== undefined ? parseInt(options.frozen, 10) : undefined;
+    if (version !== undefined && !manifest.versions.some((v) => v.version === version)) {
+      throw new Error(`Dataset "${slug}" has no frozen v${options.frozen}.`);
+    }
+    const sample = readRows(slug, { limit, version });
     if (options.json) {
       console.log(JSON.stringify({ manifest, sample }, null, 2));
       return;
@@ -206,7 +223,9 @@ export async function datasetRowUpdateCommand(options) {
     if (!slug) throw new Error('--dataset <slug> is required.');
     if (!options.id) throw new Error('--id <rowId> is required.');
     const updates = {};
-    if (options.features !== undefined) updates.features = parseFeatures(options.features);
+    if (options.features !== undefined) {
+      updates.features = parseFeatures(options.features, readManifest(slug)?.modality);
+    }
     else if (options.text !== undefined) updates.features = options.text;
     if (options.label !== undefined) updates.label = options.label;
     if (options.target !== undefined) updates.target = Number(options.target);
@@ -244,8 +263,22 @@ function readRowsFromFile(path) {
   return rows;
 }
 
-function parseFeatures(raw) {
-  // tabular features may be passed as a comma/JSON numeric vector
+/** A JSON array of names, or a comma-separated list. */
+function parseNameList(raw) {
+  const trimmed = String(raw).trim();
+  if (trimmed.startsWith('[')) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) throw new Error('--feature-names must be a list.');
+    return parsed.map(String);
+  }
+  return trimmed.split(',').map((s) => s.trim());
+}
+
+function parseFeatures(raw, modality) {
+  // Tabular features are a numeric vector, passed as JSON ("[12, 45]") or a
+  // comma list ("12, 45" or a single "12"). Anything that isn't a number stays
+  // NaN so row validation rejects it rather than storing it as text.
+  if (modality !== 'tabular') return raw;
   const trimmed = raw.trim();
   if (trimmed.startsWith('[')) {
     try {
@@ -254,10 +287,7 @@ function parseFeatures(raw) {
       /* fall through */
     }
   }
-  if (/^-?\d.*,/.test(trimmed)) {
-    return trimmed.split(',').map((n) => Number(n.trim()));
-  }
-  return raw;
+  return trimmed.split(',').map((n) => (n.trim() === '' ? NaN : Number(n.trim())));
 }
 
 function truncate(s, n) {

package/src/index.js

@@ -1010,13 +1010,17 @@ export function run() {
     .option('--label <label>', 'Class label (classification)')
     .option('--target <number>', 'Numeric target (regression)')
     .option('--split <split>', 'train | val', 'train')
+    .option('--replace', 'With --file: replace the working set instead of adding to it')
+    .option('--source <source>', 'Provenance source stamped on each row (e.g. import:csv)')
+    .option('--feature-names <list>', 'Tabular column names (JSON array or comma-separated)')
     .option('--json', 'Output raw JSON')
     .action(datasetAppendCommand);
 
   datasetCmd
     .command('show <slug>')
     .description('Show a dataset manifest and a sample of rows')
-    .option('--limit <n>', 'Number of sample rows to show', '5')
+    .option('--limit <n>', 'Number of sample rows to show (0 = all)', '5')
+    .option('--frozen <n>', 'Read frozen version n instead of the working set')
     .option('--json', 'Output raw JSON')
     .action(datasetShowCommand);
 

package/src/utils/dataset-workspace.js

@@ -202,23 +202,50 @@ export function createDataset({ name, modality, taskType = 'classification', lab
  * Append rows to a dataset's working set, stamping provenance, validating
  * against the dataset's modality/task, and updating manifest stats.
  *
+ * Every row is validated before anything is written, so a bad row leaves the
+ * dataset untouched. With `replace`, the working set is overwritten instead of
+ * extended (frozen versions are never touched) and the label list is rebuilt
+ * from the new rows. `featureNames` names the columns of a tabular vector
+ * (e.g. from a spreadsheet header); once set, every tabular row must have
+ * exactly that many numbers.
+ *
  * @param {string} slug
  * @param {Array<{features:any, label?:string, target?:number, split?:string}>} rows
  * @param {object} provenance  partial provenance ({ source, agentId, villagerId, villageId })
- * @returns {{datasetId:string, appended:number, stats:object}}
+ * @param {{replace?:boolean, featureNames?:string[]}} opts
+ * @returns {{datasetId:string, appended:number, replaced:boolean, stats:object}}
  */
-export function appendRows(slug, rows, provenance = {}) {
+export function appendRows(slug, rows, provenance = {}, { replace = false, featureNames } = {}) {
   const manifest = readManifest(slug);
   if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
   if (!Array.isArray(rows) || rows.length === 0) {
     throw new Error('No rows to append.');
   }
-  const source = provenance.source || 'cli';
-  const lines = [];
-  let appended = 0;
+  if (featureNames !== undefined) {
+    applyFeatureNames(manifest, featureNames, replace);
+  } else if (replace) {
+    delete manifest.featureNames;
+  }
 
-  for (const raw of rows) {
-    const row = normalizeRow(raw, manifest);
+  const existing = replace ? [] : readRows(slug);
+  // Without named columns, hold tabular rows to the width already on disk
+  // (the trainer needs every vector the same length).
+  let width = manifest.featureNames?.length;
+  if (width == null && manifest.modality === 'tabular') {
+    const first = existing.find((r) => Array.isArray(r.features));
+    if (first) width = first.features.length;
+  }
+
+  const source = provenance.source || 'cli';
+  const newRows = [];
+  rows.forEach((raw, i) => {
+    let row;
+    try {
+      row = normalizeRow(raw, manifest, width);
+    } catch (e) {
+      throw new Error(rows.length > 1 ? `Row ${i + 1}: ${e.message}` : e.message);
+    }
+    if (width == null && Array.isArray(row.features)) width = row.features.length;
     row.id = randomUUID();
     row.provenance = {
       source,
@@ -227,24 +254,47 @@ export function appendRows(slug, rows, provenance = {}) {
       ...(provenance.villageId ? { villageId: provenance.villageId } : {}),
       ts: nowISO(),
     };
-    lines.push(JSON.stringify(row));
+    newRows.push(row);
+  });
 
-    // stats
-    manifest.stats.rowCount += 1;
-    manifest.stats.bySplit[row.split] += 1;
-    if (row.label != null) {
-      manifest.stats.byLabel[row.label] = (manifest.stats.byLabel[row.label] || 0) + 1;
-      if (!manifest.labels.includes(row.label)) manifest.labels.push(row.label);
-    }
-    appended += 1;
+  if (replace) manifest.labels = [];
+  for (const row of newRows) {
+    if (row.label != null && !manifest.labels.includes(row.label)) manifest.labels.push(row.label);
+  }
+  if (replace) {
+    writeWorkingSet(slug, manifest, newRows);
+  } else {
+    appendFileSync(getCurrentDataFile(slug), newRows.map((r) => JSON.stringify(r)).join('\n') + '\n');
+    manifest.stats = recomputeStats([...existing, ...newRows]);
+    writeManifest(slug, manifest);
   }
+  return { datasetId: slug, appended: newRows.length, replaced: replace, stats: manifest.stats };
+}
 
-  appendFileSync(getCurrentDataFile(slug), lines.join('\n') + '\n');
-  writeManifest(slug, manifest);
-  return { datasetId: slug, appended, stats: manifest.stats };
+/** Swap the whole working set for `rows` (e.g. a spreadsheet import). */
+export function replaceRows(slug, rows, provenance = {}, { featureNames } = {}) {
+  return appendRows(slug, rows, provenance, { replace: true, featureNames });
+}
+
+/** Set a tabular dataset's column names, refusing a mismatch on append. */
+function applyFeatureNames(manifest, featureNames, replace) {
+  if (manifest.modality !== 'tabular') {
+    throw new Error('Feature names only apply to tabular datasets.');
+  }
+  const names = featureNames.map((n) => String(n).trim());
+  if (names.length === 0 || names.some((n) => !n)) {
+    throw new Error('Feature names must be non-empty.');
+  }
+  const current = manifest.featureNames;
+  if (!replace && current?.length && current.join('\u0000') !== names.join('\u0000')) {
+    throw new Error(
+      `These columns don't match the dataset. It expects: ${current.join(', ')}.`,
+    );
+  }
+  manifest.featureNames = names;
 }
 
-function normalizeRow(raw, manifest) {
+function normalizeRow(raw, manifest, width) {
   if (raw == null || typeof raw !== 'object') {
     throw new Error('Each row must be an object.');
   }
@@ -266,6 +316,18 @@ function normalizeRow(raw, manifest) {
       );
     }
   }
+  if (manifest.modality === 'tabular') {
+    const f = raw.features;
+    if (!Array.isArray(f) || f.length === 0 || f.some((n) => typeof n !== 'number' || !Number.isFinite(n))) {
+      throw new Error('Tabular rows need features that are all numbers (e.g. [12, 45, 8]).');
+    }
+    if (width != null && f.length !== width) {
+      throw new Error(`Expected ${width} number(s) in each row, got ${f.length}.`);
+    }
+  }
+  if (manifest.modality === 'text' && (typeof raw.features !== 'string' || !raw.features.trim())) {
+    throw new Error('Text rows need some text.');
+  }
   const row = { features: raw.features, split };
   if (manifest.taskType === 'classification') {
     if (raw.label == null || String(raw.label).length === 0) {
@@ -273,7 +335,7 @@ function normalizeRow(raw, manifest) {
     }
     row.label = String(raw.label);
   } else {
-    if (raw.target == null || Number.isNaN(Number(raw.target))) {
+    if (raw.target == null || raw.target === '' || !Number.isFinite(Number(raw.target))) {
       throw new Error('Regression rows require a numeric "target".');
     }
     row.target = Number(raw.target);
@@ -376,6 +438,7 @@ export function updateRow(slug, rowId, updates = {}) {
   if ((manifest.modality === 'image' || manifest.modality === 'audio') && updates.features !== undefined) {
     throw new Error(`Can't edit the asset of an ${manifest.modality} row — delete it and re-add.`);
   }
+  const sibling = rows.find((r, i) => i !== idx && Array.isArray(r.features));
   const merged = normalizeRow(
     {
       features: updates.features !== undefined ? updates.features : existing.features,
@@ -384,6 +447,7 @@ export function updateRow(slug, rowId, updates = {}) {
       target: updates.target !== undefined ? updates.target : existing.target,
     },
     manifest,
+    manifest.featureNames?.length ?? sibling?.features.length,
   );
   merged.id = existing.id;
   merged.provenance = { ...(existing.provenance || {}), editedAt: nowISO() };