@myvillage/cli 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@myvillage/cli",
3
- "version": "2.4.0",
3
+ "version": "2.6.0",
4
4
  "description": "MyVillageOS CLI for community developers",
5
5
  "type": "module",
6
6
  "bin": {
@@ -14,16 +14,19 @@
14
14
  export const DATASET_APPEND_DESCRIPTION =
15
15
  'Append labeled training example(s) to a LOCAL dataset on this machine. ' +
16
16
  'The dataset must already exist (created via the Studio Lab or ' +
17
- '`myvillage dataset create`). `features` is text, a numeric vector ' +
18
- '(tabular), or a local image file path (image datasets — the file is ' +
19
- 'copied into the dataset). For classification provide `label`; for ' +
17
+ '`myvillage dataset create`). `features` is text (words datasets); for a ' +
18
+ 'table dataset, its values in column order or an object keyed by column ' +
19
+ 'name (numbers, category text, true/false, short notes); or a local image ' +
20
+ 'file path (image datasets) or .wav file path (audio datasets) — the file is ' +
21
+ 'copied into the dataset. For classification provide `label`; for ' +
20
22
  'regression provide a numeric `target`. Data stays local; provenance ' +
21
23
  'is recorded automatically.';
22
24
 
23
25
  export const MODEL_PREDICT_DESCRIPTION =
24
26
  'Run a prediction with a LOCAL trained model on this machine. `input` is ' +
25
- 'text, a numeric vector (tabular), or a local image file path (image ' +
26
- 'models). Returns the predicted class with per-class confidence scores ' +
27
+ 'text (words models), the values of one table row as a list in column ' +
28
+ 'order or an object keyed by column name (table models), a local image file path (image ' +
29
+ 'models), or a local .wav file path (sound models). Returns the predicted class with per-class confidence scores ' +
27
30
  '(classification) or a numeric value (regression). The model must already ' +
28
31
  'exist (trained in the Studio Lab or via `myvillage model train`).';
29
32
 
@@ -65,10 +65,16 @@ export function getBuiltinTools(agentName, config) {
65
65
  .array(
66
66
  z.object({
67
67
  features: z
68
- .union([z.string(), z.array(z.number())])
68
+ .union([
69
+ z.string(),
70
+ z.array(z.union([z.number(), z.string(), z.boolean()])),
71
+ z.record(z.union([z.number(), z.string(), z.boolean()])),
72
+ ])
69
73
  .describe(
70
- 'Text content; a numeric vector for tabular data; or, for image ' +
71
- 'datasets, an absolute path to a local image file (.jpg/.png).',
74
+ 'Text content for a words dataset; for a table dataset, its values in column ' +
75
+ 'order or an object keyed by column name (numbers, category text, true/false, ' +
76
+ 'short notes); or, for image datasets, an absolute path to a local image file (.jpg/.png); ' +
77
+ 'or, for audio datasets, an absolute path to a local .wav file.',
72
78
  ),
73
79
  label: z.string().optional().describe('Class label (classification datasets)'),
74
80
  target: z.number().optional().describe('Numeric target (regression datasets)'),
@@ -90,16 +96,21 @@ export function getBuiltinTools(agentName, config) {
90
96
  const provenance = { source: `agent:${agentName}` };
91
97
  const agentId = config?.man?.village_agent_id;
92
98
  if (agentId) provenance.agentId = agentId;
93
- // For image datasets, the agent passes a file path; ingest each into
94
- // the dataset's assets/ and swap in the resulting AssetRef before the
95
- // shared appendRows() runs (same path the CLI --image flow takes).
99
+ // For image and audio datasets, the agent passes a file path; ingest
100
+ // each into the dataset's assets/ and swap in the resulting AssetRef
101
+ // before the shared appendRows() runs (same path as CLI --image/--audio).
96
102
  const manifest = readManifest(datasetId);
97
103
  let toAppend = rows;
98
- if (manifest?.modality === 'image') {
104
+ if (manifest?.modality === 'image' || manifest?.modality === 'audio') {
105
+ const kind = manifest.modality;
99
106
  toAppend = rows.map((r) => {
100
107
  const path = typeof r.features === 'string' ? r.features : r.features?.path;
101
- if (!path) throw new Error('Image rows need a local image file path in "features".');
102
- return { ...r, features: ingestAsset(datasetId, path, 'image') };
108
+ if (!path) {
109
+ throw new Error(kind === 'image'
110
+ ? 'Image rows need a local image file path in "features".'
111
+ : 'Sound rows need a local .wav file path in "features".');
112
+ }
113
+ return { ...r, features: ingestAsset(datasetId, path, kind) };
103
114
  });
104
115
  }
105
116
  const result = appendRows(datasetId, toAppend, provenance);
@@ -122,10 +133,15 @@ export function getBuiltinTools(agentName, config) {
122
133
  inputSchema: z.object({
123
134
  modelId: z.string().describe('Slug of the local trained model'),
124
135
  input: z
125
- .union([z.string(), z.array(z.number())])
136
+ .union([
137
+ z.string(),
138
+ z.array(z.union([z.number(), z.string(), z.boolean()])),
139
+ z.record(z.union([z.number(), z.string(), z.boolean()])),
140
+ ])
126
141
  .describe(
127
- 'Text; a numeric vector for tabular models; or, for image models, ' +
128
- 'an absolute path to a local image file (.jpg/.png).',
142
+ 'Text for a words model; for a table model, its values in column order or an ' +
143
+ 'object keyed by column name; or, for image models, an absolute path to a local image file (.jpg/.png); ' +
144
+ 'or, for sound models, an absolute path to a local .wav file.',
129
145
  ),
130
146
  }),
131
147
  execute: async ({ modelId, input }) => {
@@ -89,12 +89,13 @@ export async function datasetAppendCommand(options) {
89
89
  const manifest = readManifest(slug);
90
90
 
91
91
  let rows;
92
- if (options.image) {
93
- // Copy the image into the dataset's assets/ and reference it by AssetRef.
94
- if (manifest.modality !== 'image') {
95
- throw new Error(`--image is only valid for image datasets (this one is "${manifest.modality}").`);
92
+ if (options.image || options.audio) {
93
+ // Copy the picture or sound into the dataset's assets/ and reference it by AssetRef.
94
+ const kind = options.image ? 'image' : 'audio';
95
+ if (manifest.modality !== kind) {
96
+ throw new Error(`--${options.image ? 'image' : 'audio'} is only valid for ${kind} datasets (this one is "${manifest.modality}").`);
96
97
  }
97
- const asset = ingestAsset(slug, options.image, 'image');
98
+ const asset = ingestAsset(slug, options.image || options.audio, kind);
98
99
  const row = { split: options.split || 'train', features: asset };
99
100
  if (options.label !== undefined) row.label = options.label;
100
101
  if (options.target !== undefined) row.target = Number(options.target);
@@ -103,18 +104,19 @@ export async function datasetAppendCommand(options) {
103
104
  rows = readRowsFromFile(options.file);
104
105
  } else if (options.text !== undefined || options.features !== undefined) {
105
106
  const row = { split: options.split || 'train' };
106
- row.features = options.features !== undefined ? parseFeatures(options.features, manifest.modality) : options.text;
107
+ row.features = options.features !== undefined ? parseFeatures(options.features, manifest.modality, manifest) : options.text;
107
108
  if (options.label !== undefined) row.label = options.label;
108
109
  if (options.target !== undefined) row.target = Number(options.target);
109
110
  rows = [row];
110
111
  } else {
111
- throw new Error('Provide --image <path> (image datasets), --file <jsonl>, or --text/--features (with --label or --target).');
112
+ throw new Error('Provide --image <path> (image datasets), --audio <path> (audio datasets), --file <jsonl>, or --text/--features (with --label or --target).');
112
113
  }
113
114
 
114
115
  if (options.replace && !options.file) {
115
116
  throw new Error('--replace only works with --file (it swaps in the whole working set).');
116
117
  }
117
118
  const featureNames = options.featureNames !== undefined ? parseNameList(options.featureNames) : undefined;
119
+ const featureTypes = options.featureTypes !== undefined ? parseNameList(options.featureTypes) : undefined;
118
120
 
119
121
  const spinner = options.json ? null : villageSpinner('Appending rows...').start();
120
122
  let result;
@@ -122,6 +124,7 @@ export async function datasetAppendCommand(options) {
122
124
  result = appendRows(slug, rows, { source: options.source || 'cli' }, {
123
125
  replace: Boolean(options.replace),
124
126
  featureNames,
127
+ featureTypes,
125
128
  });
126
129
  } finally {
127
130
  if (spinner) spinner.stop();
@@ -224,7 +227,8 @@ export async function datasetRowUpdateCommand(options) {
224
227
  if (!options.id) throw new Error('--id <rowId> is required.');
225
228
  const updates = {};
226
229
  if (options.features !== undefined) {
227
- updates.features = parseFeatures(options.features, readManifest(slug)?.modality);
230
+ const m = readManifest(slug);
231
+ updates.features = parseFeatures(options.features, m?.modality, m);
228
232
  }
229
233
  else if (options.text !== undefined) updates.features = options.text;
230
234
  if (options.label !== undefined) updates.label = options.label;
@@ -274,12 +278,20 @@ function parseNameList(raw) {
274
278
  return trimmed.split(',').map((s) => s.trim());
275
279
  }
276
280
 
277
- function parseFeatures(raw, modality) {
281
+ function parseFeatures(raw, modality, manifest) {
278
282
  // Tabular features are a numeric vector, passed as JSON ("[12, 45]") or a
279
283
  // comma list ("12, 45" or a single "12"). Anything that isn't a number stays
280
284
  // NaN so row validation rejects it rather than storing it as text.
281
285
  if (modality !== 'tabular') return raw;
282
286
  const trimmed = raw.trim();
287
+ // Typed tables take JSON: a list in column order or an object by column name.
288
+ if (manifest?.featureTypes?.some((t) => t !== 'number')) {
289
+ try {
290
+ return JSON.parse(trimmed);
291
+ } catch {
292
+ throw new Error('This table has typed columns; pass --features as JSON, e.g. \'{"Role": "student", "Hours": 3}\'.');
293
+ }
294
+ }
283
295
  if (trimmed.startsWith('[')) {
284
296
  try {
285
297
  return JSON.parse(trimmed);
@@ -57,7 +57,7 @@ export async function modelTrainCommand(options) {
57
57
  const m = record.metrics;
58
58
  if (m.accuracy != null) info(` accuracy: ${(m.accuracy * 100).toFixed(1)}% over ${m.classes} classes`);
59
59
  if (m.mae != null) info(` MAE: ${m.mae} · R²: ${m.r2}`);
60
- const testFlag = manifest.modality === 'image' ? '--image <path>' : '--input "..."';
60
+ const testFlag = manifest.modality === 'image' ? '--image <path>' : manifest.modality === 'audio' ? '--audio <file.wav>' : '--input "..."';
61
61
  info(` test it: myvillage model predict ${manifest.slug} ${testFlag}`);
62
62
  } catch (e) {
63
63
  if (options.json) process.stdout.write(JSON.stringify({ type: 'error', error: e.message }) + '\n');
@@ -123,11 +123,16 @@ export async function modelPredictCommand(slug, options) {
123
123
  try {
124
124
  if (!modelExists(slug)) throw new Error(`Model "${slug}" not found.`);
125
125
  const record = readModelManifest(slug);
126
- // Image models take a file path (--image); text/tabular take --input.
126
+ // Image and sound models take a file path (--image / --audio); text/tabular take --input.
127
127
  const isImage = record.modality === 'image';
128
- const input = isImage ? options.image : options.input;
128
+ const isAudio = record.modality === 'audio';
129
+ const input = isImage ? options.image : isAudio ? options.audio : options.input;
129
130
  if (input === undefined) {
130
- throw new Error(isImage ? '--image <path> is required for image models.' : '--input <value> is required.');
131
+ throw new Error(
132
+ isImage ? '--image <path> is required for image models.'
133
+ : isAudio ? '--audio <file.wav> is required for sound models.'
134
+ : '--input <value> is required.',
135
+ );
131
136
  }
132
137
  const model = await loadModel(slug);
133
138
  const result = await predictWithModel(model, record, input);
package/src/index.js CHANGED
@@ -1005,14 +1005,16 @@ export function run() {
1005
1005
  .requiredOption('--dataset <slug>', 'Target dataset slug')
1006
1006
  .option('--file <path>', 'JSONL file of rows ({features,label|target,split})')
1007
1007
  .option('--image <path>', 'Image file to add (image datasets); copied into the dataset')
1008
+ .option('--audio <path>', 'WAV file to add (audio datasets); copied into the dataset')
1008
1009
  .option('--text <text>', 'Single-row text features')
1009
- .option('--features <value>', 'Single-row features (string, or numeric vector for tabular)')
1010
+ .option('--features <value>', 'Single-row features (string; numbers for a table, or JSON for a typed table)')
1010
1011
  .option('--label <label>', 'Class label (classification)')
1011
1012
  .option('--target <number>', 'Numeric target (regression)')
1012
1013
  .option('--split <split>', 'train | val', 'train')
1013
1014
  .option('--replace', 'With --file: replace the working set instead of adding to it')
1014
1015
  .option('--source <source>', 'Provenance source stamped on each row (e.g. import:csv)')
1015
1016
  .option('--feature-names <list>', 'Tabular column names (JSON array or comma-separated)')
1017
+ .option('--feature-types <list>', 'Tabular column types, same order: number | category | boolean | text')
1016
1018
  .option('--json', 'Output raw JSON')
1017
1019
  .action(datasetAppendCommand);
1018
1020
 
@@ -1045,7 +1047,7 @@ export function run() {
1045
1047
  .requiredOption('--dataset <slug>', 'Target dataset slug')
1046
1048
  .requiredOption('--id <rowId>', 'Row id to update')
1047
1049
  .option('--text <text>', 'New text features')
1048
- .option('--features <value>', 'New features (string, or numeric vector for tabular)')
1050
+ .option('--features <value>', 'New features (string; numbers for a table, or JSON for a typed table)')
1049
1051
  .option('--label <label>', 'New class label (classification)')
1050
1052
  .option('--target <number>', 'New numeric target (regression)')
1051
1053
  .option('--split <split>', 'train | val')
@@ -1081,8 +1083,9 @@ export function run() {
1081
1083
  modelCmd
1082
1084
  .command('predict <slug>')
1083
1085
  .description('Run a prediction with a trained model')
1084
- .option('--input <value>', 'Input: text, or a comma-separated numeric vector for tabular')
1086
+ .option('--input <value>', 'Input: text; comma-separated numbers for a table; or JSON for a typed table')
1085
1087
  .option('--image <path>', 'Image file to classify (image models)')
1088
+ .option('--audio <path>', 'WAV file to classify (sound models)')
1086
1089
  .option('--json', 'Output raw JSON')
1087
1090
  .action(modelPredictCommand);
1088
1091
 
@@ -0,0 +1,194 @@
1
+ /**
2
+ * Sound feature-extractor for the Studio "Lab" pillar.
3
+ *
4
+ * Turns a WAV file into a fixed-length embedding using YAMNet, which is
5
+ * **bundled inside this package** (vendor/backbones/yamnet/) and loaded from
6
+ * disk — never downloaded — the same way image-backbone.js bundles MobileNet.
7
+ *
8
+ * Pipeline: read the WAV (pure JS, any common PCM or float format) → mix to
9
+ * mono → keep the first MAX_SECONDS → resample to 16 kHz → YAMNet → one
10
+ * 1024-d embedding per 0.48 s frame → averaged into a single 1024-d vector.
11
+ *
12
+ * The embedding feeds the SAME dense head the other modalities train; sound
13
+ * only adds this featurization step. Studio records clips as 16-bit mono WAV,
14
+ * so WAV is the only format read here. tfjs is imported lazily.
15
+ */
16
+ import { readFileSync, existsSync } from 'fs';
17
+ import { join, dirname } from 'path';
18
+ import { fileURLToPath } from 'url';
19
+
20
+ export const BACKBONE_ID = 'yamnet';
21
+ export const SAMPLE_RATE = 16000;
22
+ export const EMBED_DIM = 1024;
23
+ /** Only the start of a long clip is used, so training and predicting stay quick. */
24
+ export const MAX_SECONDS = 10;
25
+ const EMBEDDINGS_OUTPUT = 'Identity_1';
26
+
27
+ const BACKBONE_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'vendor', 'backbones', BACKBONE_ID);
28
+
29
+ let _backbonePromise = null;
30
+
31
+ /** A GraphModel IOHandler reading the vendored files off disk (see image-backbone.js). */
32
+ function diskGraphHandler(dir) {
33
+ return {
34
+ load: async () => {
35
+ const manifestPath = join(dir, 'model.json');
36
+ if (!existsSync(manifestPath)) {
37
+ throw new Error(
38
+ `Bundled sound backbone not found at ${dir}. The YAMNet weights ship ` +
39
+ 'inside @myvillage/cli (vendor/backbones/); reinstall the CLI if this is missing.',
40
+ );
41
+ }
42
+ const j = JSON.parse(readFileSync(manifestPath, 'utf-8'));
43
+ const specs = [];
44
+ const buffers = [];
45
+ for (const grp of j.weightsManifest) {
46
+ for (const p of grp.paths) buffers.push(readFileSync(join(dir, p)));
47
+ specs.push(...grp.weights);
48
+ }
49
+ const concat = Buffer.concat(buffers);
50
+ return {
51
+ modelTopology: j.modelTopology,
52
+ weightSpecs: specs,
53
+ weightData: concat.buffer.slice(concat.byteOffset, concat.byteOffset + concat.byteLength),
54
+ format: j.format,
55
+ generatedBy: j.generatedBy,
56
+ convertedBy: j.convertedBy,
57
+ };
58
+ },
59
+ };
60
+ }
61
+
62
+ async function getBackbone() {
63
+ if (!_backbonePromise) {
64
+ _backbonePromise = (async () => {
65
+ const tf = await import('@tensorflow/tfjs');
66
+ await tf.setBackend('cpu');
67
+ await tf.ready();
68
+ return tf.loadGraphModel(diskGraphHandler(BACKBONE_DIR));
69
+ })().catch((e) => {
70
+ _backbonePromise = null;
71
+ throw e;
72
+ });
73
+ }
74
+ return _backbonePromise;
75
+ }
76
+
77
+ /**
78
+ * A WAV file's bytes → { sampleRate, samples: Float32Array (mono, [-1, 1]) }.
79
+ * Reads 8/16/24/32-bit PCM and 32/64-bit float, any number of channels
80
+ * (mixed down by averaging), including WAVE_FORMAT_EXTENSIBLE files.
81
+ */
82
+ export function decodeWav(buf) {
83
+ const bytes = buf instanceof Uint8Array ? buf : new Uint8Array(buf);
84
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
85
+ const tag = (at) => String.fromCharCode(bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]);
86
+ if (bytes.length < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') {
87
+ throw new Error('That sound isn\'t a WAV file. Record it in Studio, or save it as WAV first.');
88
+ }
89
+ let fmt = null;
90
+ let data = null;
91
+ for (let at = 12; at + 8 <= bytes.length; ) {
92
+ const id = tag(at);
93
+ const size = view.getUint32(at + 4, true);
94
+ const body = at + 8;
95
+ if (id === 'fmt ') {
96
+ let format = view.getUint16(body, true);
97
+ // WAVE_FORMAT_EXTENSIBLE: the real format is the first 2 bytes of the sub-format GUID.
98
+ if (format === 0xfffe && size >= 26) format = view.getUint16(body + 24, true);
99
+ fmt = {
100
+ format,
101
+ channels: view.getUint16(body + 2, true),
102
+ sampleRate: view.getUint32(body + 4, true),
103
+ bits: view.getUint16(body + 14, true),
104
+ };
105
+ } else if (id === 'data') {
106
+ data = { start: body, length: Math.min(size, bytes.length - body) };
107
+ }
108
+ at = body + size + (size % 2); // chunks are padded to an even length
109
+ }
110
+ if (!fmt || !data) throw new Error('That WAV file is missing its sound data.');
111
+ const { format, channels, sampleRate, bits } = fmt;
112
+ const pcm = format === 1;
113
+ const float = format === 3;
114
+ if (!(pcm && [8, 16, 24, 32].includes(bits)) && !(float && (bits === 32 || bits === 64))) {
115
+ throw new Error('That WAV file uses a format Studio can\'t read. Save it as 16-bit PCM WAV.');
116
+ }
117
+ if (!channels || !sampleRate) throw new Error('That WAV file looks damaged.');
118
+ const width = bits / 8;
119
+ const frames = Math.floor(data.length / (width * channels));
120
+ const read = (o) => {
121
+ if (float) return bits === 32 ? view.getFloat32(o, true) : view.getFloat64(o, true);
122
+ if (bits === 8) return (bytes[o] - 128) / 128;
123
+ if (bits === 16) return view.getInt16(o, true) / 32768;
124
+ if (bits === 24) {
125
+ const v = bytes[o] | (bytes[o + 1] << 8) | (bytes[o + 2] << 16);
126
+ return (v & 0x800000 ? v - 0x1000000 : v) / 8388608;
127
+ }
128
+ return view.getInt32(o, true) / 2147483648;
129
+ };
130
+ const samples = new Float32Array(frames);
131
+ for (let i = 0, o = data.start; i < frames; i++) {
132
+ let s = 0;
133
+ for (let c = 0; c < channels; c++, o += width) s += read(o);
134
+ samples[i] = s / channels;
135
+ }
136
+ return { sampleRate, samples };
137
+ }
138
+
139
+ /**
140
+ * Mono samples at any rate → the first MAX_SECONDS at 16 kHz. Going down, each
141
+ * output sample averages the input samples it covers (a box filter, so higher
142
+ * pitches don't fold back in); going up, it interpolates linearly. No trig,
143
+ * so the result is the same on every computer.
144
+ */
145
+ export function toModelRate(samples, sampleRate) {
146
+ const input = samples.subarray(0, Math.min(samples.length, Math.round(sampleRate * MAX_SECONDS)));
147
+ if (sampleRate === SAMPLE_RATE) return Float32Array.from(input);
148
+ const ratio = sampleRate / SAMPLE_RATE;
149
+ const n = Math.floor(input.length / ratio);
150
+ const out = new Float32Array(n);
151
+ if (ratio > 1) {
152
+ for (let k = 0; k < n; k++) {
153
+ const from = k * ratio;
154
+ const to = from + ratio;
155
+ let sum = 0;
156
+ for (let i = Math.floor(from); i < to && i < input.length; i++) {
157
+ const overlap = Math.min(i + 1, to) - Math.max(i, from);
158
+ sum += input[i] * overlap;
159
+ }
160
+ out[k] = sum / ratio;
161
+ }
162
+ } else {
163
+ for (let k = 0; k < n; k++) {
164
+ const x = k * ratio;
165
+ const i = Math.floor(x);
166
+ const t = x - i;
167
+ out[k] = input[i] * (1 - t) + (i + 1 < input.length ? input[i + 1] : input[i]) * t;
168
+ }
169
+ }
170
+ return out;
171
+ }
172
+
173
+ /** 16 kHz mono samples → the averaged 1024-d YAMNet embedding. */
174
+ export async function embedSamples(samples16k) {
175
+ if (!samples16k.length) throw new Error('That sound is empty.');
176
+ const tf = await import('@tensorflow/tfjs');
177
+ const model = await getBackbone();
178
+ return tf.tidy(() => {
179
+ const frames = model.execute(tf.tensor1d(samples16k), EMBEDDINGS_OUTPUT); // [frames, 1024]
180
+ return Array.from(tf.mean(frames, 0).dataSync());
181
+ });
182
+ }
183
+
184
+ /** A WAV file → Float array of length EMBED_DIM (1024). Used by training and predict. */
185
+ export async function embedAudio(path) {
186
+ if (!existsSync(path)) throw new Error(`Sound not found: ${path}`);
187
+ const { samples, sampleRate } = decodeWav(readFileSync(path));
188
+ return embedSamples(toModelRate(samples, sampleRate));
189
+ }
190
+
191
+ /** Featurization params persisted in the model record for sound models. */
192
+ export function audioFeaturization() {
193
+ return { modality: 'audio', dim: EMBED_DIM, backbone: BACKBONE_ID, sampleRate: SAMPLE_RATE, maxSeconds: MAX_SECONDS };
194
+ }
@@ -30,6 +30,7 @@ import {
30
30
  import { join, extname, basename } from 'path';
31
31
  import { randomUUID } from 'crypto';
32
32
  import { getConfigDir } from './config.js';
33
+ import { columnTypesOf, normalizeTypedFeatures, validateColumnTypes } from './table-columns.js';
33
34
 
34
35
  const DATASETS_DIR_NAME = 'datasets';
35
36
  export const DATASET_SCHEMA_VERSION = '1.0.0';
@@ -205,26 +206,34 @@ export function createDataset({ name, modality, taskType = 'classification', lab
205
206
  * Every row is validated before anything is written, so a bad row leaves the
206
207
  * dataset untouched. With `replace`, the working set is overwritten instead of
207
208
  * extended (frozen versions are never touched) and the label list is rebuilt
208
- * from the new rows. `featureNames` names the columns of a tabular vector
209
- * (e.g. from a spreadsheet header); once set, every tabular row must have
210
- * exactly that many numbers.
209
+ * from the new rows. `featureNames` names the columns of a table (tabular)
210
+ * dataset and `featureTypes` optionally types them (number / category /
211
+ * boolean / text, see table-columns.js); without types every column is a
212
+ * number, as before. Rows may then give their values as a list in column
213
+ * order or as an object keyed by column name. Once set, every row must fit
214
+ * those columns.
211
215
  *
212
216
  * @param {string} slug
213
217
  * @param {Array<{features:any, label?:string, target?:number, split?:string}>} rows
214
218
  * @param {object} provenance partial provenance ({ source, agentId, villagerId, villageId })
215
- * @param {{replace?:boolean, featureNames?:string[]}} opts
219
+ * @param {{replace?:boolean, featureNames?:string[], featureTypes?:string[]}} opts
216
220
  * @returns {{datasetId:string, appended:number, replaced:boolean, stats:object}}
217
221
  */
218
- export function appendRows(slug, rows, provenance = {}, { replace = false, featureNames } = {}) {
222
+ export function appendRows(slug, rows, provenance = {}, { replace = false, featureNames, featureTypes } = {}) {
219
223
  const manifest = readManifest(slug);
220
224
  if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
221
225
  if (!Array.isArray(rows) || rows.length === 0) {
222
226
  throw new Error('No rows to append.');
223
227
  }
228
+ if (featureTypes !== undefined && featureNames === undefined) {
229
+ throw new Error('Column types need column names too.');
230
+ }
224
231
  if (featureNames !== undefined) {
225
232
  applyFeatureNames(manifest, featureNames, replace);
233
+ applyFeatureTypes(manifest, featureTypes, replace);
226
234
  } else if (replace) {
227
235
  delete manifest.featureNames;
236
+ delete manifest.featureTypes;
228
237
  }
229
238
 
230
239
  const existing = replace ? [] : readRows(slug);
@@ -272,8 +281,8 @@ export function appendRows(slug, rows, provenance = {}, { replace = false, featu
272
281
  }
273
282
 
274
283
  /** Swap the whole working set for `rows` (e.g. a spreadsheet import). */
275
- export function replaceRows(slug, rows, provenance = {}, { featureNames } = {}) {
276
- return appendRows(slug, rows, provenance, { replace: true, featureNames });
284
+ export function replaceRows(slug, rows, provenance = {}, { featureNames, featureTypes } = {}) {
285
+ return appendRows(slug, rows, provenance, { replace: true, featureNames, featureTypes });
277
286
  }
278
287
 
279
288
  /** Set a tabular dataset's column names, refusing a mismatch on append. */
@@ -286,7 +295,7 @@ function applyFeatureNames(manifest, featureNames, replace) {
286
295
  throw new Error('Feature names must be non-empty.');
287
296
  }
288
297
  const current = manifest.featureNames;
289
- if (!replace && current?.length && current.join('\u0000') !== names.join('\u0000')) {
298
+ if (!replace && manifest.stats.rowCount > 0 && current?.length && current.join('\u0000') !== names.join('\u0000')) {
290
299
  throw new Error(
291
300
  `These columns don't match the dataset. It expects: ${current.join(', ')}.`,
292
301
  );
@@ -294,6 +303,24 @@ function applyFeatureNames(manifest, featureNames, replace) {
294
303
  manifest.featureNames = names;
295
304
  }
296
305
 
306
+ /** Set (replace) or check (append) a table's column types, alongside its names. */
307
+ function applyFeatureTypes(manifest, featureTypes, replace) {
308
+ const incoming = featureTypes === undefined ? undefined : featureTypes.map((t) => String(t).trim());
309
+ if (incoming) validateColumnTypes(incoming, manifest.featureNames);
310
+ // An empty dataset takes whatever columns the first rows bring.
311
+ if (replace || manifest.stats.rowCount === 0) {
312
+ if (incoming && incoming.some((t) => t !== 'number')) manifest.featureTypes = incoming;
313
+ else delete manifest.featureTypes;
314
+ return;
315
+ }
316
+ const current = columnTypesOf(manifest) ?? manifest.featureNames.map(() => 'number');
317
+ const want = incoming ?? manifest.featureNames.map(() => 'number');
318
+ if (current.join('\u0000') !== want.join('\u0000')) {
319
+ const describe = manifest.featureNames.map((n, k) => `${n} (${current[k]})`).join(', ');
320
+ throw new Error(`These columns don't match the dataset. It expects: ${describe}.`);
321
+ }
322
+ }
323
+
297
324
  function normalizeRow(raw, manifest, width) {
298
325
  if (raw == null || typeof raw !== 'object') {
299
326
  throw new Error('Each row must be an object.');
@@ -316,8 +343,16 @@ function normalizeRow(raw, manifest, width) {
316
343
  );
317
344
  }
318
345
  }
319
- if (manifest.modality === 'tabular') {
320
- const f = raw.features;
346
+ let features = raw.features;
347
+ const types = manifest.modality === 'tabular' ? columnTypesOf(manifest) : null;
348
+ if (types) {
349
+ features = normalizeTypedFeatures(raw.features, manifest.featureNames, types);
350
+ } else if (manifest.modality === 'tabular') {
351
+ // An object keyed by column name is fine for a named all-numbers table too.
352
+ if (features && typeof features === 'object' && !Array.isArray(features) && manifest.featureNames?.length) {
353
+ features = normalizeTypedFeatures(features, manifest.featureNames, manifest.featureNames.map(() => 'number'));
354
+ }
355
+ const f = features;
321
356
  if (!Array.isArray(f) || f.length === 0 || f.some((n) => typeof n !== 'number' || !Number.isFinite(n))) {
322
357
  throw new Error('Tabular rows need features that are all numbers (e.g. [12, 45, 8]).');
323
358
  }
@@ -328,7 +363,7 @@ function normalizeRow(raw, manifest, width) {
328
363
  if (manifest.modality === 'text' && (typeof raw.features !== 'string' || !raw.features.trim())) {
329
364
  throw new Error('Text rows need some text.');
330
365
  }
331
- const row = { features: raw.features, split };
366
+ const row = { features, split };
332
367
  if (manifest.taskType === 'classification') {
333
368
  if (raw.label == null || String(raw.label).length === 0) {
334
369
  throw new Error('Classification rows require a "label".');