@myvillage/cli 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent-runtime/builtin-tools-registry.js +8 -5
- package/src/agent-runtime/builtin-tools.js +28 -12
- package/src/commands/dataset.js +21 -9
- package/src/commands/model.js +9 -4
- package/src/index.js +6 -3
- package/src/utils/audio-backbone.js +194 -0
- package/src/utils/dataset-workspace.js +46 -11
- package/src/utils/model-trainer.js +56 -34
- package/src/utils/table-columns.js +162 -0
- package/vendor/backbones/yamnet/SOURCE.md +22 -0
- package/vendor/backbones/yamnet/group1-shard1of4.bin +0 -0
- package/vendor/backbones/yamnet/group1-shard2of4.bin +0 -0
- package/vendor/backbones/yamnet/group1-shard3of4.bin +0 -0
- package/vendor/backbones/yamnet/group1-shard4of4.bin +0 -0
- package/vendor/backbones/yamnet/model.json +1 -0
package/package.json
CHANGED
|
@@ -14,16 +14,19 @@
|
|
|
14
14
|
export const DATASET_APPEND_DESCRIPTION =
|
|
15
15
|
'Append labeled training example(s) to a LOCAL dataset on this machine. ' +
|
|
16
16
|
'The dataset must already exist (created via the Studio Lab or ' +
|
|
17
|
-
'`myvillage dataset create`). `features` is text
|
|
18
|
-
'
|
|
19
|
-
'
|
|
17
|
+
'`myvillage dataset create`). `features` is text (words datasets); for a ' +
|
|
18
|
+
'table dataset, its values in column order or an object keyed by column ' +
|
|
19
|
+
'name (numbers, category text, true/false, short notes); or a local image ' +
|
|
20
|
+
'file path (image datasets) or .wav file path (audio datasets) — the file is ' +
|
|
21
|
+
'copied into the dataset. For classification provide `label`; for ' +
|
|
20
22
|
'regression provide a numeric `target`. Data stays local; provenance ' +
|
|
21
23
|
'is recorded automatically.';
|
|
22
24
|
|
|
23
25
|
export const MODEL_PREDICT_DESCRIPTION =
|
|
24
26
|
'Run a prediction with a LOCAL trained model on this machine. `input` is ' +
|
|
25
|
-
'text,
|
|
26
|
-
'models)
|
|
27
|
+
'text (words models), the values of one table row as a list in column ' +
|
|
28
|
+
'order or an object keyed by column name (table models), a local image file path (image ' +
|
|
29
|
+
'models), or a local .wav file path (sound models). Returns the predicted class with per-class confidence scores ' +
|
|
27
30
|
'(classification) or a numeric value (regression). The model must already ' +
|
|
28
31
|
'exist (trained in the Studio Lab or via `myvillage model train`).';
|
|
29
32
|
|
|
@@ -65,10 +65,16 @@ export function getBuiltinTools(agentName, config) {
|
|
|
65
65
|
.array(
|
|
66
66
|
z.object({
|
|
67
67
|
features: z
|
|
68
|
-
.union([
|
|
68
|
+
.union([
|
|
69
|
+
z.string(),
|
|
70
|
+
z.array(z.union([z.number(), z.string(), z.boolean()])),
|
|
71
|
+
z.record(z.union([z.number(), z.string(), z.boolean()])),
|
|
72
|
+
])
|
|
69
73
|
.describe(
|
|
70
|
-
'Text content
|
|
71
|
-
'
|
|
74
|
+
'Text content for a words dataset; for a table dataset, its values in column ' +
|
|
75
|
+
'order or an object keyed by column name (numbers, category text, true/false, ' +
|
|
76
|
+
'short notes); or, for image datasets, an absolute path to a local image file (.jpg/.png); ' +
|
|
77
|
+
'or, for audio datasets, an absolute path to a local .wav file.',
|
|
72
78
|
),
|
|
73
79
|
label: z.string().optional().describe('Class label (classification datasets)'),
|
|
74
80
|
target: z.number().optional().describe('Numeric target (regression datasets)'),
|
|
@@ -90,16 +96,21 @@ export function getBuiltinTools(agentName, config) {
|
|
|
90
96
|
const provenance = { source: `agent:${agentName}` };
|
|
91
97
|
const agentId = config?.man?.village_agent_id;
|
|
92
98
|
if (agentId) provenance.agentId = agentId;
|
|
93
|
-
// For image datasets, the agent passes a file path; ingest
|
|
94
|
-
// the dataset's assets/ and swap in the resulting AssetRef
|
|
95
|
-
// shared appendRows() runs (same path
|
|
99
|
+
// For image and audio datasets, the agent passes a file path; ingest
|
|
100
|
+
// each into the dataset's assets/ and swap in the resulting AssetRef
|
|
101
|
+
// before the shared appendRows() runs (same path as CLI --image/--audio).
|
|
96
102
|
const manifest = readManifest(datasetId);
|
|
97
103
|
let toAppend = rows;
|
|
98
|
-
if (manifest?.modality === 'image') {
|
|
104
|
+
if (manifest?.modality === 'image' || manifest?.modality === 'audio') {
|
|
105
|
+
const kind = manifest.modality;
|
|
99
106
|
toAppend = rows.map((r) => {
|
|
100
107
|
const path = typeof r.features === 'string' ? r.features : r.features?.path;
|
|
101
|
-
if (!path)
|
|
102
|
-
|
|
108
|
+
if (!path) {
|
|
109
|
+
throw new Error(kind === 'image'
|
|
110
|
+
? 'Image rows need a local image file path in "features".'
|
|
111
|
+
: 'Sound rows need a local .wav file path in "features".');
|
|
112
|
+
}
|
|
113
|
+
return { ...r, features: ingestAsset(datasetId, path, kind) };
|
|
103
114
|
});
|
|
104
115
|
}
|
|
105
116
|
const result = appendRows(datasetId, toAppend, provenance);
|
|
@@ -122,10 +133,15 @@ export function getBuiltinTools(agentName, config) {
|
|
|
122
133
|
inputSchema: z.object({
|
|
123
134
|
modelId: z.string().describe('Slug of the local trained model'),
|
|
124
135
|
input: z
|
|
125
|
-
.union([
|
|
136
|
+
.union([
|
|
137
|
+
z.string(),
|
|
138
|
+
z.array(z.union([z.number(), z.string(), z.boolean()])),
|
|
139
|
+
z.record(z.union([z.number(), z.string(), z.boolean()])),
|
|
140
|
+
])
|
|
126
141
|
.describe(
|
|
127
|
-
'Text
|
|
128
|
-
'an absolute path to a local image file (.jpg/.png)
|
|
142
|
+
'Text for a words model; for a table model, its values in column order or an ' +
|
|
143
|
+
'object keyed by column name; or, for image models, an absolute path to a local image file (.jpg/.png); ' +
|
|
144
|
+
'or, for sound models, an absolute path to a local .wav file.',
|
|
129
145
|
),
|
|
130
146
|
}),
|
|
131
147
|
execute: async ({ modelId, input }) => {
|
package/src/commands/dataset.js
CHANGED
|
@@ -89,12 +89,13 @@ export async function datasetAppendCommand(options) {
|
|
|
89
89
|
const manifest = readManifest(slug);
|
|
90
90
|
|
|
91
91
|
let rows;
|
|
92
|
-
if (options.image) {
|
|
93
|
-
// Copy the
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
if (options.image || options.audio) {
|
|
93
|
+
// Copy the picture or sound into the dataset's assets/ and reference it by AssetRef.
|
|
94
|
+
const kind = options.image ? 'image' : 'audio';
|
|
95
|
+
if (manifest.modality !== kind) {
|
|
96
|
+
throw new Error(`--${options.image ? 'image' : 'audio'} is only valid for ${kind} datasets (this one is "${manifest.modality}").`);
|
|
96
97
|
}
|
|
97
|
-
const asset = ingestAsset(slug, options.image,
|
|
98
|
+
const asset = ingestAsset(slug, options.image || options.audio, kind);
|
|
98
99
|
const row = { split: options.split || 'train', features: asset };
|
|
99
100
|
if (options.label !== undefined) row.label = options.label;
|
|
100
101
|
if (options.target !== undefined) row.target = Number(options.target);
|
|
@@ -103,18 +104,19 @@ export async function datasetAppendCommand(options) {
|
|
|
103
104
|
rows = readRowsFromFile(options.file);
|
|
104
105
|
} else if (options.text !== undefined || options.features !== undefined) {
|
|
105
106
|
const row = { split: options.split || 'train' };
|
|
106
|
-
row.features = options.features !== undefined ? parseFeatures(options.features, manifest.modality) : options.text;
|
|
107
|
+
row.features = options.features !== undefined ? parseFeatures(options.features, manifest.modality, manifest) : options.text;
|
|
107
108
|
if (options.label !== undefined) row.label = options.label;
|
|
108
109
|
if (options.target !== undefined) row.target = Number(options.target);
|
|
109
110
|
rows = [row];
|
|
110
111
|
} else {
|
|
111
|
-
throw new Error('Provide --image <path> (image datasets), --file <jsonl>, or --text/--features (with --label or --target).');
|
|
112
|
+
throw new Error('Provide --image <path> (image datasets), --audio <path> (audio datasets), --file <jsonl>, or --text/--features (with --label or --target).');
|
|
112
113
|
}
|
|
113
114
|
|
|
114
115
|
if (options.replace && !options.file) {
|
|
115
116
|
throw new Error('--replace only works with --file (it swaps in the whole working set).');
|
|
116
117
|
}
|
|
117
118
|
const featureNames = options.featureNames !== undefined ? parseNameList(options.featureNames) : undefined;
|
|
119
|
+
const featureTypes = options.featureTypes !== undefined ? parseNameList(options.featureTypes) : undefined;
|
|
118
120
|
|
|
119
121
|
const spinner = options.json ? null : villageSpinner('Appending rows...').start();
|
|
120
122
|
let result;
|
|
@@ -122,6 +124,7 @@ export async function datasetAppendCommand(options) {
|
|
|
122
124
|
result = appendRows(slug, rows, { source: options.source || 'cli' }, {
|
|
123
125
|
replace: Boolean(options.replace),
|
|
124
126
|
featureNames,
|
|
127
|
+
featureTypes,
|
|
125
128
|
});
|
|
126
129
|
} finally {
|
|
127
130
|
if (spinner) spinner.stop();
|
|
@@ -224,7 +227,8 @@ export async function datasetRowUpdateCommand(options) {
|
|
|
224
227
|
if (!options.id) throw new Error('--id <rowId> is required.');
|
|
225
228
|
const updates = {};
|
|
226
229
|
if (options.features !== undefined) {
|
|
227
|
-
|
|
230
|
+
const m = readManifest(slug);
|
|
231
|
+
updates.features = parseFeatures(options.features, m?.modality, m);
|
|
228
232
|
}
|
|
229
233
|
else if (options.text !== undefined) updates.features = options.text;
|
|
230
234
|
if (options.label !== undefined) updates.label = options.label;
|
|
@@ -274,12 +278,20 @@ function parseNameList(raw) {
|
|
|
274
278
|
return trimmed.split(',').map((s) => s.trim());
|
|
275
279
|
}
|
|
276
280
|
|
|
277
|
-
function parseFeatures(raw, modality) {
|
|
281
|
+
function parseFeatures(raw, modality, manifest) {
|
|
278
282
|
// Tabular features are a numeric vector, passed as JSON ("[12, 45]") or a
|
|
279
283
|
// comma list ("12, 45" or a single "12"). Anything that isn't a number stays
|
|
280
284
|
// NaN so row validation rejects it rather than storing it as text.
|
|
281
285
|
if (modality !== 'tabular') return raw;
|
|
282
286
|
const trimmed = raw.trim();
|
|
287
|
+
// Typed tables take JSON: a list in column order or an object by column name.
|
|
288
|
+
if (manifest?.featureTypes?.some((t) => t !== 'number')) {
|
|
289
|
+
try {
|
|
290
|
+
return JSON.parse(trimmed);
|
|
291
|
+
} catch {
|
|
292
|
+
throw new Error('This table has typed columns; pass --features as JSON, e.g. \'{"Role": "student", "Hours": 3}\'.');
|
|
293
|
+
}
|
|
294
|
+
}
|
|
283
295
|
if (trimmed.startsWith('[')) {
|
|
284
296
|
try {
|
|
285
297
|
return JSON.parse(trimmed);
|
package/src/commands/model.js
CHANGED
|
@@ -57,7 +57,7 @@ export async function modelTrainCommand(options) {
|
|
|
57
57
|
const m = record.metrics;
|
|
58
58
|
if (m.accuracy != null) info(` accuracy: ${(m.accuracy * 100).toFixed(1)}% over ${m.classes} classes`);
|
|
59
59
|
if (m.mae != null) info(` MAE: ${m.mae} · R²: ${m.r2}`);
|
|
60
|
-
const testFlag = manifest.modality === 'image' ? '--image <path>' : '--input "..."';
|
|
60
|
+
const testFlag = manifest.modality === 'image' ? '--image <path>' : manifest.modality === 'audio' ? '--audio <file.wav>' : '--input "..."';
|
|
61
61
|
info(` test it: myvillage model predict ${manifest.slug} ${testFlag}`);
|
|
62
62
|
} catch (e) {
|
|
63
63
|
if (options.json) process.stdout.write(JSON.stringify({ type: 'error', error: e.message }) + '\n');
|
|
@@ -123,11 +123,16 @@ export async function modelPredictCommand(slug, options) {
|
|
|
123
123
|
try {
|
|
124
124
|
if (!modelExists(slug)) throw new Error(`Model "${slug}" not found.`);
|
|
125
125
|
const record = readModelManifest(slug);
|
|
126
|
-
// Image models take a file path (--image); text/tabular take --input.
|
|
126
|
+
// Image and sound models take a file path (--image / --audio); text/tabular take --input.
|
|
127
127
|
const isImage = record.modality === 'image';
|
|
128
|
-
const
|
|
128
|
+
const isAudio = record.modality === 'audio';
|
|
129
|
+
const input = isImage ? options.image : isAudio ? options.audio : options.input;
|
|
129
130
|
if (input === undefined) {
|
|
130
|
-
throw new Error(
|
|
131
|
+
throw new Error(
|
|
132
|
+
isImage ? '--image <path> is required for image models.'
|
|
133
|
+
: isAudio ? '--audio <file.wav> is required for sound models.'
|
|
134
|
+
: '--input <value> is required.',
|
|
135
|
+
);
|
|
131
136
|
}
|
|
132
137
|
const model = await loadModel(slug);
|
|
133
138
|
const result = await predictWithModel(model, record, input);
|
package/src/index.js
CHANGED
|
@@ -1005,14 +1005,16 @@ export function run() {
|
|
|
1005
1005
|
.requiredOption('--dataset <slug>', 'Target dataset slug')
|
|
1006
1006
|
.option('--file <path>', 'JSONL file of rows ({features,label|target,split})')
|
|
1007
1007
|
.option('--image <path>', 'Image file to add (image datasets); copied into the dataset')
|
|
1008
|
+
.option('--audio <path>', 'WAV file to add (audio datasets); copied into the dataset')
|
|
1008
1009
|
.option('--text <text>', 'Single-row text features')
|
|
1009
|
-
.option('--features <value>', 'Single-row features (string, or
|
|
1010
|
+
.option('--features <value>', 'Single-row features (string; numbers for a table, or JSON for a typed table)')
|
|
1010
1011
|
.option('--label <label>', 'Class label (classification)')
|
|
1011
1012
|
.option('--target <number>', 'Numeric target (regression)')
|
|
1012
1013
|
.option('--split <split>', 'train | val', 'train')
|
|
1013
1014
|
.option('--replace', 'With --file: replace the working set instead of adding to it')
|
|
1014
1015
|
.option('--source <source>', 'Provenance source stamped on each row (e.g. import:csv)')
|
|
1015
1016
|
.option('--feature-names <list>', 'Tabular column names (JSON array or comma-separated)')
|
|
1017
|
+
.option('--feature-types <list>', 'Tabular column types, same order: number | category | boolean | text')
|
|
1016
1018
|
.option('--json', 'Output raw JSON')
|
|
1017
1019
|
.action(datasetAppendCommand);
|
|
1018
1020
|
|
|
@@ -1045,7 +1047,7 @@ export function run() {
|
|
|
1045
1047
|
.requiredOption('--dataset <slug>', 'Target dataset slug')
|
|
1046
1048
|
.requiredOption('--id <rowId>', 'Row id to update')
|
|
1047
1049
|
.option('--text <text>', 'New text features')
|
|
1048
|
-
.option('--features <value>', 'New features (string, or
|
|
1050
|
+
.option('--features <value>', 'New features (string; numbers for a table, or JSON for a typed table)')
|
|
1049
1051
|
.option('--label <label>', 'New class label (classification)')
|
|
1050
1052
|
.option('--target <number>', 'New numeric target (regression)')
|
|
1051
1053
|
.option('--split <split>', 'train | val')
|
|
@@ -1081,8 +1083,9 @@ export function run() {
|
|
|
1081
1083
|
modelCmd
|
|
1082
1084
|
.command('predict <slug>')
|
|
1083
1085
|
.description('Run a prediction with a trained model')
|
|
1084
|
-
.option('--input <value>', 'Input: text
|
|
1086
|
+
.option('--input <value>', 'Input: text; comma-separated numbers for a table; or JSON for a typed table')
|
|
1085
1087
|
.option('--image <path>', 'Image file to classify (image models)')
|
|
1088
|
+
.option('--audio <path>', 'WAV file to classify (sound models)')
|
|
1086
1089
|
.option('--json', 'Output raw JSON')
|
|
1087
1090
|
.action(modelPredictCommand);
|
|
1088
1091
|
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sound feature-extractor for the Studio "Lab" pillar.
|
|
3
|
+
*
|
|
4
|
+
* Turns a WAV file into a fixed-length embedding using YAMNet, which is
|
|
5
|
+
* **bundled inside this package** (vendor/backbones/yamnet/) and loaded from
|
|
6
|
+
* disk — never downloaded — the same way image-backbone.js bundles MobileNet.
|
|
7
|
+
*
|
|
8
|
+
* Pipeline: read the WAV (pure JS, any common PCM or float format) → mix to
|
|
9
|
+
* mono → keep the first MAX_SECONDS → resample to 16 kHz → YAMNet → one
|
|
10
|
+
* 1024-d embedding per 0.48 s frame → averaged into a single 1024-d vector.
|
|
11
|
+
*
|
|
12
|
+
* The embedding feeds the SAME dense head the other modalities train; sound
|
|
13
|
+
* only adds this featurization step. Studio records clips as 16-bit mono WAV,
|
|
14
|
+
* so WAV is the only format read here. tfjs is imported lazily.
|
|
15
|
+
*/
|
|
16
|
+
import { readFileSync, existsSync } from 'fs';
|
|
17
|
+
import { join, dirname } from 'path';
|
|
18
|
+
import { fileURLToPath } from 'url';
|
|
19
|
+
|
|
20
|
+
export const BACKBONE_ID = 'yamnet';
|
|
21
|
+
export const SAMPLE_RATE = 16000;
|
|
22
|
+
export const EMBED_DIM = 1024;
|
|
23
|
+
/** Only the start of a long clip is used, so training and predicting stay quick. */
|
|
24
|
+
export const MAX_SECONDS = 10;
|
|
25
|
+
const EMBEDDINGS_OUTPUT = 'Identity_1';
|
|
26
|
+
|
|
27
|
+
const BACKBONE_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'vendor', 'backbones', BACKBONE_ID);
|
|
28
|
+
|
|
29
|
+
let _backbonePromise = null;
|
|
30
|
+
|
|
31
|
+
/** A GraphModel IOHandler reading the vendored files off disk (see image-backbone.js). */
|
|
32
|
+
function diskGraphHandler(dir) {
|
|
33
|
+
return {
|
|
34
|
+
load: async () => {
|
|
35
|
+
const manifestPath = join(dir, 'model.json');
|
|
36
|
+
if (!existsSync(manifestPath)) {
|
|
37
|
+
throw new Error(
|
|
38
|
+
`Bundled sound backbone not found at ${dir}. The YAMNet weights ship ` +
|
|
39
|
+
'inside @myvillage/cli (vendor/backbones/); reinstall the CLI if this is missing.',
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
const j = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
43
|
+
const specs = [];
|
|
44
|
+
const buffers = [];
|
|
45
|
+
for (const grp of j.weightsManifest) {
|
|
46
|
+
for (const p of grp.paths) buffers.push(readFileSync(join(dir, p)));
|
|
47
|
+
specs.push(...grp.weights);
|
|
48
|
+
}
|
|
49
|
+
const concat = Buffer.concat(buffers);
|
|
50
|
+
return {
|
|
51
|
+
modelTopology: j.modelTopology,
|
|
52
|
+
weightSpecs: specs,
|
|
53
|
+
weightData: concat.buffer.slice(concat.byteOffset, concat.byteOffset + concat.byteLength),
|
|
54
|
+
format: j.format,
|
|
55
|
+
generatedBy: j.generatedBy,
|
|
56
|
+
convertedBy: j.convertedBy,
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function getBackbone() {
|
|
63
|
+
if (!_backbonePromise) {
|
|
64
|
+
_backbonePromise = (async () => {
|
|
65
|
+
const tf = await import('@tensorflow/tfjs');
|
|
66
|
+
await tf.setBackend('cpu');
|
|
67
|
+
await tf.ready();
|
|
68
|
+
return tf.loadGraphModel(diskGraphHandler(BACKBONE_DIR));
|
|
69
|
+
})().catch((e) => {
|
|
70
|
+
_backbonePromise = null;
|
|
71
|
+
throw e;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return _backbonePromise;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* A WAV file's bytes → { sampleRate, samples: Float32Array (mono, [-1, 1]) }.
|
|
79
|
+
* Reads 8/16/24/32-bit PCM and 32/64-bit float, any number of channels
|
|
80
|
+
* (mixed down by averaging), including WAVE_FORMAT_EXTENSIBLE files.
|
|
81
|
+
*/
|
|
82
|
+
export function decodeWav(buf) {
|
|
83
|
+
const bytes = buf instanceof Uint8Array ? buf : new Uint8Array(buf);
|
|
84
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
85
|
+
const tag = (at) => String.fromCharCode(bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]);
|
|
86
|
+
if (bytes.length < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') {
|
|
87
|
+
throw new Error('That sound isn\'t a WAV file. Record it in Studio, or save it as WAV first.');
|
|
88
|
+
}
|
|
89
|
+
let fmt = null;
|
|
90
|
+
let data = null;
|
|
91
|
+
for (let at = 12; at + 8 <= bytes.length; ) {
|
|
92
|
+
const id = tag(at);
|
|
93
|
+
const size = view.getUint32(at + 4, true);
|
|
94
|
+
const body = at + 8;
|
|
95
|
+
if (id === 'fmt ') {
|
|
96
|
+
let format = view.getUint16(body, true);
|
|
97
|
+
// WAVE_FORMAT_EXTENSIBLE: the real format is the first 2 bytes of the sub-format GUID.
|
|
98
|
+
if (format === 0xfffe && size >= 26) format = view.getUint16(body + 24, true);
|
|
99
|
+
fmt = {
|
|
100
|
+
format,
|
|
101
|
+
channels: view.getUint16(body + 2, true),
|
|
102
|
+
sampleRate: view.getUint32(body + 4, true),
|
|
103
|
+
bits: view.getUint16(body + 14, true),
|
|
104
|
+
};
|
|
105
|
+
} else if (id === 'data') {
|
|
106
|
+
data = { start: body, length: Math.min(size, bytes.length - body) };
|
|
107
|
+
}
|
|
108
|
+
at = body + size + (size % 2); // chunks are padded to an even length
|
|
109
|
+
}
|
|
110
|
+
if (!fmt || !data) throw new Error('That WAV file is missing its sound data.');
|
|
111
|
+
const { format, channels, sampleRate, bits } = fmt;
|
|
112
|
+
const pcm = format === 1;
|
|
113
|
+
const float = format === 3;
|
|
114
|
+
if (!(pcm && [8, 16, 24, 32].includes(bits)) && !(float && (bits === 32 || bits === 64))) {
|
|
115
|
+
throw new Error('That WAV file uses a format Studio can\'t read. Save it as 16-bit PCM WAV.');
|
|
116
|
+
}
|
|
117
|
+
if (!channels || !sampleRate) throw new Error('That WAV file looks damaged.');
|
|
118
|
+
const width = bits / 8;
|
|
119
|
+
const frames = Math.floor(data.length / (width * channels));
|
|
120
|
+
const read = (o) => {
|
|
121
|
+
if (float) return bits === 32 ? view.getFloat32(o, true) : view.getFloat64(o, true);
|
|
122
|
+
if (bits === 8) return (bytes[o] - 128) / 128;
|
|
123
|
+
if (bits === 16) return view.getInt16(o, true) / 32768;
|
|
124
|
+
if (bits === 24) {
|
|
125
|
+
const v = bytes[o] | (bytes[o + 1] << 8) | (bytes[o + 2] << 16);
|
|
126
|
+
return (v & 0x800000 ? v - 0x1000000 : v) / 8388608;
|
|
127
|
+
}
|
|
128
|
+
return view.getInt32(o, true) / 2147483648;
|
|
129
|
+
};
|
|
130
|
+
const samples = new Float32Array(frames);
|
|
131
|
+
for (let i = 0, o = data.start; i < frames; i++) {
|
|
132
|
+
let s = 0;
|
|
133
|
+
for (let c = 0; c < channels; c++, o += width) s += read(o);
|
|
134
|
+
samples[i] = s / channels;
|
|
135
|
+
}
|
|
136
|
+
return { sampleRate, samples };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Mono samples at any rate → the first MAX_SECONDS at 16 kHz. Going down, each
|
|
141
|
+
* output sample averages the input samples it covers (a box filter, so higher
|
|
142
|
+
* pitches don't fold back in); going up, it interpolates linearly. No trig,
|
|
143
|
+
* so the result is the same on every computer.
|
|
144
|
+
*/
|
|
145
|
+
export function toModelRate(samples, sampleRate) {
|
|
146
|
+
const input = samples.subarray(0, Math.min(samples.length, Math.round(sampleRate * MAX_SECONDS)));
|
|
147
|
+
if (sampleRate === SAMPLE_RATE) return Float32Array.from(input);
|
|
148
|
+
const ratio = sampleRate / SAMPLE_RATE;
|
|
149
|
+
const n = Math.floor(input.length / ratio);
|
|
150
|
+
const out = new Float32Array(n);
|
|
151
|
+
if (ratio > 1) {
|
|
152
|
+
for (let k = 0; k < n; k++) {
|
|
153
|
+
const from = k * ratio;
|
|
154
|
+
const to = from + ratio;
|
|
155
|
+
let sum = 0;
|
|
156
|
+
for (let i = Math.floor(from); i < to && i < input.length; i++) {
|
|
157
|
+
const overlap = Math.min(i + 1, to) - Math.max(i, from);
|
|
158
|
+
sum += input[i] * overlap;
|
|
159
|
+
}
|
|
160
|
+
out[k] = sum / ratio;
|
|
161
|
+
}
|
|
162
|
+
} else {
|
|
163
|
+
for (let k = 0; k < n; k++) {
|
|
164
|
+
const x = k * ratio;
|
|
165
|
+
const i = Math.floor(x);
|
|
166
|
+
const t = x - i;
|
|
167
|
+
out[k] = input[i] * (1 - t) + (i + 1 < input.length ? input[i + 1] : input[i]) * t;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return out;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** 16 kHz mono samples → the averaged 1024-d YAMNet embedding. */
|
|
174
|
+
export async function embedSamples(samples16k) {
|
|
175
|
+
if (!samples16k.length) throw new Error('That sound is empty.');
|
|
176
|
+
const tf = await import('@tensorflow/tfjs');
|
|
177
|
+
const model = await getBackbone();
|
|
178
|
+
return tf.tidy(() => {
|
|
179
|
+
const frames = model.execute(tf.tensor1d(samples16k), EMBEDDINGS_OUTPUT); // [frames, 1024]
|
|
180
|
+
return Array.from(tf.mean(frames, 0).dataSync());
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** A WAV file → Float array of length EMBED_DIM (1024). Used by training and predict. */
|
|
185
|
+
export async function embedAudio(path) {
|
|
186
|
+
if (!existsSync(path)) throw new Error(`Sound not found: ${path}`);
|
|
187
|
+
const { samples, sampleRate } = decodeWav(readFileSync(path));
|
|
188
|
+
return embedSamples(toModelRate(samples, sampleRate));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Featurization params persisted in the model record for sound models. */
|
|
192
|
+
export function audioFeaturization() {
|
|
193
|
+
return { modality: 'audio', dim: EMBED_DIM, backbone: BACKBONE_ID, sampleRate: SAMPLE_RATE, maxSeconds: MAX_SECONDS };
|
|
194
|
+
}
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
import { join, extname, basename } from 'path';
|
|
31
31
|
import { randomUUID } from 'crypto';
|
|
32
32
|
import { getConfigDir } from './config.js';
|
|
33
|
+
import { columnTypesOf, normalizeTypedFeatures, validateColumnTypes } from './table-columns.js';
|
|
33
34
|
|
|
34
35
|
const DATASETS_DIR_NAME = 'datasets';
|
|
35
36
|
export const DATASET_SCHEMA_VERSION = '1.0.0';
|
|
@@ -205,26 +206,34 @@ export function createDataset({ name, modality, taskType = 'classification', lab
|
|
|
205
206
|
* Every row is validated before anything is written, so a bad row leaves the
|
|
206
207
|
* dataset untouched. With `replace`, the working set is overwritten instead of
|
|
207
208
|
* extended (frozen versions are never touched) and the label list is rebuilt
|
|
208
|
-
* from the new rows. `featureNames` names the columns of a tabular
|
|
209
|
-
*
|
|
210
|
-
*
|
|
209
|
+
* from the new rows. `featureNames` names the columns of a table (tabular)
|
|
210
|
+
* dataset and `featureTypes` optionally types them (number / category /
|
|
211
|
+
* boolean / text, see table-columns.js); without types every column is a
|
|
212
|
+
* number, as before. Rows may then give their values as a list in column
|
|
213
|
+
* order or as an object keyed by column name. Once set, every row must fit
|
|
214
|
+
* those columns.
|
|
211
215
|
*
|
|
212
216
|
* @param {string} slug
|
|
213
217
|
* @param {Array<{features:any, label?:string, target?:number, split?:string}>} rows
|
|
214
218
|
* @param {object} provenance partial provenance ({ source, agentId, villagerId, villageId })
|
|
215
|
-
* @param {{replace?:boolean, featureNames?:string[]}} opts
|
|
219
|
+
* @param {{replace?:boolean, featureNames?:string[], featureTypes?:string[]}} opts
|
|
216
220
|
* @returns {{datasetId:string, appended:number, replaced:boolean, stats:object}}
|
|
217
221
|
*/
|
|
218
|
-
export function appendRows(slug, rows, provenance = {}, { replace = false, featureNames } = {}) {
|
|
222
|
+
export function appendRows(slug, rows, provenance = {}, { replace = false, featureNames, featureTypes } = {}) {
|
|
219
223
|
const manifest = readManifest(slug);
|
|
220
224
|
if (!manifest) throw new Error(`Dataset "${slug}" not found.`);
|
|
221
225
|
if (!Array.isArray(rows) || rows.length === 0) {
|
|
222
226
|
throw new Error('No rows to append.');
|
|
223
227
|
}
|
|
228
|
+
if (featureTypes !== undefined && featureNames === undefined) {
|
|
229
|
+
throw new Error('Column types need column names too.');
|
|
230
|
+
}
|
|
224
231
|
if (featureNames !== undefined) {
|
|
225
232
|
applyFeatureNames(manifest, featureNames, replace);
|
|
233
|
+
applyFeatureTypes(manifest, featureTypes, replace);
|
|
226
234
|
} else if (replace) {
|
|
227
235
|
delete manifest.featureNames;
|
|
236
|
+
delete manifest.featureTypes;
|
|
228
237
|
}
|
|
229
238
|
|
|
230
239
|
const existing = replace ? [] : readRows(slug);
|
|
@@ -272,8 +281,8 @@ export function appendRows(slug, rows, provenance = {}, { replace = false, featu
|
|
|
272
281
|
}
|
|
273
282
|
|
|
274
283
|
/** Swap the whole working set for `rows` (e.g. a spreadsheet import). */
|
|
275
|
-
export function replaceRows(slug, rows, provenance = {}, { featureNames } = {}) {
|
|
276
|
-
return appendRows(slug, rows, provenance, { replace: true, featureNames });
|
|
284
|
+
export function replaceRows(slug, rows, provenance = {}, { featureNames, featureTypes } = {}) {
|
|
285
|
+
return appendRows(slug, rows, provenance, { replace: true, featureNames, featureTypes });
|
|
277
286
|
}
|
|
278
287
|
|
|
279
288
|
/** Set a tabular dataset's column names, refusing a mismatch on append. */
|
|
@@ -286,7 +295,7 @@ function applyFeatureNames(manifest, featureNames, replace) {
|
|
|
286
295
|
throw new Error('Feature names must be non-empty.');
|
|
287
296
|
}
|
|
288
297
|
const current = manifest.featureNames;
|
|
289
|
-
if (!replace && current?.length && current.join('\u0000') !== names.join('\u0000')) {
|
|
298
|
+
if (!replace && manifest.stats.rowCount > 0 && current?.length && current.join('\u0000') !== names.join('\u0000')) {
|
|
290
299
|
throw new Error(
|
|
291
300
|
`These columns don't match the dataset. It expects: ${current.join(', ')}.`,
|
|
292
301
|
);
|
|
@@ -294,6 +303,24 @@ function applyFeatureNames(manifest, featureNames, replace) {
|
|
|
294
303
|
manifest.featureNames = names;
|
|
295
304
|
}
|
|
296
305
|
|
|
306
|
+
/** Set (replace) or check (append) a table's column types, alongside its names. */
|
|
307
|
+
function applyFeatureTypes(manifest, featureTypes, replace) {
|
|
308
|
+
const incoming = featureTypes === undefined ? undefined : featureTypes.map((t) => String(t).trim());
|
|
309
|
+
if (incoming) validateColumnTypes(incoming, manifest.featureNames);
|
|
310
|
+
// An empty dataset takes whatever columns the first rows bring.
|
|
311
|
+
if (replace || manifest.stats.rowCount === 0) {
|
|
312
|
+
if (incoming && incoming.some((t) => t !== 'number')) manifest.featureTypes = incoming;
|
|
313
|
+
else delete manifest.featureTypes;
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
const current = columnTypesOf(manifest) ?? manifest.featureNames.map(() => 'number');
|
|
317
|
+
const want = incoming ?? manifest.featureNames.map(() => 'number');
|
|
318
|
+
if (current.join('\u0000') !== want.join('\u0000')) {
|
|
319
|
+
const describe = manifest.featureNames.map((n, k) => `${n} (${current[k]})`).join(', ');
|
|
320
|
+
throw new Error(`These columns don't match the dataset. It expects: ${describe}.`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
297
324
|
function normalizeRow(raw, manifest, width) {
|
|
298
325
|
if (raw == null || typeof raw !== 'object') {
|
|
299
326
|
throw new Error('Each row must be an object.');
|
|
@@ -316,8 +343,16 @@ function normalizeRow(raw, manifest, width) {
|
|
|
316
343
|
);
|
|
317
344
|
}
|
|
318
345
|
}
|
|
319
|
-
|
|
320
|
-
|
|
346
|
+
let features = raw.features;
|
|
347
|
+
const types = manifest.modality === 'tabular' ? columnTypesOf(manifest) : null;
|
|
348
|
+
if (types) {
|
|
349
|
+
features = normalizeTypedFeatures(raw.features, manifest.featureNames, types);
|
|
350
|
+
} else if (manifest.modality === 'tabular') {
|
|
351
|
+
// An object keyed by column name is fine for a named all-numbers table too.
|
|
352
|
+
if (features && typeof features === 'object' && !Array.isArray(features) && manifest.featureNames?.length) {
|
|
353
|
+
features = normalizeTypedFeatures(features, manifest.featureNames, manifest.featureNames.map(() => 'number'));
|
|
354
|
+
}
|
|
355
|
+
const f = features;
|
|
321
356
|
if (!Array.isArray(f) || f.length === 0 || f.some((n) => typeof n !== 'number' || !Number.isFinite(n))) {
|
|
322
357
|
throw new Error('Tabular rows need features that are all numbers (e.g. [12, 45, 8]).');
|
|
323
358
|
}
|
|
@@ -328,7 +363,7 @@ function normalizeRow(raw, manifest, width) {
|
|
|
328
363
|
if (manifest.modality === 'text' && (typeof raw.features !== 'string' || !raw.features.trim())) {
|
|
329
364
|
throw new Error('Text rows need some text.');
|
|
330
365
|
}
|
|
331
|
-
const row = { features
|
|
366
|
+
const row = { features, split };
|
|
332
367
|
if (manifest.taskType === 'classification') {
|
|
333
368
|
if (raw.label == null || String(raw.label).length === 0) {
|
|
334
369
|
throw new Error('Classification rows require a "label".');
|