expressible 0.1.0

package/dist/commands/setup.js

@@ -0,0 +1,15 @@
+import { getGlobalModelCacheDir, ensureDir } from '../utils/paths.js';
+import { success, info } from '../utils/display.js';
+export async function setupCommand() {
+    info('Setting up distill...');
+    const cacheDir = getGlobalModelCacheDir();
+    ensureDir(cacheDir);
+    info('Downloading embedding model (all-MiniLM-L6-v2, ~80MB)...');
+    const transformers = await import('@xenova/transformers');
+    transformers.env.cacheDir = cacheDir;
+    transformers.env.allowLocalModels = true;
+    await transformers.pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+    success('Embedding model downloaded and cached.');
+    success('Setup complete. You can now use distill offline.');
+}
+//# sourceMappingURL=setup.js.map

package/dist/commands/setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/commands/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAE9B,MAAM,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IAC1C,SAAS,CAAC,QAAQ,CAAC,CAAC;IAEpB,IAAI,CAAC,0DAA0D,CAAC,CAAC;IAEjE,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAC1D,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACrC,YAAY,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAEzC,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,yBAAyB,CAAC,CAAC;IAE7E,OAAO,CAAC,wCAAwC,CAAC,CAAC;IAClD,OAAO,CAAC,kDAAkD,CAAC,CAAC;AAC9D,CAAC"}

package/dist/commands/stats.d.ts

@@ -0,0 +1,2 @@
+export declare function statsCommand(): Promise<void>;
+//# sourceMappingURL=stats.d.ts.map

package/dist/commands/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/commands/stats.ts"],"names":[],"mappings":"AAOA,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAoDlD"}

package/dist/commands/stats.js

@@ -0,0 +1,52 @@
+import fs from 'node:fs';
+import { findTaskDir, getModelDir, getModelMetadataPath } from '../utils/paths.js';
+import { readConfig } from '../core/config.js';
+import { loadSamples, loadValidationResults } from '../core/data.js';
+import { heading, table } from '../utils/display.js';
+import { getDirectorySize, formatBytes } from '../utils/fs.js';
+export async function statsCommand() {
+    const taskDir = findTaskDir();
+    const config = readConfig(taskDir);
+    const samples = loadSamples(taskDir);
+    const validation = loadValidationResults(taskDir);
+    heading(`Distill Project: ${config.name}`);
+    const rows = [
+        ['Task type', config.type],
+        ['Training samples', String(samples.length)],
+    ];
+    // Model info
+    const metadataPath = getModelMetadataPath(taskDir);
+    if (fs.existsSync(metadataPath)) {
+        const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf-8'));
+        const trainedAt = new Date(metadata.trainedAt).toLocaleString();
+        rows.push(['Last trained', trainedAt]);
+        if (metadata.accuracy !== undefined) {
+            rows.push(['Model accuracy', `${(metadata.accuracy * 100).toFixed(1)}%`]);
+        }
+        if (metadata.categories) {
+            rows.push(['Categories', metadata.categories.join(', ')]);
+        }
+        const modelSize = getDirectorySize(getModelDir(taskDir));
+        rows.push(['Model size', formatBytes(modelSize)]);
+    }
+    else {
+        rows.push(['Model', 'Not trained yet']);
+    }
+    // Validation info
+    if (validation.items.length > 0) {
+        const reviewed = validation.items.filter((i) => i.reviewedAt).length;
+        const approved = validation.items.filter((i) => i.reviewedAt && i.approved === true).length;
+        const approvalRate = reviewed > 0 ? ((approved / reviewed) * 100).toFixed(1) : 'N/A';
+        rows.push(['Reviewed items', String(reviewed)]);
+        rows.push(['Approval rate', `${approvalRate}%`]);
+    }
+    else {
+        rows.push(['Reviewed items', '0']);
+    }
+    if (config.description) {
+        rows.push(['Description', config.description]);
+    }
+    table(rows);
+    console.log();
+}
+//# sourceMappingURL=stats.js.map

package/dist/commands/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/commands/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAc,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAElD,OAAO,CAAC,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAuB;QAC/B,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC;QAC1B,CAAC,kBAAkB,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;KAC7C,CAAC;IAEF,aAAa;IACb,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACnD,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,CAAC;QAChE,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC;QAEvC,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,kBAAkB;IAClB,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QAC5F,MAAM,YAAY,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAErF,IAAI,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,CAAC;IACZ,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}

package/dist/commands/train.d.ts

@@ -0,0 +1,2 @@
+export declare function trainCommand(): Promise<void>;
+//# sourceMappingURL=train.d.ts.map

package/dist/commands/train.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"train.d.ts","sourceRoot":"","sources":["../../src/commands/train.ts"],"names":[],"mappings":"AAOA,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAwElD"}

package/dist/commands/train.js

@@ -0,0 +1,63 @@
+import { findTaskDir } from '../utils/paths.js';
+import { readConfig } from '../core/config.js';
+import { loadSamples, getUniqueOutputCategories } from '../core/data.js';
+import { embedTexts } from '../core/embeddings.js';
+import { trainClassifier } from '../core/classifier.js';
+import { success, error, warn, info, heading, table } from '../utils/display.js';
+export async function trainCommand() {
+    const taskDir = findTaskDir();
+    const config = readConfig(taskDir);
+    const samples = loadSamples(taskDir);
+    // Check minimum samples
+    if (samples.length < 10) {
+        error(`Not enough training samples. You have ${samples.length}, but at least 10 are required.`);
+        info(`Add more samples with: expressible distill add`);
+        process.exit(1);
+    }
+    heading(`Training model "${config.name}"`);
+    info(`${samples.length} training samples loaded`);
+    const startTime = Date.now();
+    // Embed all inputs
+    info('Generating embeddings for training data...');
+    const inputs = samples.map((s) => s.input);
+    const embeddings = await embedTexts(inputs, taskDir);
+    info(`Embedded ${embeddings.length} samples`);
+    const categories = getUniqueOutputCategories(samples);
+    info(`Categories found: ${categories.join(', ')}`);
+    // Warn about categories with few samples before training
+    const categoryCounts = new Map();
+    for (const s of samples) {
+        categoryCounts.set(s.output, (categoryCounts.get(s.output) || 0) + 1);
+    }
+    for (const [cat, count] of categoryCounts) {
+        if (count < 5) {
+            warn(`Category "${cat}" has only ${count} sample(s). Aim for at least 10 per category.`);
+        }
+    }
+    const labels = samples.map((s) => s.output);
+    const result = await trainClassifier(embeddings, labels, taskDir);
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    heading('Training Complete');
+    table([
+        ['Samples', String(result.numSamples)],
+        ['Categories', result.categories.join(', ')],
+        ['Training accuracy', `${(result.accuracy * 100).toFixed(1)}%`],
+        ['Validation accuracy', `${(result.valAccuracy * 100).toFixed(1)}%`],
+        ['Epochs', String(result.epochs)],
+        ['Time elapsed', `${elapsed}s`],
+    ]);
+    // Contextual post-training warnings
+    const valFraction = result.numSamples < 30 ? 0.1 : 0.2;
+    const valSamples = Math.max(1, Math.floor(result.numSamples * valFraction));
+    if (valSamples < 5) {
+        warn(`Validation set is very small (${valSamples} sample(s)). Accuracy estimate may be unreliable. Add more training data.`);
+    }
+    else if (result.accuracy > 0.9 && result.valAccuracy < 0.5) {
+        warn(`High training accuracy (${(result.accuracy * 100).toFixed(0)}%) but low validation accuracy (${(result.valAccuracy * 100).toFixed(0)}%) suggests overfitting. Add more samples, especially for underrepresented categories.`);
+    }
+    else if (result.valAccuracy < 0.6) {
+        warn(`Validation accuracy is low (${(result.valAccuracy * 100).toFixed(0)}%). Consider adding more diverse training examples.`);
+    }
+    success('Model saved to model/');
+}
+//# sourceMappingURL=train.js.map

package/dist/commands/train.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"train.js","sourceRoot":"","sources":["../../src/commands/train.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAEjF,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAErC,wBAAwB;IACxB,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACxB,KAAK,CACH,yCAAyC,OAAO,CAAC,MAAM,iCAAiC,CACzF,CAAC;QACF,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,mBAAmB,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,0BAA0B,CAAC,CAAC;IAElD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,mBAAmB;IACnB,IAAI,CAAC,4CAA4C,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,YAAY,UAAU,CAAC,MAAM,UAAU,CAAC,CAAC;IAE9C,MAAM,UAAU,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,CAAC,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEnD,yDAAyD;IACzD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,cAAc,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,aAAa,GAAG,cAAc,KAAK,+CAA+C,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE7D,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC7B,KAAK,CAAC;QACJ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,mBAAmB,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC/D,CAAC,qBAAqB,EAAE,GAAG,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,cAAc,EAAE,GAAG,OAAO,GAAG,CAAC;KAChC,CAAC,CAAC;IAEH,oCAAoC;IACpC,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC;IAC5E,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,IAAI,CACF,iCAAiC,UAAU,2EAA2E,CACvH,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC,WAAW,GAAG,GAAG,EAAE,CAAC;QAC7D,IAAI,CACF,2BAA2B,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wFAAwF,CAC9N,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,CAAC,WAAW,GAAG,GAAG,EAAE,CAAC;QACpC,IAAI,CACF,+BAA+B,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qDAAqD,CAC1H,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,uBAAuB,CAAC,CAAC;AACnC,CAAC"}

package/dist/core/classifier.d.ts

@@ -0,0 +1,18 @@
+export interface TrainResult {
+    accuracy: number;
+    valAccuracy: number;
+    epochs: number;
+    numSamples: number;
+    categories: string[];
+}
+export declare function trainClassifier(embeddings: number[][], labels: string[], taskDir: string): Promise<TrainResult>;
+export interface PredictionResult {
+    category: string;
+    confidence: number;
+    allScores: {
+        category: string;
+        confidence: number;
+    }[];
+}
+export declare function predict(embedding: number[], taskDir: string): Promise<PredictionResult>;
+//# sourceMappingURL=classifier.d.ts.map

package/dist/core/classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../src/core/classifier.ts"],"names":[],"mappings":"AA6BA,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,WAAW,CAAC,CAqMtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD;AASD,wBAAsB,OAAO,CAC3B,SAAS,EAAE,MAAM,EAAE,EACnB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,gBAAgB,CAAC,CA6C3B"}

package/dist/core/classifier.js

@@ -0,0 +1,220 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { getModelDir, ensureDir } from '../utils/paths.js';
+import { EMBEDDING_DIM } from './embeddings.js';
+import { loadTf } from './tf.js';
+import { fileIOHandler } from './model-io.js';
+import { info, warn } from '../utils/display.js';
+/** Simple seeded PRNG (mulberry32) for reproducible shuffles */
+function seededRandom(seed) {
+    return () => {
+        seed |= 0;
+        seed = (seed + 0x6d2b79f5) | 0;
+        let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+export async function trainClassifier(embeddings, labels, taskDir) {
+    const tf = await loadTf();
+    const categories = Array.from(new Set(labels)).sort();
+    const numCategories = categories.length;
+    // Check for underrepresented classes
+    const classCounts = new Map();
+    for (const label of labels) {
+        classCounts.set(label, (classCounts.get(label) ?? 0) + 1);
+    }
+    for (const [category, count] of classCounts) {
+        if (count < 3) {
+            warn(`Category "${category}" has only ${count} example(s). Consider adding at least 3 examples per category.`);
+        }
+    }
+    // Convert labels to one-hot
+    const labelIndices = labels.map((l) => categories.indexOf(l));
+    const oneHot = labelIndices.map((idx) => {
+        const arr = new Array(numCategories).fill(0);
+        arr[idx] = 1;
+        return arr;
+    });
+    // Stratified train/val split — proportional representation of each class
+    // For small datasets (<30), use 90/10 split to keep more training data
+    // For larger datasets, use standard 80/20
+    const valFraction = embeddings.length < 30 ? 0.1 : 0.2;
+    // Deterministic seed derived from data
+    const dataHash = crypto.createHash('md5')
+        .update(labels.join('|'))
+        .digest()
+        .readUInt32LE(0);
+    const rand = seededRandom(dataHash);
+    // Group indices by category for stratified split
+    const categoryIndices = new Map();
+    for (let i = 0; i < labels.length; i++) {
+        const cat = labels[i];
+        if (!categoryIndices.has(cat))
+            categoryIndices.set(cat, []);
+        categoryIndices.get(cat).push(i);
+    }
+    const trainIndices = [];
+    const valIndices = [];
+    for (const [, indices] of categoryIndices) {
+        // Shuffle within category
+        for (let i = indices.length - 1; i > 0; i--) {
+            const j = Math.floor(rand() * (i + 1));
+            [indices[i], indices[j]] = [indices[j], indices[i]];
+        }
+        // Take proportional validation samples (at least 1 per class if possible)
+        const numValForClass = Math.max(1, Math.floor(indices.length * valFraction));
+        valIndices.push(...indices.slice(0, numValForClass));
+        trainIndices.push(...indices.slice(numValForClass));
+    }
+    const numTrain = trainIndices.length;
+    const numVal = valIndices.length;
+    const trainX = tf.tensor2d(trainIndices.map((i) => embeddings[i]));
+    const trainY = tf.tensor2d(trainIndices.map((i) => oneHot[i]));
+    const valX = tf.tensor2d(valIndices.map((i) => embeddings[i]));
+    const valY = tf.tensor2d(valIndices.map((i) => oneHot[i]));
+    // Build model
+    const model = tf.sequential();
+    model.add(tf.layers.dense({
+        inputShape: [EMBEDDING_DIM],
+        units: 128,
+        activation: 'relu',
+    }));
+    model.add(tf.layers.dropout({ rate: 0.2 }));
+    model.add(tf.layers.dense({
+        units: 64,
+        activation: 'relu',
+    }));
+    model.add(tf.layers.dropout({ rate: 0.2 }));
+    model.add(tf.layers.dense({
+        units: numCategories,
+        activation: 'softmax',
+    }));
+    model.compile({
+        optimizer: tf.train.adam(0.001),
+        loss: 'categoricalCrossentropy',
+        metrics: ['accuracy'],
+    });
+    info(`Training classifier with ${numTrain} samples, validating on ${numVal}...`);
+    // Train with early stopping logic
+    let bestValLoss = Infinity;
+    let patienceCounter = 0;
+    const patience = 10;
+    let bestEpoch = 0;
+    const maxEpochs = 100;
+    let finalAccuracy = 0;
+    let finalValAccuracy = 0;
+    let bestAccuracy = 0;
+    let bestValAccuracy = 0;
+    let totalEpochs = 0;
+    let bestWeights = null;
+    for (let epoch = 0; epoch < maxEpochs; epoch++) {
+        const history = await model.fit(trainX, trainY, {
+            epochs: 1,
+            validationData: [valX, valY],
+            verbose: 0,
+        });
+        const h = history.history;
+        const valLoss = (h['val_loss']?.[0] ?? h['val_Loss']?.[0] ?? 0);
+        const valAcc = (h['val_acc']?.[0] ?? h['val_accuracy']?.[0] ?? 0);
+        const trainAcc = (h['acc']?.[0] ?? h['accuracy']?.[0] ?? 0);
+        finalAccuracy = trainAcc;
+        finalValAccuracy = valAcc;
+        totalEpochs = epoch + 1;
+        if (valLoss < bestValLoss) {
+            bestValLoss = valLoss;
+            patienceCounter = 0;
+            bestEpoch = epoch;
+            bestAccuracy = trainAcc;
+            bestValAccuracy = valAcc;
+            // Checkpoint best weights
+            bestWeights = model.getWeights().map((w) => w.dataSync().slice().buffer);
+        }
+        else {
+            patienceCounter++;
+            if (patienceCounter >= patience) {
+                info(`Early stopping at epoch ${epoch + 1} (best at epoch ${bestEpoch + 1})`);
+                break;
+            }
+        }
+    }
+    // Restore best weights before saving
+    if (bestWeights && totalEpochs > bestEpoch + 1) {
+        const currentWeights = model.getWeights();
+        const restoredWeights = bestWeights.map((buf, i) => tf.tensor(new Float32Array(buf), currentWeights[i].shape));
+        model.setWeights(restoredWeights);
+        restoredWeights.forEach((w) => w.dispose());
+        finalAccuracy = bestAccuracy;
+        finalValAccuracy = bestValAccuracy;
+    }
+    // Save model (invalidate prediction cache)
+    cachedClassifier = null;
+    const modelDir = getModelDir(taskDir);
+    ensureDir(modelDir);
+    await model.save(fileIOHandler(modelDir));
+    // Save metadata
+    const metadata = {
+        type: 'classifier',
+        categories,
+        trainedAt: new Date().toISOString(),
+        numSamples: embeddings.length,
+        accuracy: finalValAccuracy,
+        epochs: totalEpochs,
+    };
+    fs.writeFileSync(path.join(modelDir, 'metadata.json'), JSON.stringify(metadata, null, 2) + '\n', 'utf-8');
+    // Clean up tensors
+    trainX.dispose();
+    trainY.dispose();
+    valX.dispose();
+    valY.dispose();
+    model.dispose();
+    return {
+        accuracy: finalAccuracy,
+        valAccuracy: finalValAccuracy,
+        epochs: totalEpochs,
+        numSamples: embeddings.length,
+        categories,
+    };
+}
+// Cache for loaded classifier model to avoid re-reading from disk on every predict call
+let cachedClassifier = null;
+export async function predict(embedding, taskDir) {
+    const tf = await loadTf();
+    const modelDir = getModelDir(taskDir);
+    const metadataPath = path.join(modelDir, 'metadata.json');
+    if (!fs.existsSync(metadataPath)) {
+        throw new Error('No trained model found. Run "expressible distill train" first.');
+    }
+    let model;
+    let metadata;
+    if (cachedClassifier && cachedClassifier.modelDir === modelDir) {
+        model = cachedClassifier.model;
+        metadata = cachedClassifier.metadata;
+    }
+    else {
+        if (cachedClassifier) {
+            cachedClassifier.model.dispose();
+        }
+        metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf-8'));
+        model = await tf.loadLayersModel(fileIOHandler(modelDir));
+        cachedClassifier = { modelDir, model, metadata };
+    }
+    const inputTensor = tf.tensor2d([embedding]);
+    const output = model.predict(inputTensor);
+    const scores = await output.data();
+    const allScores = metadata.categories.map((category, i) => ({
+        category,
+        confidence: scores[i],
+    }));
+    allScores.sort((a, b) => b.confidence - a.confidence);
+    const best = allScores[0];
+    inputTensor.dispose();
+    output.dispose();
+    return {
+        category: best.category,
+        confidence: best.confidence,
+        allScores,
+    };
+}
+//# sourceMappingURL=classifier.js.map

package/dist/core/classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../../src/core/classifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAW,MAAM,qBAAqB,CAAC;AAE1D,gEAAgE;AAChE,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,GAAG,EAAE;QACV,IAAI,IAAI,CAAC,CAAC;QACV,IAAI,GAAG,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAClD,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAmBD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAsB,EACtB,MAAgB,EAChB,OAAe;IAEf,MAAM,EAAE,GAAG,MAAM,MAAM,EAAE,CAAC;IAE1B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC;IAExC,qCAAqC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CACF,aAAa,QAAQ,cAAc,KAAK,gEAAgE,CACzG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC;QACzD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,uEAAuE;IACvE,0CAA0C;IAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAEvD,uCAAuC;IACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;SACtC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACxB,MAAM,EAAE;SACR,YAAY,CAAC,CAAC,CAAC,CAAC;IACnB,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEpC,iDAAiD;IACjD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC5D,eAAe,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,CAAC,EAAE,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC;QAC1C,0BAA0B;QAC1B,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;QAC7E,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;QACrD,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC;IACrC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAEjC,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3D,cAAc;IACd,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC;IAC9B,KAAK,CAAC,GAAG,CACP,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;QACd,UAAU,EAAE,CAAC,aAAa,CAAC;QAC3B,KAAK,EAAE,GAAG;QACV,UAAU,EAAE,MAAM;KACnB,CAAC,CACH,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5C,KAAK,CAAC,GAAG,CACP,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;QACd,KAAK,EAAE,EAAE;QACT,UAAU,EAAE,MAAM;KACnB,CAAC,CACH,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5C,KAAK,CAAC,GAAG,CACP,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;QACd,KAAK,EAAE,aAAa;QACpB,UAAU,EAAE,SAAS;KACtB,CAAC,CACH,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC;QACZ,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,EAAE,yBAAyB;QAC/B,OAAO,EAAE,CAAC,UAAU,CAAC;KACtB,CAAC,CAAC;IAEH,IAAI,CAAC,4BAA4B,QAAQ,2BAA2B,MAAM,KAAK,CAAC,CAAC;IAEjF,kCAAkC;IAClC,IAAI,WAAW,GAAG,QAAQ,CAAC;IAC3B,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,GAAG,CAAC;IAEtB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAyB,IAAI,CAAC;IAE7C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE;YAC9C,MAAM,EAAE,CAAC;YACT,cAAc,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;YAC5B,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAW,CAAC;QAC1E,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAW,CAAC;QAC5E,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAW,CAAC;QAEtE,aAAa,GAAG,QAAQ,CAAC;QACzB,gBAAgB,GAAG,MAAM,CAAC;QAC1B,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC;QAExB,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;YAC1B,WAAW,GAAG,OAAO,CAAC;YACtB,eAAe,GAAG,CAAC,CAAC;YACpB,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,QAAQ,CAAC;YACxB,eAAe,GAAG,MAAM,CAAC;YACzB,0BAA0B;YAC1B,WAAW,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,eAAe,EAAE,CAAC;YAClB,IAAI,eAAe,IAAI,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,2BAA2B,KAAK,GAAG,CAAC,mBAAmB,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9E,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,IAAI,WAAW,IAAI,WAAW,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CACjD,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAC1D,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAClC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5C,aAAa,GAAG,YAAY,CAAC;QAC7B,gBAAgB,GAAG,eAAe,CAAC;IACrC,CAAC;IAED,2CAA2C;IAC3C,gBAAgB,GAAG,IAAI,CAAC;IACxB,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAEpB,MAAM,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE1C,gBAAgB;IAChB,MAAM,QAAQ,GAAuB;QACnC,IAAI,EAAE,YAAY;QAClB,UAAU;QACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,UAAU,CAAC,MAAM;QAC7B,QAAQ,EAAE,gBAAgB;QAC1B,MAAM,EAAE,WAAW;KACpB,CAAC;IACF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,EACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EACxC,OAAO,CACR,CAAC;IAEF,mBAAmB;IACnB,MAAM,CAAC,OAAO,EAAE,CAAC;IACjB,MAAM,CAAC,OAAO,EAAE,CAAC;IACjB,IAAI,CAAC,OAAO,EAAE,CAAC;IACf,IAAI,CAAC,OAAO,EAAE,CAAC;IACf,KAAK,CAAC,OAAO,EAAE,CAAC;IAEhB,OAAO;QACL,QAAQ,EAAE,aAAa;QACvB,WAAW,EAAE,gBAAgB;QAC7B,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,UAAU,CAAC,MAAM;QAC7B,UAAU;KACX,CAAC;AACJ,CAAC;AAQD,wFAAwF;AACxF,IAAI,gBAAgB,GAIT,IAAI,CAAC;AAEhB,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,SAAmB,EACnB,OAAe;IAEf,MAAM,EAAE,GAAG,MAAM,MAAM,EAAE,CAAC;IAE1B,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE1D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,KAAqD,CAAC;IAC1D,IAAI,QAA4B,CAAC;IAEjC,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC/D,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC;QAC/B,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,IAAI,gBAAgB,EAAE,CAAC;YACrB,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,CAAC;QACD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9D,KAAK,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1D,gBAAgB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAiC,CAAC;IAC1E,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAEnC,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,QAAQ;QACR,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;IACJ,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEtD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAE1B,WAAW,CAAC,OAAO,EAAE,CAAC;IACtB,MAAM,CAAC,OAAO,EAAE,CAAC;IAEjB,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/core/config.d.ts

@@ -0,0 +1,11 @@
+export type TaskType = 'classify';
+export interface DistillConfig {
+    name: string;
+    type: TaskType;
+    description: string;
+    createdAt: string;
+    version: string;
+}
+export declare function readConfig(taskDir: string): DistillConfig;
+export declare function writeConfig(taskDir: string, config: DistillConfig): void;
+//# sourceMappingURL=config.d.ts.map

package/dist/core/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/core/config.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,QAAQ,GAAG,UAAU,CAAC;AAElC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,CASzD;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAGxE"}

package/dist/core/config.js

@@ -0,0 +1,15 @@
+import fs from 'node:fs';
+import { getConfigPath } from '../utils/paths.js';
+export function readConfig(taskDir) {
+    const configPath = getConfigPath(taskDir);
+    if (!fs.existsSync(configPath)) {
+        throw new Error(`No distill.config.json found in ${taskDir}. Are you in a distill project directory?`);
+    }
+    const raw = fs.readFileSync(configPath, 'utf-8');
+    return JSON.parse(raw);
+}
+export function writeConfig(taskDir, config) {
+    const configPath = getConfigPath(taskDir);
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf-8');
+}
+//# sourceMappingURL=config.js.map

package/dist/core/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/core/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAYlD,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,mCAAmC,OAAO,2CAA2C,CACtF,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAe,EAAE,MAAqB;IAChE,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChF,CAAC"}

package/dist/core/data.d.ts

@@ -0,0 +1,23 @@
+export interface SamplePair {
+    id: string;
+    input: string;
+    output: string;
+}
+export interface ValidationResult {
+    id: string;
+    input: string;
+    predictedOutput: string;
+    approved: boolean;
+    correctedOutput?: string;
+    reviewedAt?: string;
+}
+export interface ValidationResults {
+    items: ValidationResult[];
+}
+export declare function loadSamples(taskDir: string): SamplePair[];
+export declare function getNextSampleId(taskDir: string): string;
+export declare function saveSample(taskDir: string, id: string, input: string, output: string, isJson?: boolean): void;
+export declare function loadValidationResults(taskDir: string): ValidationResults;
+export declare function saveValidationResults(taskDir: string, results: ValidationResults): void;
+export declare function getUniqueOutputCategories(samples: SamplePair[]): string[];
+//# sourceMappingURL=data.d.ts.map

package/dist/core/data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/core/data.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE,CA4BzD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAgBvD;AAED,wBAAgB,UAAU,CACxB,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,OAAe,GACtB,IAAI,CAON;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,iBAAiB,CAOxE;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,IAAI,CAKvF;AAED,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAGzE"}

package/dist/core/data.js

@@ -0,0 +1,66 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { getSamplesDir, getValidationResultsPath } from '../utils/paths.js';
+export function loadSamples(taskDir) {
+    const samplesDir = getSamplesDir(taskDir);
+    if (!fs.existsSync(samplesDir)) {
+        return [];
+    }
+    const files = fs.readdirSync(samplesDir);
+    const inputFiles = files
+        .filter((f) => f.includes('.input.'))
+        .sort();
+    const samples = [];
+    for (const inputFile of inputFiles) {
+        const id = inputFile.split('.input.')[0];
+        const outputFile = files.find((f) => f.startsWith(id + '.output.'));
+        if (!outputFile) {
+            continue;
+        }
+        const input = fs.readFileSync(path.join(samplesDir, inputFile), 'utf-8').trim();
+        const output = fs.readFileSync(path.join(samplesDir, outputFile), 'utf-8').trim();
+        samples.push({ id, input, output });
+    }
+    return samples;
+}
+export function getNextSampleId(taskDir) {
+    const samplesDir = getSamplesDir(taskDir);
+    if (!fs.existsSync(samplesDir)) {
+        return '001';
+    }
+    const files = fs.readdirSync(samplesDir);
+    const ids = files
+        .filter((f) => f.includes('.input.'))
+        .map((f) => parseInt(f.split('.input.')[0], 10))
+        .filter((n) => !isNaN(n));
+    if (ids.length === 0)
+        return '001';
+    const max = Math.max(...ids);
+    return String(max + 1).padStart(3, '0');
+}
+export function saveSample(taskDir, id, input, output, isJson = false) {
+    const samplesDir = getSamplesDir(taskDir);
+    fs.mkdirSync(samplesDir, { recursive: true });
+    const ext = isJson ? 'json' : 'txt';
+    fs.writeFileSync(path.join(samplesDir, `${id}.input.${ext}`), input, 'utf-8');
+    fs.writeFileSync(path.join(samplesDir, `${id}.output.${ext}`), output, 'utf-8');
+}
+export function loadValidationResults(taskDir) {
+    const resultsPath = getValidationResultsPath(taskDir);
+    if (!fs.existsSync(resultsPath)) {
+        return { items: [] };
+    }
+    const raw = fs.readFileSync(resultsPath, 'utf-8');
+    return JSON.parse(raw);
+}
+export function saveValidationResults(taskDir, results) {
+    const resultsPath = getValidationResultsPath(taskDir);
+    const dir = path.dirname(resultsPath);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(resultsPath, JSON.stringify(results, null, 2) + '\n', 'utf-8');
+}
+export function getUniqueOutputCategories(samples) {
+    const categories = new Set(samples.map((s) => s.output));
+    return Array.from(categories).sort();
+}
+//# sourceMappingURL=data.js.map

package/dist/core/data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.js","sourceRoot":"","sources":["../../src/core/data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAqB5E,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,KAAK;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SACpC,IAAI,EAAE,CAAC;IAEV,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;QAEpE,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAChF,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAElF,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,KAAK;SACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;SAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,OAAe,EACf,EAAU,EACV,KAAa,EACb,MAAc,EACd,SAAkB,KAAK;IAEvB,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;IACpC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,UAAU,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9E,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,WAAW,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACvB,CAAC;IACD,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe,EAAE,OAA0B;IAC/E,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,OAAqB;IAC7D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC"}

package/dist/core/embeddings.d.ts

@@ -0,0 +1,4 @@
+export declare function embedTexts(texts: string[], taskDir: string): Promise<number[][]>;
+export declare function embedText(text: string, taskDir: string): Promise<number[]>;
+export declare const EMBEDDING_DIM = 384;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/core/embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/core/embeddings.ts"],"names":[],"mappings":"AAmEA,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAiCrB;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGhF;AAED,eAAO,MAAM,aAAa,MAAM,CAAC"}