@neural-tools/fine-tune 0.1.3 → 0.1.5

package/dist/index.d.mts

@@ -0,0 +1,71 @@
+interface FineTuneConfig {
+    provider: 'openai' | 'anthropic' | 'custom';
+    model: string;
+    datasetPath: string;
+    validationSplit?: number;
+    epochs?: number;
+    learningRate?: number;
+    batchSize?: number;
+}
+interface TrainingExample {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+}
+interface FineTuneJob {
+    id: string;
+    status: 'pending' | 'running' | 'completed' | 'failed';
+    model: string;
+    trainingFile?: string;
+    validationFile?: string;
+    createdAt: number;
+    finishedAt?: number;
+    error?: string;
+}
+/**
+ * Prepare training data for fine-tuning
+ */
+declare function prepareTrainingData(examples: TrainingExample[], options?: {
+    validationSplit?: number;
+    outputDir?: string;
+}): Promise<{
+    trainingFile: string;
+    validationFile?: string;
+}>;
+/**
+ * Validate training data format
+ */
+declare function validateTrainingData(examples: TrainingExample[]): {
+    valid: boolean;
+    errors: string[];
+};
+/**
+ * Create a fine-tune job (placeholder - actual implementation would call provider APIs)
+ */
+declare function createFineTuneJob(config: FineTuneConfig): Promise<FineTuneJob>;
+/**
+ * Get fine-tune job status (placeholder)
+ */
+declare function getFineTuneJob(jobId: string): Promise<FineTuneJob>;
+/**
+ * Convert conversations to training examples
+ */
+declare function conversationsToExamples(conversations: Array<{
+    system?: string;
+    messages: Array<{
+        role: 'user' | 'assistant';
+        content: string;
+    }>;
+}>): TrainingExample[];
+/**
+ * Calculate dataset statistics
+ */
+declare function analyzeDataset(examples: TrainingExample[]): {
+    totalExamples: number;
+    avgMessagesPerExample: number;
+    avgTokensPerMessage: number;
+    roleDistribution: Record<string, number>;
+};
+
+export { type FineTuneConfig, type FineTuneJob, type TrainingExample, analyzeDataset, conversationsToExamples, createFineTuneJob, getFineTuneJob, prepareTrainingData, validateTrainingData };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export interface FineTuneConfig {
+interface FineTuneConfig {
     provider: 'openai' | 'anthropic' | 'custom';
     model: string;
     datasetPath: string;
@@ -7,13 +7,13 @@ export interface FineTuneConfig {
     learningRate?: number;
     batchSize?: number;
 }
-export interface TrainingExample {
+interface TrainingExample {
     messages: Array<{
         role: 'system' | 'user' | 'assistant';
         content: string;
     }>;
 }
-export interface FineTuneJob {
+interface FineTuneJob {
     id: string;
     status: 'pending' | 'running' | 'completed' | 'failed';
     model: string;
@@ -26,7 +26,7 @@ export interface FineTuneJob {
 /**
  * Prepare training data for fine-tuning
  */
-export declare function prepareTrainingData(examples: TrainingExample[], options?: {
+declare function prepareTrainingData(examples: TrainingExample[], options?: {
     validationSplit?: number;
     outputDir?: string;
 }): Promise<{
@@ -36,22 +36,22 @@ export declare function prepareTrainingData(examples: TrainingExample[], options
 /**
  * Validate training data format
  */
-export declare function validateTrainingData(examples: TrainingExample[]): {
+declare function validateTrainingData(examples: TrainingExample[]): {
     valid: boolean;
     errors: string[];
 };
 /**
  * Create a fine-tune job (placeholder - actual implementation would call provider APIs)
  */
-export declare function createFineTuneJob(config: FineTuneConfig): Promise<FineTuneJob>;
+declare function createFineTuneJob(config: FineTuneConfig): Promise<FineTuneJob>;
 /**
  * Get fine-tune job status (placeholder)
  */
-export declare function getFineTuneJob(jobId: string): Promise<FineTuneJob>;
+declare function getFineTuneJob(jobId: string): Promise<FineTuneJob>;
 /**
  * Convert conversations to training examples
  */
-export declare function conversationsToExamples(conversations: Array<{
+declare function conversationsToExamples(conversations: Array<{
     system?: string;
     messages: Array<{
         role: 'user' | 'assistant';
@@ -61,9 +61,11 @@ export declare function conversationsToExamples(conversations: Array<{
 /**
  * Calculate dataset statistics
  */
-export declare function analyzeDataset(examples: TrainingExample[]): {
+declare function analyzeDataset(examples: TrainingExample[]): {
     totalExamples: number;
     avgMessagesPerExample: number;
     avgTokensPerMessage: number;
     roleDistribution: Record<string, number>;
 };
+
+export { type FineTuneConfig, type FineTuneJob, type TrainingExample, analyzeDataset, conversationsToExamples, createFineTuneJob, getFineTuneJob, prepareTrainingData, validateTrainingData };

package/dist/index.js

@@ -1,150 +1,3 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.prepareTrainingData = prepareTrainingData;
-exports.validateTrainingData = validateTrainingData;
-exports.createFineTuneJob = createFineTuneJob;
-exports.getFineTuneJob = getFineTuneJob;
-exports.conversationsToExamples = conversationsToExamples;
-exports.analyzeDataset = analyzeDataset;
-const core_1 = require("@neural-tools/core");
-const promises_1 = __importDefault(require("fs/promises"));
-const path_1 = __importDefault(require("path"));
-/**
- * Prepare training data for fine-tuning
- */
-async function prepareTrainingData(examples, options = {}) {
-    await (0, core_1.requireFeature)('fine-tuning', 'Fine-tuning');
-    const validationSplit = options.validationSplit || 0;
-    const outputDir = options.outputDir || './fine-tune-data';
-    // Create output directory
-    await promises_1.default.mkdir(outputDir, { recursive: true });
-    // Split data
-    const splitIndex = Math.floor(examples.length * (1 - validationSplit));
-    const trainingExamples = examples.slice(0, splitIndex);
-    const validationExamples = validationSplit > 0 ? examples.slice(splitIndex) : [];
-    // Write training file
-    const trainingFile = path_1.default.join(outputDir, 'training.jsonl');
-    const trainingContent = trainingExamples
-        .map(ex => JSON.stringify(ex))
-        .join('\n');
-    await promises_1.default.writeFile(trainingFile, trainingContent, 'utf-8');
-    // Write validation file if needed
-    let validationFile;
-    if (validationExamples.length > 0) {
-        validationFile = path_1.default.join(outputDir, 'validation.jsonl');
-        const validationContent = validationExamples
-            .map(ex => JSON.stringify(ex))
-            .join('\n');
-        await promises_1.default.writeFile(validationFile, validationContent, 'utf-8');
-    }
-    return { trainingFile, validationFile };
-}
-/**
- * Validate training data format
- */
-function validateTrainingData(examples) {
-    const errors = [];
-    if (examples.length === 0) {
-        errors.push('No training examples provided');
-        return { valid: false, errors };
-    }
-    examples.forEach((example, index) => {
-        if (!example.messages || !Array.isArray(example.messages)) {
-            errors.push(`Example ${index}: Missing or invalid messages array`);
-            return;
-        }
-        if (example.messages.length === 0) {
-            errors.push(`Example ${index}: Messages array is empty`);
-        }
-        example.messages.forEach((message, msgIndex) => {
-            if (!message.role || !['system', 'user', 'assistant'].includes(message.role)) {
-                errors.push(`Example ${index}, Message ${msgIndex}: Invalid role`);
-            }
-            if (!message.content || typeof message.content !== 'string') {
-                errors.push(`Example ${index}, Message ${msgIndex}: Missing or invalid content`);
-            }
-        });
-    });
-    return {
-        valid: errors.length === 0,
-        errors
-    };
-}
-/**
- * Create a fine-tune job (placeholder - actual implementation would call provider APIs)
- */
-async function createFineTuneJob(config) {
-    await (0, core_1.requireFeature)('fine-tuning', 'Fine-tuning');
-    // Validate dataset exists
-    try {
-        await promises_1.default.access(config.datasetPath);
-    }
-    catch {
-        throw new Error(`Dataset file not found: ${config.datasetPath}`);
-    }
-    // In production, this would:
-    // 1. Upload training data to provider
-    // 2. Start fine-tuning job
-    // 3. Return job details
-    const job = {
-        id: `ft-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
-        status: 'pending',
-        model: config.model,
-        trainingFile: config.datasetPath,
-        createdAt: Date.now()
-    };
-    return job;
-}
-/**
- * Get fine-tune job status (placeholder)
- */
-async function getFineTuneJob(jobId) {
-    await (0, core_1.requireFeature)('fine-tuning', 'Fine-tuning');
-    // In production, this would query the provider API
-    throw new Error('Fine-tune job tracking coming soon');
-}
-/**
- * Convert conversations to training examples
- */
-function conversationsToExamples(conversations) {
-    return conversations.map(conv => {
-        const messages = [];
-        if (conv.system) {
-            messages.push({
-                role: 'system',
-                content: conv.system
-            });
-        }
-        messages.push(...conv.messages);
-        return { messages };
-    });
-}
-/**
- * Calculate dataset statistics
- */
-function analyzeDataset(examples) {
-    const stats = {
-        totalExamples: examples.length,
-        avgMessagesPerExample: 0,
-        avgTokensPerMessage: 0,
-        roleDistribution: {}
-    };
-    let totalMessages = 0;
-    let totalTokens = 0;
-    examples.forEach(example => {
-        totalMessages += example.messages.length;
-        example.messages.forEach(message => {
-            // Simple token estimation (real implementation would use tiktoken)
-            const tokens = message.content.split(/\s+/).length;
-            totalTokens += tokens;
-            stats.roleDistribution[message.role] =
-                (stats.roleDistribution[message.role] || 0) + 1;
-        });
-    });
-    stats.avgMessagesPerExample = totalMessages / examples.length;
-    stats.avgTokensPerMessage = totalTokens / totalMessages;
-    return stats;
-}
+"use strict";var v=Object.create;var l=Object.defineProperty;var E=Object.getOwnPropertyDescriptor;var y=Object.getOwnPropertyNames;var F=Object.getPrototypeOf,T=Object.prototype.hasOwnProperty;var x=(n,t)=>{for(var e in t)l(n,e,{get:t[e],enumerable:!0})},d=(n,t,e,r)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of y(t))!T.call(n,i)&&i!==e&&l(n,i,{get:()=>t[i],enumerable:!(r=E(t,i))||r.enumerable});return n};var f=(n,t,e)=>(e=n!=null?v(F(n)):{},d(t||!n||!n.__esModule?l(e,"default",{value:n,enumerable:!0}):e,n)),w=n=>d(l({},"__esModule",{value:!0}),n);var J={};x(J,{analyzeDataset:()=>A,conversationsToExamples:()=>$,createFineTuneJob:()=>P,getFineTuneJob:()=>j,prepareTrainingData:()=>M,validateTrainingData:()=>D});module.exports=w(J);var g=require("@neural-tools/core"),s=f(require("fs/promises")),p=f(require("path"));async function M(n,t={}){await(0,g.requireFeature)("fine-tuning","Fine-tuning");let e=t.validationSplit||0,r=t.outputDir||"./fine-tune-data";await s.default.mkdir(r,{recursive:!0});let i=Math.floor(n.length*(1-e)),a=n.slice(0,i),o=e>0?n.slice(i):[],c=p.default.join(r,"training.jsonl"),h=a.map(m=>JSON.stringify(m)).join(`
+`);await s.default.writeFile(c,h,"utf-8");let u;if(o.length>0){u=p.default.join(r,"validation.jsonl");let m=o.map(b=>JSON.stringify(b)).join(`
+`);await s.default.writeFile(u,m,"utf-8")}return{trainingFile:c,validationFile:u}}function D(n){let t=[];return n.length===0?(t.push("No training examples provided"),{valid:!1,errors:t}):(n.forEach((e,r)=>{if(!e.messages||!Array.isArray(e.messages)){t.push(`Example ${r}: Missing or invalid messages array`);return}e.messages.length===0&&t.push(`Example ${r}: Messages array is empty`),e.messages.forEach((i,a)=>{(!i.role||!["system","user","assistant"].includes(i.role))&&t.push(`Example ${r}, Message ${a}: Invalid role`),(!i.content||typeof i.content!="string")&&t.push(`Example ${r}, Message ${a}: Missing or invalid content`)})}),{valid:t.length===0,errors:t})}async function P(n){await(0,g.requireFeature)("fine-tuning","Fine-tuning");try{await s.default.access(n.datasetPath)}catch{throw new Error(`Dataset file not found: ${n.datasetPath}`)}return{id:`ft-${Date.now()}-${Math.random().toString(36).substr(2,9)}`,status:"pending",model:n.model,trainingFile:n.datasetPath,createdAt:Date.now()}}async function j(n){throw await(0,g.requireFeature)("fine-tuning","Fine-tuning"),new Error("Fine-tune job tracking coming soon")}function $(n){return n.map(t=>{let e=[];return t.system&&e.push({role:"system",content:t.system}),e.push(...t.messages),{messages:e}})}function A(n){let t={totalExamples:n.length,avgMessagesPerExample:0,avgTokensPerMessage:0,roleDistribution:{}},e=0,r=0;return n.forEach(i=>{e+=i.messages.length,i.messages.forEach(a=>{let o=a.content.split(/\s+/).length;r+=o,t.roleDistribution[a.role]=(t.roleDistribution[a.role]||0)+1})}),t.avgMessagesPerExample=e/n.length,t.avgTokensPerMessage=r/e,t}0&&(module.exports={analyzeDataset,conversationsToExamples,createFineTuneJob,getFineTuneJob,prepareTrainingData,validateTrainingData});

package/dist/index.mjs

@@ -0,0 +1,3 @@
+import{requireFeature as u}from"@neural-tools/core";import o from"fs/promises";import p from"path";async function v(n,t={}){await u("fine-tuning","Fine-tuning");let e=t.validationSplit||0,i=t.outputDir||"./fine-tune-data";await o.mkdir(i,{recursive:!0});let r=Math.floor(n.length*(1-e)),a=n.slice(0,r),s=e>0?n.slice(r):[],m=p.join(i,"training.jsonl"),c=a.map(g=>JSON.stringify(g)).join(`
+`);await o.writeFile(m,c,"utf-8");let l;if(s.length>0){l=p.join(i,"validation.jsonl");let g=s.map(d=>JSON.stringify(d)).join(`
+`);await o.writeFile(l,g,"utf-8")}return{trainingFile:m,validationFile:l}}function E(n){let t=[];return n.length===0?(t.push("No training examples provided"),{valid:!1,errors:t}):(n.forEach((e,i)=>{if(!e.messages||!Array.isArray(e.messages)){t.push(`Example ${i}: Missing or invalid messages array`);return}e.messages.length===0&&t.push(`Example ${i}: Messages array is empty`),e.messages.forEach((r,a)=>{(!r.role||!["system","user","assistant"].includes(r.role))&&t.push(`Example ${i}, Message ${a}: Invalid role`),(!r.content||typeof r.content!="string")&&t.push(`Example ${i}, Message ${a}: Missing or invalid content`)})}),{valid:t.length===0,errors:t})}async function y(n){await u("fine-tuning","Fine-tuning");try{await o.access(n.datasetPath)}catch{throw new Error(`Dataset file not found: ${n.datasetPath}`)}return{id:`ft-${Date.now()}-${Math.random().toString(36).substr(2,9)}`,status:"pending",model:n.model,trainingFile:n.datasetPath,createdAt:Date.now()}}async function F(n){throw await u("fine-tuning","Fine-tuning"),new Error("Fine-tune job tracking coming soon")}function T(n){return n.map(t=>{let e=[];return t.system&&e.push({role:"system",content:t.system}),e.push(...t.messages),{messages:e}})}function x(n){let t={totalExamples:n.length,avgMessagesPerExample:0,avgTokensPerMessage:0,roleDistribution:{}},e=0,i=0;return n.forEach(r=>{e+=r.messages.length,r.messages.forEach(a=>{let s=a.content.split(/\s+/).length;i+=s,t.roleDistribution[a.role]=(t.roleDistribution[a.role]||0)+1})}),t.avgMessagesPerExample=e/n.length,t.avgTokensPerMessage=i/e,t}export{x as analyzeDataset,T as conversationsToExamples,y as createFineTuneJob,F as getFineTuneJob,v as prepareTrainingData,E as validateTrainingData};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neural-tools/fine-tune",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Fine-tuning utilities for Neural Tools",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -10,11 +10,23 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/MacLeanLuke/ai-toolkit.git",
+    "url": "https://github.com/MacLeanLuke/neural-tools.git",
     "directory": "packages/fine-tune"
   },
+  "homepage": "https://neural-tools.com/docs/fine-tune.html",
+  "bugs": {
+    "url": "https://github.com/MacLeanLuke/neural-tools/issues"
+  },
+  "keywords": [
+    "fine-tuning",
+    "openai",
+    "llm",
+    "neural-tools",
+    "machine-learning",
+    "ai"
+  ],
   "dependencies": {
-    "@neural-tools/core": "0.1.3"
+    "@neural-tools/core": "0.1.5"
   },
   "devDependencies": {
     "@types/node": "^20.11.5",
@@ -32,8 +44,8 @@
     "dist"
   ],
   "scripts": {
-    "build": "tsc",
-    "dev": "tsc --watch",
+    "build": "tsup",
+    "dev": "tsup --watch",
     "clean": "rm -rf dist",
     "test": "echo 'Tests coming soon'"
   }