@aci-metrics/score 0.0.1

package/test-model.js

@@ -0,0 +1,232 @@
+/**
+ * ACI Local Scorer - Model Test Script
+ *
+ * This script verifies that the inference abstraction layer works correctly:
+ * 1. Loads configuration from config/default.json
+ * 2. Creates the appropriate provider based on settings
+ * 3. Initializes the model
+ * 4. Tests JSON schema enforcement with a sample prompt
+ *
+ * Run with: npm test
+ *
+ * To test different configurations:
+ *   - Edit config/default.json to change model or provider
+ *   - Or use command line: node test-model.js --model=llama-3.2-3b --provider=ollama
+ */
+
+var path = require('path');
+var createProvider = require('./lib/provider-factory');
+
+/**
+ * Parse command line arguments
+ *
+ * Supports --model=<id> and --provider=<name> flags
+ *
+ * @returns {Object} Parsed options
+ */
+function parseArgs() {
+    var options = {};
+    var args = process.argv.slice(2);
+
+    for (var i = 0; i < args.length; i++) {
+        var arg = args[i];
+
+        // Handle --model=value format
+        if (arg.startsWith('--model=')) {
+            options.modelId = arg.substring('--model='.length);
+        }
+
+        // Handle --provider=value format
+        if (arg.startsWith('--provider=')) {
+            options.provider = arg.substring('--provider='.length);
+        }
+
+        // Handle --help flag
+        if (arg === '--help' || arg === '-h') {
+            options.help = true;
+        }
+    }
+
+    return options;
+}
+
+/**
+ * Show help message
+ */
+function showHelp() {
+    console.log('');
+    console.log('ACI Local Scorer - Model Test');
+    console.log('');
+    console.log('Usage: node test-model.js [options]');
+    console.log('');
+    console.log('Options:');
+    console.log('  --model=<id>       Override model from config');
+    console.log('                     Available: qwen2.5-1.5b, phi-3.5-mini, llama-3.2-3b, gemma-2-2b');
+    console.log('');
+    console.log('  --provider=<name>  Override inference provider');
+    console.log('                     Available: node-llama-cpp, ollama');
+    console.log('');
+    console.log('  --help, -h         Show this help message');
+    console.log('');
+    console.log('Examples:');
+    console.log('  node test-model.js                           # Use config defaults');
+    console.log('  node test-model.js --model=llama-3.2-3b      # Test with Llama');
+    console.log('  node test-model.js --provider=ollama         # Use Ollama backend');
+    console.log('');
+}
+
+/**
+ * Run the model test
+ *
+ * @param {Object} options - Command line options
+ */
+async function runTest(options) {
+    console.log('='.repeat(60));
+    console.log('ACI Local Scorer - Model Test');
+    console.log('='.repeat(60));
+    console.log('');
+
+    // Create the provider
+    console.log('Creating provider...');
+    var provider;
+    try {
+        provider = createProvider({
+            modelId: options.modelId,
+            provider: options.provider,
+            basePath: __dirname
+        });
+    } catch (error) {
+        console.error('');
+        console.error('Failed to create provider:');
+        console.error('  ' + error.message);
+        console.error('');
+        process.exit(1);
+    }
+
+    // Show provider info
+    var info = provider.getInfo();
+    console.log('');
+    console.log('Provider: ' + info.provider);
+    console.log('Model: ' + info.model);
+    console.log('');
+
+    // Initialize the provider
+    console.log('Initializing provider...');
+    console.log('');
+    try {
+        await provider.initialize();
+    } catch (error) {
+        console.error('');
+        console.error('Initialization failed:');
+        console.error('');
+        console.error(error.message);
+        console.error('');
+        process.exit(1);
+    }
+
+    // Define test schema
+    var testSchema = {
+        type: 'object',
+        properties: {
+            task_type: {
+                type: 'string',
+                enum: ['feature', 'bugfix', 'refactor', 'docs', 'test', 'config', 'other']
+            },
+            complexity: {
+                type: 'number',
+                minimum: 1,
+                maximum: 10
+            },
+            description: {
+                type: 'string'
+            }
+        },
+        required: ['task_type', 'complexity', 'description']
+    };
+
+    // Test prompt
+    var testPrompt = 'Classify this coding task: "Add error handling to the API endpoint and write unit tests". Respond with JSON containing task_type, complexity (1-10), and description.';
+
+    console.log('');
+    console.log('Testing JSON generation with schema enforcement...');
+    console.log('');
+    console.log('Prompt:');
+    console.log('  ' + testPrompt);
+    console.log('');
+
+    // Generate response
+    try {
+        console.log('Generating...');
+        console.log('');
+        var result = await provider.generate(testPrompt, testSchema);
+
+        console.log('Response:');
+        console.log('  task_type: ' + result.task_type);
+        console.log('  complexity: ' + result.complexity);
+        console.log('  description: ' + result.description);
+        console.log('');
+        console.log('JSON validation: PASSED');
+        console.log('');
+
+    } catch (error) {
+        console.error('');
+        console.error('Generation failed:');
+        console.error('  ' + error.message);
+        console.error('');
+        await provider.destroy();
+        process.exit(1);
+    }
+
+    // Clean up
+    console.log('Cleaning up...');
+    await provider.destroy();
+
+    // Success
+    console.log('');
+    console.log('='.repeat(60));
+    console.log('TEST RESULT: PASSED');
+    console.log('');
+    console.log('The local scorer is configured correctly!');
+    console.log('');
+    console.log('Current configuration:');
+    console.log('  Model: ' + info.model);
+    console.log('  Provider: ' + info.provider);
+    console.log('');
+    console.log('To change models, edit config/default.json');
+    console.log('='.repeat(60));
+}
+
+/**
+ * Main entry point
+ */
+async function main() {
+    var options = parseArgs();
+
+    // Show help if requested
+    if (options.help) {
+        showHelp();
+        process.exit(0);
+    }
+
+    // Run the test
+    try {
+        await runTest(options);
+        process.exit(0);
+    } catch (error) {
+        console.error('');
+        console.error('='.repeat(60));
+        console.error('TEST RESULT: FAILED');
+        console.error('');
+        console.error('Error: ' + error.message);
+        if (error.stack) {
+            console.error('');
+            console.error('Stack trace:');
+            console.error(error.stack);
+        }
+        console.error('='.repeat(60));
+        process.exit(1);
+    }
+}
+
+// Run main
+main();