@aci-metrics/score 0.0.1

package/config/default.json

@@ -0,0 +1,29 @@
+{
+  "model": {
+    "id": "qwen2.5-1.5b",
+    "comment": "Change this to switch models: qwen2.5-1.5b, phi-3.5-mini, llama-3.2-3b, gemma-2-2b"
+  },
+  "runtime": {
+    "provider": "node-llama-cpp",
+    "comment": "Options: node-llama-cpp (local GGUF) or ollama (HTTP API)"
+  },
+  "inference": {
+    "temperature": 0.1,
+    "maxTokens": 1024,
+    "comment": "Low temperature for consistent structured output"
+  },
+  "schemas": {
+    "taskClassification": "schemas/task-classification.json",
+    "archetypeAssignment": "schemas/archetype-assignment.json",
+    "comment": "Paths to JSON schema files for grammar enforcement"
+  },
+  "paths": {
+    "modelsDir": "models",
+    "promptsDir": "prompts",
+    "schemasDir": "schemas"
+  },
+  "ollama": {
+    "host": "http://localhost:11434",
+    "comment": "Ollama server endpoint (only used when runtime.provider is ollama)"
+  }
+}

package/config/models.json

@@ -0,0 +1,49 @@
+{
+  "models": {
+    "qwen2.5-1.5b": {
+      "id": "qwen2.5-1.5b",
+      "name": "Qwen 2.5 1.5B Instruct",
+      "filename": "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf",
+      "huggingfaceRepo": "bartowski/Qwen2.5-1.5B-Instruct-GGUF",
+      "downloadCommand": "huggingface-cli download bartowski/Qwen2.5-1.5B-Instruct-GGUF Qwen2.5-1.5B-Instruct-Q4_K_M.gguf --local-dir ./models",
+      "promptTemplate": "prompts/qwen.txt",
+      "ollamaModel": "qwen2.5:1.5b",
+      "sizeGB": 0.9,
+      "description": "Excellent instruction following, reliable JSON output. Good balance of speed and quality."
+    },
+    "phi-3.5-mini": {
+      "id": "phi-3.5-mini",
+      "name": "Phi 3.5 Mini Instruct",
+      "filename": "Phi-3.5-mini-instruct-Q4_K_M.gguf",
+      "huggingfaceRepo": "bartowski/Phi-3.5-mini-instruct-GGUF",
+      "downloadCommand": "huggingface-cli download bartowski/Phi-3.5-mini-instruct-GGUF Phi-3.5-mini-instruct-Q4_K_M.gguf --local-dir ./models",
+      "promptTemplate": "prompts/phi.txt",
+      "ollamaModel": "phi3.5:3.8b",
+      "sizeGB": 2.2,
+      "description": "Microsoft's compact model. Strong reasoning capabilities for its size."
+    },
+    "llama-3.2-3b": {
+      "id": "llama-3.2-3b",
+      "name": "Llama 3.2 3B Instruct",
+      "filename": "Llama-3.2-3B-Instruct-Q4_K_M.gguf",
+      "huggingfaceRepo": "bartowski/Llama-3.2-3B-Instruct-GGUF",
+      "downloadCommand": "huggingface-cli download bartowski/Llama-3.2-3B-Instruct-GGUF Llama-3.2-3B-Instruct-Q4_K_M.gguf --local-dir ./models",
+      "promptTemplate": "prompts/llama.txt",
+      "ollamaModel": "llama3.2:3b",
+      "sizeGB": 1.8,
+      "description": "Meta's latest small model. Excellent general capabilities."
+    },
+    "gemma-2-2b": {
+      "id": "gemma-2-2b",
+      "name": "Gemma 2 2B Instruct",
+      "filename": "gemma-2-2b-it-Q4_K_M.gguf",
+      "huggingfaceRepo": "bartowski/gemma-2-2b-it-GGUF",
+      "downloadCommand": "huggingface-cli download bartowski/gemma-2-2b-it-GGUF gemma-2-2b-it-Q4_K_M.gguf --local-dir ./models",
+      "promptTemplate": "prompts/gemma.txt",
+      "ollamaModel": "gemma2:2b",
+      "sizeGB": 1.4,
+      "description": "Google's efficient model. Fast inference with good quality."
+    }
+  },
+  "default": "qwen2.5-1.5b"
+}

package/lib/provider-factory.js

@@ -0,0 +1,181 @@
+/**
+ * Provider Factory - Creates the appropriate inference provider
+ *
+ * This module reads the configuration files and returns the correct
+ * provider instance based on the settings. It's the main entry point
+ * for getting an inference provider.
+ *
+ * Usage:
+ *   const createProvider = require('./lib/provider-factory');
+ *   const provider = createProvider();
+ *   await provider.initialize();
+ *   const result = await provider.generate(prompt, schema);
+ *   await provider.destroy();
+ *
+ * To switch models or runtimes, edit config/default.json:
+ *   - Change model.id to use a different model
+ *   - Change runtime.provider to use a different inference backend
+ */
+
+var fs = require('fs');
+var path = require('path');
+
+// Import provider implementations
+var NodeLlamaCppProvider = require('./providers/node-llama-cpp');
+var OllamaProvider = require('./providers/ollama');
+
+/**
+ * Load and parse a JSON configuration file
+ *
+ * @param {string} filePath - Path to the JSON file
+ * @returns {Object} Parsed JSON content
+ * @throws {Error} If file not found or invalid JSON
+ */
+function loadConfig(filePath) {
+    if (!fs.existsSync(filePath)) {
+        throw new Error('Configuration file not found: ' + filePath);
+    }
+
+    var content = fs.readFileSync(filePath, 'utf-8');
+
+    try {
+        return JSON.parse(content);
+    } catch (parseError) {
+        throw new Error('Invalid JSON in ' + filePath + ': ' + parseError.message);
+    }
+}
+
+/**
+ * Create an inference provider based on configuration
+ *
+ * Reads config/default.json and config/models.json to determine
+ * which provider to instantiate and with what settings.
+ *
+ * @param {Object} [options] - Optional overrides
+ * @param {string} [options.modelId] - Override the model ID
+ * @param {string} [options.provider] - Override the runtime provider
+ * @param {string} [options.basePath] - Override the base path for file resolution
+ * @returns {BaseProvider} Configured provider instance (not yet initialized)
+ */
+function createProvider(options) {
+    // Default options
+    options = options || {};
+
+    // Determine base path (where config and models directories are)
+    var basePath = options.basePath || path.join(__dirname, '..');
+
+    // Load configuration files
+    var configPath = path.join(basePath, 'config', 'default.json');
+    var modelsPath = path.join(basePath, 'config', 'models.json');
+
+    console.log('[provider-factory] Loading configuration...');
+    var config = loadConfig(configPath);
+    var modelsRegistry = loadConfig(modelsPath);
+
+    // Determine which model to use
+    var modelId = options.modelId || config.model.id || modelsRegistry.default;
+    console.log('[provider-factory] Selected model: ' + modelId);
+
+    // Get model configuration
+    var modelConfig = modelsRegistry.models[modelId];
+    if (!modelConfig) {
+        var availableModels = Object.keys(modelsRegistry.models);
+        throw new Error(
+            'Unknown model: ' + modelId + '\n' +
+            'Available models: ' + availableModels.join(', ')
+        );
+    }
+
+    // Determine which provider to use
+    var providerType = options.provider || config.runtime.provider || 'node-llama-cpp';
+    console.log('[provider-factory] Selected provider: ' + providerType);
+
+    // Create the appropriate provider
+    var provider;
+    switch (providerType) {
+        case 'node-llama-cpp':
+            provider = new NodeLlamaCppProvider(config, modelConfig, basePath);
+            break;
+
+        case 'ollama':
+            provider = new OllamaProvider(config, modelConfig, basePath);
+            break;
+
+        default:
+            throw new Error(
+                'Unknown provider: ' + providerType + '\n' +
+                'Available providers: node-llama-cpp, ollama'
+            );
+    }
+
+    console.log('[provider-factory] Provider created successfully');
+    return provider;
+}
+
+/**
+ * Get list of available models
+ *
+ * @param {string} [basePath] - Base path for config files
+ * @returns {Object} Models registry with all available models
+ */
+function getAvailableModels(basePath) {
+    basePath = basePath || path.join(__dirname, '..');
+    var modelsPath = path.join(basePath, 'config', 'models.json');
+    return loadConfig(modelsPath);
+}
+
+/**
+ * Get current configuration
+ *
+ * @param {string} [basePath] - Base path for config files
+ * @returns {Object} Current runtime configuration
+ */
+function getConfig(basePath) {
+    basePath = basePath || path.join(__dirname, '..');
+    var configPath = path.join(basePath, 'config', 'default.json');
+    return loadConfig(configPath);
+}
+
+/**
+ * Load a prompt template and replace placeholders
+ *
+ * @param {string} templatePath - Path to the template file (relative to basePath)
+ * @param {Object} variables - Key-value pairs to replace in the template
+ * @param {string} [basePath] - Base path for file resolution
+ * @returns {string} Processed prompt with placeholders replaced
+ */
+function loadPromptTemplate(templatePath, variables, basePath) {
+    basePath = basePath || path.join(__dirname, '..');
+
+    var fullPath = path.join(basePath, templatePath);
+
+    if (!fs.existsSync(fullPath)) {
+        throw new Error('Prompt template not found: ' + fullPath);
+    }
+
+    var template = fs.readFileSync(fullPath, 'utf-8');
+
+    // Replace all {{VARIABLE}} placeholders
+    var variableNames = Object.keys(variables);
+    for (var i = 0; i < variableNames.length; i++) {
+        var name = variableNames[i];
+        var value = variables[name];
+        var placeholder = '{{' + name + '}}';
+
+        // Replace all occurrences
+        while (template.indexOf(placeholder) !== -1) {
+            template = template.replace(placeholder, value);
+        }
+    }
+
+    return template;
+}
+
+// Export the factory function as the main export
+module.exports = createProvider;
+
+// Also export helper functions
+module.exports.createProvider = createProvider;
+module.exports.getAvailableModels = getAvailableModels;
+module.exports.getConfig = getConfig;
+module.exports.loadPromptTemplate = loadPromptTemplate;

package/lib/providers/base.js

@@ -0,0 +1,218 @@
+/**
+ * BaseProvider - Abstract base class for inference providers
+ *
+ * This defines the interface that all inference providers must implement.
+ * Both node-llama-cpp (local GGUF) and Ollama (HTTP API) providers extend this.
+ *
+ * Usage:
+ *   const provider = new SomeProvider(config, modelConfig);
+ *   await provider.initialize();
+ *   const result = await provider.generate(prompt, schema);
+ *   await provider.destroy();
+ */
+
+/**
+ * Abstract base class for inference providers
+ *
+ * @class BaseProvider
+ */
+class BaseProvider {
+    /**
+     * Create a new provider instance
+     *
+     * @param {Object} config - Runtime configuration from config/default.json
+     * @param {Object} modelConfig - Model configuration from config/models.json
+     */
+    constructor(config, modelConfig) {
+        // Store configuration for subclasses
+        this.config = config;
+        this.modelConfig = modelConfig;
+
+        // Track initialization state
+        this.isInitialized = false;
+
+        // Provider name for logging (override in subclass)
+        this.providerName = 'base';
+    }
+
+    /**
+     * Initialize the provider
+     *
+     * This method must be called before generate(). It loads the model,
+     * establishes connections, or performs any other setup required.
+     *
+     * @abstract
+     * @returns {Promise<void>}
+     * @throws {Error} If initialization fails
+     */
+    async initialize() {
+        throw new Error('BaseProvider.initialize() must be implemented by subclass');
+    }
+
+    /**
+     * Generate a response from the model
+     *
+     * Takes a prompt and optional JSON schema, returns parsed JSON response.
+     * The schema is used for grammar enforcement (node-llama-cpp) or
+     * manual validation (Ollama).
+     *
+     * @abstract
+     * @param {string} prompt - The prompt to send to the model
+     * @param {Object} [schema] - JSON schema for structured output
+     * @returns {Promise<Object>} Parsed JSON response from the model
+     * @throws {Error} If generation fails or response doesn't match schema
+     */
+    async generate(prompt, schema) {
+        throw new Error('BaseProvider.generate() must be implemented by subclass');
+    }
+
+    /**
+     * Clean up resources
+     *
+     * Call this when you're done with the provider. Releases model memory,
+     * closes connections, etc.
+     *
+     * @abstract
+     * @returns {Promise<void>}
+     */
+    async destroy() {
+        throw new Error('BaseProvider.destroy() must be implemented by subclass');
+    }
+
+    /**
+     * Check if the provider is ready for generation
+     *
+     * @returns {boolean} True if initialized and ready
+     */
+    isReady() {
+        return this.isInitialized;
+    }
+
+    /**
+     * Get provider information for logging
+     *
+     * @returns {Object} Provider info including name and model
+     */
+    getInfo() {
+        return {
+            provider: this.providerName,
+            model: this.modelConfig ? this.modelConfig.id : 'unknown',
+            isReady: this.isInitialized
+        };
+    }
+
+    /**
+     * Log a message with provider context
+     *
+     * @protected
+     * @param {string} message - Message to log
+     */
+    log(message) {
+        console.log('[' + this.providerName + '] ' + message);
+    }
+
+    /**
+     * Log an error with provider context
+     *
+     * @protected
+     * @param {string} message - Error message
+     * @param {Error} [error] - Optional error object
+     */
+    logError(message, error) {
+        console.error('[' + this.providerName + ' ERROR] ' + message);
+        if (error) {
+            console.error('  Details:', error.message);
+        }
+    }
+
+    /**
+     * Validate that a response matches a JSON schema
+     *
+     * This is a simple validator for required fields and basic types.
+     * For production use, consider a full JSON Schema validator like Ajv.
+     *
+     * @protected
+     * @param {Object} response - Parsed JSON response
+     * @param {Object} schema - JSON schema to validate against
+     * @returns {Object} Validation result with isValid and errors
+     */
+    validateResponse(response, schema) {
+        var errors = [];
+
+        // Check if response is an object
+        if (typeof response !== 'object' || response === null) {
+            return {
+                isValid: false,
+                errors: ['Response must be an object']
+            };
+        }
+
+        // Check required fields
+        if (schema.required && Array.isArray(schema.required)) {
+            for (var i = 0; i < schema.required.length; i++) {
+                var field = schema.required[i];
+                if (!(field in response)) {
+                    errors.push('Missing required field: ' + field);
+                }
+            }
+        }
+
+        // Check property types
+        if (schema.properties) {
+            var properties = Object.keys(schema.properties);
+            for (var j = 0; j < properties.length; j++) {
+                var propName = properties[j];
+                var propSchema = schema.properties[propName];
+
+                // Skip if property not present (unless required, caught above)
+                if (!(propName in response)) {
+                    continue;
+                }
+
+                var value = response[propName];
+
+                // Check type
+                if (propSchema.type) {
+                    var expectedType = propSchema.type;
+                    var actualType = typeof value;
+
+                    // Handle array type specially
+                    if (expectedType === 'array' && !Array.isArray(value)) {
+                        errors.push(propName + ' must be an array');
+                    } else if (expectedType === 'integer' || expectedType === 'number') {
+                        if (actualType !== 'number') {
+                            errors.push(propName + ' must be a number');
+                        }
+                    } else if (expectedType !== 'array' && actualType !== expectedType) {
+                        errors.push(propName + ' must be ' + expectedType + ', got ' + actualType);
+                    }
+                }
+
+                // Check enum
+                if (propSchema.enum && Array.isArray(propSchema.enum)) {
+                    if (propSchema.enum.indexOf(value) === -1) {
+                        errors.push(propName + ' must be one of: ' + propSchema.enum.join(', '));
+                    }
+                }
+
+                // Check minimum/maximum for numbers
+                if (typeof value === 'number') {
+                    if (propSchema.minimum !== undefined && value < propSchema.minimum) {
+                        errors.push(propName + ' must be >= ' + propSchema.minimum);
+                    }
+                    if (propSchema.maximum !== undefined && value > propSchema.maximum) {
+                        errors.push(propName + ' must be <= ' + propSchema.maximum);
+                    }
+                }
+            }
+        }
+
+        return {
+            isValid: errors.length === 0,
+            errors: errors
+        };
+    }
+}
+
+// Export the class
+module.exports = BaseProvider;

package/lib/providers/node-llama-cpp.js

@@ -0,0 +1,196 @@
+/**
+ * NodeLlamaCppProvider - Inference provider using node-llama-cpp
+ *
+ * This provider loads GGUF model files directly and uses node-llama-cpp
+ * for inference. It supports JSON schema grammar enforcement for
+ * guaranteed valid structured output.
+ *
+ * Benefits:
+ * - Air-gapped operation (no network required after download)
+ * - Grammar enforcement ensures valid JSON output
+ * - Native performance via llama.cpp
+ *
+ * Requirements:
+ * - GGUF model file in the models/ directory
+ * - node-llama-cpp installed (npm install)
+ * - Xcode Command Line Tools (macOS)
+ */
+
+var path = require('path');
+var fs = require('fs');
+var BaseProvider = require('./base');
+
+/**
+ * Provider implementation for node-llama-cpp
+ *
+ * @class NodeLlamaCppProvider
+ * @extends BaseProvider
+ */
+class NodeLlamaCppProvider extends BaseProvider {
+    /**
+     * Create a new node-llama-cpp provider
+     *
+     * @param {Object} config - Runtime configuration
+     * @param {Object} modelConfig - Model configuration
+     * @param {string} basePath - Base path for resolving relative paths
+     */
+    constructor(config, modelConfig, basePath) {
+        super(config, modelConfig);
+
+        this.providerName = 'node-llama-cpp';
+        this.basePath = basePath;
+
+        // These will be set during initialization
+        this.llama = null;
+        this.model = null;
+        this.context = null;
+        this.session = null;
+    }
+
+    /**
+     * Initialize the model
+     *
+     * Loads the GGUF file and creates a chat session.
+     * This can take several seconds on first run.
+     *
+     * @returns {Promise<void>}
+     * @throws {Error} If model file not found or initialization fails
+     */
+    async initialize() {
+        this.log('Initializing...');
+
+        // Build path to model file
+        var modelsDir = this.config.paths ? this.config.paths.modelsDir : 'models';
+        var modelPath = path.join(this.basePath, modelsDir, this.modelConfig.filename);
+
+        // Check if model file exists
+        this.log('Looking for model: ' + this.modelConfig.filename);
+        if (!fs.existsSync(modelPath)) {
+            var errorMessage = 'Model file not found: ' + modelPath + '\n\n';
+            errorMessage += 'Download the model with:\n';
+            errorMessage += '  ' + this.modelConfig.downloadCommand;
+            throw new Error(errorMessage);
+        }
+
+        var stats = fs.statSync(modelPath);
+        var sizeMB = (stats.size / (1024 * 1024)).toFixed(2);
+        this.log('Model file found (' + sizeMB + ' MB)');
+
+        // Import node-llama-cpp (ESM module requires dynamic import)
+        this.log('Loading node-llama-cpp...');
+        var llamaModule = await import('node-llama-cpp');
+        var getLlama = llamaModule.getLlama;
+        var LlamaChatSession = llamaModule.LlamaChatSession;
+
+        // Store the chat session class for later use
+        this.LlamaChatSession = LlamaChatSession;
+
+        // Get the llama instance
+        this.log('Getting llama instance...');
+        this.llama = await getLlama();
+
+        // Load the model
+        this.log('Loading model (this may take a moment)...');
+        this.model = await this.llama.loadModel({
+            modelPath: modelPath
+        });
+
+        // Create a context
+        this.log('Creating context...');
+        this.context = await this.model.createContext();
+
+        // Create a chat session
+        this.log('Creating chat session...');
+        this.session = new this.LlamaChatSession({
+            contextSequence: this.context.getSequence()
+        });
+
+        this.isInitialized = true;
+        this.log('Ready');
+    }
+
+    /**
+     * Generate a response with optional JSON schema enforcement
+     *
+     * @param {string} prompt - The prompt to send to the model
+     * @param {Object} [schema] - JSON schema for grammar enforcement
+     * @returns {Promise<Object>} Parsed JSON response
+     * @throws {Error} If not initialized or generation fails
+     */
+    async generate(prompt, schema) {
+        // Check initialization
+        if (!this.isInitialized) {
+            throw new Error('Provider not initialized. Call initialize() first.');
+        }
+
+        // Get inference settings from config
+        var temperature = this.config.inference ? this.config.inference.temperature : 0.1;
+        var maxTokens = this.config.inference ? this.config.inference.maxTokens : 1024;
+
+        // Build generation options
+        var options = {
+            maxTokens: maxTokens,
+            temperature: temperature
+        };
+
+        // If schema provided, create grammar for enforcement
+        if (schema) {
+            this.log('Creating grammar from JSON schema...');
+            var grammar = await this.llama.createGrammarForJsonSchema(schema);
+            options.grammar = grammar;
+        }
+
+        // Generate response
+        this.log('Generating response...');
+        var rawResponse = await this.session.prompt(prompt, options);
+
+        // Parse the JSON response
+        this.log('Parsing response...');
+        var parsed;
+        try {
+            parsed = JSON.parse(rawResponse);
+        } catch (parseError) {
+            this.logError('Failed to parse JSON response', parseError);
+            this.log('Raw response: ' + rawResponse);
+            throw new Error('Model returned invalid JSON: ' + parseError.message);
+        }
+
+        // Validate against schema if provided
+        if (schema) {
+            var validation = this.validateResponse(parsed, schema);
+            if (!validation.isValid) {
+                this.logError('Response failed schema validation');
+                for (var i = 0; i < validation.errors.length; i++) {
+                    this.log('  - ' + validation.errors[i]);
+                }
+                throw new Error('Schema validation failed: ' + validation.errors.join('; '));
+            }
+        }
+
+        this.log('Generation complete');
+        return parsed;
+    }
+
+    /**
+     * Clean up resources
+     *
+     * Releases model memory and closes the context.
+     *
+     * @returns {Promise<void>}
+     */
+    async destroy() {
+        this.log('Cleaning up...');
+
+        // Clear references (garbage collection will handle cleanup)
+        this.session = null;
+        this.context = null;
+        this.model = null;
+        this.llama = null;
+
+        this.isInitialized = false;
+        this.log('Cleanup complete');
+    }
+}
+
+// Export the class
+module.exports = NodeLlamaCppProvider;