@aci-metrics/score 0.0.1

package/lib/providers/ollama.js

@@ -0,0 +1,432 @@
+/**
+ * OllamaProvider - Inference provider using Ollama HTTP API
+ *
+ * This provider connects to a locally running Ollama server.
+ * It uses the format: "json" option for structured output,
+ * with manual validation against the schema.
+ *
+ * Benefits:
+ * - Easy model management (ollama pull/run)
+ * - Multiple models can be loaded simultaneously
+ * - Works with any Ollama-compatible model
+ *
+ * Requirements:
+ * - Ollama installed and running (https://ollama.ai)
+ * - Model pulled (e.g., ollama pull qwen2.5:1.5b)
+ *
+ * Note: Ollama's JSON mode doesn't enforce schema strictly,
+ * so we validate the output manually after generation.
+ */
+
+var BaseProvider = require('./base');
+
+/**
+ * Provider implementation for Ollama HTTP API
+ *
+ * @class OllamaProvider
+ * @extends BaseProvider
+ */
+class OllamaProvider extends BaseProvider {
+    /**
+     * Create a new Ollama provider
+     *
+     * @param {Object} config - Runtime configuration
+     * @param {Object} modelConfig - Model configuration
+     * @param {string} basePath - Base path (not used for Ollama)
+     */
+    constructor(config, modelConfig, basePath) {
+        super(config, modelConfig);
+
+        this.providerName = 'ollama';
+        this.basePath = basePath;
+
+        // Get Ollama host from config or use default
+        this.host = (config.ollama && config.ollama.host) || 'http://localhost:11434';
+
+        // Model name to use in Ollama
+        this.ollamaModel = modelConfig.ollamaModel || modelConfig.id;
+    }
+
+    /**
+     * Initialize the provider
+     *
+     * Verifies that Ollama is running and the model is available.
+     *
+     * @returns {Promise<void>}
+     * @throws {Error} If Ollama is not running or model not found
+     */
+    async initialize() {
+        this.log('Initializing...');
+        this.log('Host: ' + this.host);
+        this.log('Model: ' + this.ollamaModel);
+
+        // Check if Ollama is running by calling the API
+        this.log('Checking Ollama server...');
+        try {
+            var response = await this.httpGet(this.host + '/api/tags');
+
+            if (!response.ok) {
+                throw new Error('Ollama returned status ' + response.status);
+            }
+
+            var data = response.data;
+            this.log('Ollama server is running');
+
+            // Check if the model is available
+            var modelFound = false;
+            if (data.models && Array.isArray(data.models)) {
+                for (var i = 0; i < data.models.length; i++) {
+                    var modelName = data.models[i].name;
+                    // Ollama model names may include :latest or version tags
+                    if (modelName === this.ollamaModel || modelName.startsWith(this.ollamaModel + ':')) {
+                        modelFound = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!modelFound) {
+                var errorMessage = 'Model not found in Ollama: ' + this.ollamaModel + '\n\n';
+                errorMessage += 'Pull the model with:\n';
+                errorMessage += '  ollama pull ' + this.ollamaModel + '\n\n';
+                errorMessage += 'Available models:\n';
+                if (data.models && data.models.length > 0) {
+                    for (var j = 0; j < data.models.length; j++) {
+                        errorMessage += '  - ' + data.models[j].name + '\n';
+                    }
+                } else {
+                    errorMessage += '  (none)\n';
+                }
+                throw new Error(errorMessage);
+            }
+
+            this.log('Model is available');
+            this.isInitialized = true;
+            this.log('Ready');
+
+        } catch (error) {
+            if (error.code === 'ECONNREFUSED') {
+                var connectionError = 'Cannot connect to Ollama at ' + this.host + '\n\n';
+                connectionError += 'Make sure Ollama is running:\n';
+                connectionError += '  1. Install from https://ollama.ai\n';
+                connectionError += '  2. Run: ollama serve\n';
+                throw new Error(connectionError);
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Generate a response with JSON output
+     *
+     * Uses Ollama's format: "json" option and validates against schema.
+     *
+     * @param {string} prompt - The prompt to send to the model
+     * @param {Object} [schema] - JSON schema for validation
+     * @returns {Promise<Object>} Parsed JSON response
+     * @throws {Error} If not initialized, generation fails, or validation fails
+     */
+    async generate(prompt, schema) {
+        // Check initialization
+        if (!this.isInitialized) {
+            throw new Error('Provider not initialized. Call initialize() first.');
+        }
+
+        // Get inference settings from config
+        var temperature = this.config.inference ? this.config.inference.temperature : 0.1;
+        var maxTokens = this.config.inference ? this.config.inference.maxTokens : 1024;
+
+        // Build the request body
+        // Enhance prompt with schema information if provided
+        var enhancedPrompt = prompt;
+        if (schema) {
+            enhancedPrompt = this.buildSchemaPrompt(prompt, schema);
+        }
+
+        var requestBody = {
+            model: this.ollamaModel,
+            prompt: enhancedPrompt,
+            stream: false,
+            format: 'json',
+            options: {
+                temperature: temperature,
+                num_predict: maxTokens
+            }
+        };
+
+        // Make the API call
+        this.log('Generating response...');
+        var response = await this.httpPost(this.host + '/api/generate', requestBody);
+
+        if (!response.ok) {
+            throw new Error('Ollama API error: ' + response.status);
+        }
+
+        var data = response.data;
+
+        if (!data.response) {
+            throw new Error('Ollama returned empty response');
+        }
+
+        // Parse the JSON response
+        this.log('Parsing response...');
+        var parsed;
+        try {
+            parsed = JSON.parse(data.response);
+        } catch (parseError) {
+            this.logError('Failed to parse JSON response', parseError);
+            this.log('Raw response: ' + data.response);
+            throw new Error('Model returned invalid JSON: ' + parseError.message);
+        }
+
+        // Validate against schema if provided
+        if (schema) {
+            var validation = this.validateResponse(parsed, schema);
+            if (!validation.isValid) {
+                this.logError('Response failed schema validation');
+                for (var i = 0; i < validation.errors.length; i++) {
+                    this.log('  - ' + validation.errors[i]);
+                }
+
+                // Try to fix common issues and retry validation
+                var fixed = this.attemptFix(parsed, schema);
+                if (fixed) {
+                    var revalidation = this.validateResponse(fixed, schema);
+                    if (revalidation.isValid) {
+                        this.log('Fixed validation issues');
+                        return fixed;
+                    }
+                }
+
+                throw new Error('Schema validation failed: ' + validation.errors.join('; '));
+            }
+        }
+
+        this.log('Generation complete');
+        return parsed;
+    }
+
+    /**
+     * Build a prompt that includes schema information
+     *
+     * This helps the model understand the expected output format
+     * since Ollama doesn't have native schema enforcement.
+     *
+     * @private
+     * @param {string} prompt - Original prompt
+     * @param {Object} schema - JSON schema
+     * @returns {string} Enhanced prompt
+     */
+    buildSchemaPrompt(prompt, schema) {
+        var schemaDescription = 'You must respond with a JSON object with the following structure:\n\n';
+
+        if (schema.properties) {
+            var propNames = Object.keys(schema.properties);
+            for (var i = 0; i < propNames.length; i++) {
+                var name = propNames[i];
+                var prop = schema.properties[name];
+                var required = schema.required && schema.required.indexOf(name) !== -1;
+
+                schemaDescription += '- "' + name + '" (' + prop.type + ')';
+                if (required) {
+                    schemaDescription += ' [required]';
+                }
+                if (prop.enum) {
+                    schemaDescription += ' - one of: ' + prop.enum.join(', ');
+                }
+                if (prop.minimum !== undefined || prop.maximum !== undefined) {
+                    schemaDescription += ' - range: ' +
+                        (prop.minimum !== undefined ? prop.minimum : 'any') +
+                        ' to ' +
+                        (prop.maximum !== undefined ? prop.maximum : 'any');
+                }
+                schemaDescription += '\n';
+            }
+        }
+
+        return schemaDescription + '\n' + prompt;
+    }
+
+    /**
+     * Attempt to fix common validation issues
+     *
+     * @private
+     * @param {Object} response - Parsed response
+     * @param {Object} schema - JSON schema
+     * @returns {Object|null} Fixed response or null
+     */
+    attemptFix(response, schema) {
+        var fixed = JSON.parse(JSON.stringify(response)); // Deep clone
+
+        if (!schema.properties) {
+            return null;
+        }
+
+        var propNames = Object.keys(schema.properties);
+        for (var i = 0; i < propNames.length; i++) {
+            var name = propNames[i];
+            var prop = schema.properties[name];
+
+            if (!(name in fixed)) {
+                continue;
+            }
+
+            // Try to coerce types
+            if (prop.type === 'number' || prop.type === 'integer') {
+                if (typeof fixed[name] === 'string') {
+                    var num = Number(fixed[name]);
+                    if (!isNaN(num)) {
+                        fixed[name] = num;
+                    }
+                }
+            }
+
+            // Clamp numbers to range
+            if (typeof fixed[name] === 'number') {
+                if (prop.minimum !== undefined && fixed[name] < prop.minimum) {
+                    fixed[name] = prop.minimum;
+                }
+                if (prop.maximum !== undefined && fixed[name] > prop.maximum) {
+                    fixed[name] = prop.maximum;
+                }
+            }
+        }
+
+        return fixed;
+    }
+
+    /**
+     * Clean up resources
+     *
+     * Nothing to clean up for HTTP-based provider.
+     *
+     * @returns {Promise<void>}
+     */
+    async destroy() {
+        this.log('Cleaning up...');
+        this.isInitialized = false;
+        this.log('Cleanup complete');
+    }
+
+    /**
+     * Make an HTTP GET request
+     *
+     * Uses Node.js built-in http/https modules.
+     *
+     * @private
+     * @param {string} url - URL to fetch
+     * @returns {Promise<Object>} Response with ok, status, and data
+     */
+    async httpGet(url) {
+        return new Promise(function(resolve, reject) {
+            var parsedUrl = new URL(url);
+            var httpModule = parsedUrl.protocol === 'https:' ? require('https') : require('http');
+
+            var options = {
+                hostname: parsedUrl.hostname,
+                port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80),
+                path: parsedUrl.pathname + parsedUrl.search,
+                method: 'GET',
+                headers: {
+                    'Accept': 'application/json'
+                }
+            };
+
+            var req = httpModule.request(options, function(res) {
+                var chunks = [];
+
+                res.on('data', function(chunk) {
+                    chunks.push(chunk);
+                });
+
+                res.on('end', function() {
+                    var body = Buffer.concat(chunks).toString();
+                    var data = null;
+
+                    try {
+                        data = JSON.parse(body);
+                    } catch (e) {
+                        data = body;
+                    }
+
+                    resolve({
+                        ok: res.statusCode >= 200 && res.statusCode < 300,
+                        status: res.statusCode,
+                        data: data
+                    });
+                });
+            });
+
+            req.on('error', function(err) {
+                reject(err);
+            });
+
+            req.end();
+        });
+    }
+
+    /**
+     * Make an HTTP POST request
+     *
+     * @private
+     * @param {string} url - URL to post to
+     * @param {Object} body - Request body (will be JSON stringified)
+     * @returns {Promise<Object>} Response with ok, status, and data
+     */
+    async httpPost(url, body) {
+        var self = this;
+
+        return new Promise(function(resolve, reject) {
+            var parsedUrl = new URL(url);
+            var httpModule = parsedUrl.protocol === 'https:' ? require('https') : require('http');
+            var bodyStr = JSON.stringify(body);
+
+            var options = {
+                hostname: parsedUrl.hostname,
+                port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80),
+                path: parsedUrl.pathname + parsedUrl.search,
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(bodyStr),
+                    'Accept': 'application/json'
+                }
+            };
+
+            var req = httpModule.request(options, function(res) {
+                var chunks = [];
+
+                res.on('data', function(chunk) {
+                    chunks.push(chunk);
+                });
+
+                res.on('end', function() {
+                    var responseBody = Buffer.concat(chunks).toString();
+                    var data = null;
+
+                    try {
+                        data = JSON.parse(responseBody);
+                    } catch (e) {
+                        data = responseBody;
+                    }
+
+                    resolve({
+                        ok: res.statusCode >= 200 && res.statusCode < 300,
+                        status: res.statusCode,
+                        data: data
+                    });
+                });
+            });
+
+            req.on('error', function(err) {
+                reject(err);
+            });
+
+            req.write(bodyStr);
+            req.end();
+        });
+    }
+}
+
+// Export the class
+module.exports = OllamaProvider;

package/models/.gitkeep

@@ -0,0 +1,2 @@
+# This file ensures the models directory exists in git
+# Model files (*.gguf) are ignored - download separately

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@aci-metrics/score",
+  "version": "0.0.1",
+  "description": "The ACI scoring CLI is in beta. For an invite, contact us at acimetrics.com.",
+  "main": "aci-score.js",
+  "scripts": {
+    "score": "node aci-score.js",
+    "test": "node test-model.js",
+    "download-model": "echo 'Run: huggingface-cli download bartowski/Qwen2.5-1.5B-Instruct-GGUF Qwen2.5-1.5B-Instruct-Q4_K_M.gguf --local-dir ./models'"
+  },
+  "bin": {
+    "aci-score": "./aci-score.js"
+  },
+  "keywords": [
+    "aci",
+    "ai-collaboration",
+    "llm",
+    "scoring",
+    "aci-metrics"
+  ],
+  "author": "Original Engineering LLC",
+  "license": "UNLICENSED",
+  "private": false,
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "node-llama-cpp": "^3.3.0",
+    "simple-statistics": "^7.8.3"
+  }
+}

package/prompts/gemma.txt

@@ -0,0 +1,15 @@
+<start_of_turn>user
+You are an AI collaboration analyst. Analyze the following coding session metrics and respond with a JSON object only.
+
+Session Metrics:
+{{METRICS}}
+
+Required JSON fields:
+- task_type: One of "feature", "bugfix", "refactor", "docs", "test", "config", "other"
+- complexity: Integer from 1 to 10
+- collaboration_quality: Integer from 1 to 10
+- archetype: One of "orchestrator", "refiner", "sprinter", "diver", "director", "partner"
+- summary: Brief description (1-2 sentences)
+
+Respond with valid JSON only, no other text.<end_of_turn>
+<start_of_turn>model

package/prompts/llama.txt

@@ -0,0 +1,17 @@
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+You are an AI collaboration analyst. Analyze coding session metrics and provide structured JSON assessments. Always respond with valid JSON only.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Analyze these session metrics:
+
+{{METRICS}}
+
+Provide a JSON object with these fields:
+- task_type: One of (feature, bugfix, refactor, docs, test, config, other)
+- complexity: Integer 1-10 based on scope and duration
+- collaboration_quality: Integer 1-10 for human-AI collaboration effectiveness
+- archetype: One of (orchestrator, refiner, sprinter, diver, director, partner)
+- summary: Brief 1-2 sentence description
+
+JSON response only:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

package/prompts/phi.txt

@@ -0,0 +1,16 @@
+<|system|>
+You are an AI collaboration analyst specializing in evaluating human-AI coding sessions. You analyze metrics and provide structured assessments in JSON format. Always respond with valid JSON only, no additional text.
+<|end|>
+<|user|>
+Please analyze the following coding session metrics and provide your assessment:
+
+{{METRICS}}
+
+Your JSON response must include:
+- task_type: The primary type of task (feature, bugfix, refactor, docs, test, config, or other)
+- complexity: A score from 1 to 10 indicating task complexity
+- collaboration_quality: A score from 1 to 10 rating the human-AI collaboration
+- archetype: The collaboration style (orchestrator, refiner, sprinter, diver, director, or partner)
+- summary: A brief 1-2 sentence summary of what was accomplished
+<|end|>
+<|assistant|>

package/prompts/qwen.txt

@@ -0,0 +1,18 @@
+You are an AI collaboration analyst. Your task is to analyze metrics from AI-assisted coding sessions and provide structured assessments.
+
+Analyze the following session metrics and provide your assessment.
+
+## Session Metrics
+
+{{METRICS}}
+
+## Instructions
+
+Based on the metrics above, provide a JSON response with:
+1. task_type: Classify the primary task type (feature, bugfix, refactor, docs, test, config, other)
+2. complexity: Rate complexity from 1-10 based on scope, files touched, and session duration
+3. collaboration_quality: Rate the human-AI collaboration quality from 1-10
+4. archetype: Assign a primary archetype (orchestrator, refiner, sprinter, diver, director, partner)
+5. summary: Brief 1-2 sentence description of what was accomplished
+
+Respond with valid JSON only. No additional text or explanation.