n2-qln 3.1.0

package/lib/config.js

@@ -0,0 +1,65 @@
+// QLN — Config loader (default + local deep merge)
+const path = require('path');
+const fs = require('fs');
+
+/** Default configuration */
+const defaults = {
+    /** Data directory (SQLite, indices, etc.) */
+    dataDir: path.join(__dirname, '..', 'data'),
+
+    /** Embedding configuration */
+    embedding: {
+        enabled: true,
+        model: 'nomic-embed-text',
+        endpoint: 'http://127.0.0.1:11434',
+    },
+
+    /** Tool execution configuration */
+    executor: {
+        httpEndpoint: null,
+        timeout: 20000,
+    },
+
+    /** Search configuration */
+    search: {
+        defaultTopK: 5,
+        threshold: 0.1,
+    },
+};
+
+/**
+ * Load config — apply config.local.js overrides on top of defaults.
+ * @returns {object} Merged config
+ */
+function loadConfig() {
+    const config = JSON.parse(JSON.stringify(defaults));
+    const localPath = path.join(__dirname, '..', 'config.local.js');
+
+    if (fs.existsSync(localPath)) {
+        try {
+            const local = require(localPath);
+            deepMerge(config, local);
+        } catch (e) {
+            console.warn(`[QLN] config.local.js load failed: ${e.message}`);
+        }
+    }
+    return config;
+}
+
+/**
+ * Deep merge (merge source into target).
+ * @param {object} target
+ * @param {object} source
+ */
+function deepMerge(target, source) {
+    for (const key of Object.keys(source)) {
+        if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])
+            && target[key] && typeof target[key] === 'object') {
+            deepMerge(target[key], source[key]);
+        } else {
+            target[key] = source[key];
+        }
+    }
+}
+
+module.exports = { loadConfig, defaults };

package/lib/embedding.js

@@ -0,0 +1,131 @@
+// QLN — Embedding engine (Ollama nomic-embed-text)
+// Vector embedding generation for semantic search (Stage 3)
+const http = require('http');
+
+/**
+ * Ollama-based local embedding engine.
+ * Graceful degradation when unavailable — Stage 1+2 still work.
+ */
+class Embedding {
+    /**
+     * @param {object} config
+     * @param {string} [config.model='nomic-embed-text'] - Ollama model
+     * @param {string} [config.endpoint='http://127.0.0.1:11434'] - Ollama endpoint
+     */
+    constructor(config = {}) {
+        this.model = config.model || 'nomic-embed-text';
+        this.endpoint = config.endpoint || 'http://127.0.0.1:11434';
+        this.dimensions = null;
+        this._available = null;
+    }
+
+    /**
+     * Check Ollama availability (cached).
+     * @returns {Promise<boolean>}
+     */
+    async isAvailable() {
+        if (this._available !== null) return this._available;
+        try {
+            const vec = await this.embed('test');
+            this._available = vec.length > 0;
+            this.dimensions = vec.length;
+            return this._available;
+        } catch {
+            this._available = false;
+            return false;
+        }
+    }
+
+    /**
+     * Generate vector embedding from text.
+     * @param {string} text
+     * @returns {Promise<number[]>}
+     */
+    async embed(text) {
+        if (!text || text.trim().length === 0) return [];
+        const input = text.length > 2000 ? text.slice(0, 2000) : text;
+
+        for (const apiPath of ['/api/embeddings', '/api/embed']) {
+            try {
+                const body = apiPath === '/api/embeddings'
+                    ? { model: this.model, prompt: input }
+                    : { model: this.model, input: input };
+                const result = await this._post(apiPath, body);
+
+                if (result.embedding && Array.isArray(result.embedding)) {
+                    this.dimensions = result.embedding.length;
+                    return result.embedding;
+                }
+                if (result.embeddings && Array.isArray(result.embeddings) && result.embeddings[0]) {
+                    this.dimensions = result.embeddings[0].length;
+                    return result.embeddings[0];
+                }
+            } catch { continue; }
+        }
+        return [];
+    }
+
+    /**
+     * Batch embedding generation.
+     * @param {string[]} texts
+     * @returns {Promise<number[][]>}
+     */
+    async embedBatch(texts) {
+        const results = [];
+        for (const text of texts) {
+            results.push(await this.embed(text));
+        }
+        return results;
+    }
+
+    /**
+     * Cosine similarity between two vectors.
+     * @param {number[]} a
+     * @param {number[]} b
+     * @returns {number} 0~1
+     */
+    cosineSimilarity(a, b) {
+        if (!a || !b || a.length === 0 || a.length !== b.length) return 0;
+        let dot = 0, normA = 0, normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const denom = Math.sqrt(normA) * Math.sqrt(normB);
+        return denom === 0 ? 0 : dot / denom;
+    }
+
+    /** @private HTTP POST to Ollama API */
+    _post(apiPath, body) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(this.endpoint);
+            const options = {
+                hostname: url.hostname,
+                port: url.port || 11434,
+                path: apiPath,
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                timeout: 30000,
+            };
+            const req = http.request(options, res => {
+                let data = '';
+                res.on('data', chunk => { data += chunk; });
+                res.on('end', () => {
+                    if (res.statusCode >= 400) {
+                        reject(new Error(`Ollama ${res.statusCode}: ${data.slice(0, 200)}`));
+                        return;
+                    }
+                    try { resolve(JSON.parse(data)); }
+                    catch { reject(new Error(`Invalid JSON: ${data.slice(0, 100)}`)); }
+                });
+            });
+            req.on('error', reject);
+            req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+            req.write(JSON.stringify(body));
+            req.end();
+        });
+    }
+}
+
+module.exports = { Embedding };

package/lib/executor.js

@@ -0,0 +1,104 @@
+// QLN — L3 tool executor
+// Execute tools via HTTP (localhost) or registered local handlers
+const http = require('http');
+
+/**
+ * Tool executor — HTTP proxy + local function calls.
+ *
+ * Execution priority:
+ *   1. Local registered handler (via addHandler)
+ *   2. HTTP proxy (when endpoint configured)
+ */
+class Executor {
+    /**
+     * @param {object} config
+     * @param {string} [config.httpEndpoint] - Tool execution HTTP endpoint (e.g. "http://127.0.0.1:PORT")
+     * @param {number} [config.timeout=20000] - HTTP timeout (ms)
+     */
+    constructor(config = {}) {
+        this._httpEndpoint = config.httpEndpoint || null;
+        this._timeout = config.timeout || 20000;
+        /** @type {Map<string, Function>} Local handlers */
+        this._handlers = new Map();
+    }
+
+    /**
+     * Register a local tool handler.
+     * @param {string} name - Tool name
+     * @param {Function} handler - (args) => Promise<unknown>
+     */
+    addHandler(name, handler) {
+        this._handlers.set(name, handler);
+    }
+
+    /**
+     * Execute a tool.
+     * @param {string} name - Tool name
+     * @param {object} args - Tool arguments
+     * @returns {Promise<{result: unknown, source: string, elapsed: number}>}
+     */
+    async exec(name, args = {}) {
+        const t0 = Date.now();
+
+        // 1. Local handler first
+        if (this._handlers.has(name)) {
+            const handler = this._handlers.get(name);
+            const result = await handler(args);
+            return { result, source: 'local', elapsed: Date.now() - t0 };
+        }
+
+        // 2. HTTP proxy
+        if (this._httpEndpoint) {
+            const result = await this._execHttp(name, args);
+            return { result, source: 'http', elapsed: Date.now() - t0 };
+        }
+
+        throw new Error(`No handler found for tool: ${name}. Register with addHandler() or set httpEndpoint.`);
+    }
+
+    /**
+     * Dynamically set HTTP endpoint.
+     * @param {string} endpoint - "http://127.0.0.1:PORT" format
+     */
+    setHttpEndpoint(endpoint) {
+        this._httpEndpoint = endpoint;
+    }
+
+    /** @private HTTP POST /call → tool execution */
+    _execHttp(name, args) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(this._httpEndpoint);
+            const bodyStr = JSON.stringify({ tool: name, args });
+            const timer = setTimeout(() => reject(new Error(`timeout (${this._timeout}ms)`)), this._timeout);
+
+            const req = http.request({
+                hostname: url.hostname,
+                port: url.port,
+                path: '/call',
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(bodyStr),
+                },
+            }, (res) => {
+                let body = '';
+                res.on('data', c => body += c);
+                res.on('end', () => {
+                    clearTimeout(timer);
+                    try {
+                        const parsed = JSON.parse(body);
+                        if (parsed.error) reject(new Error(parsed.error));
+                        else resolve(parsed.result);
+                    } catch {
+                        reject(new Error(`Invalid response: ${body.slice(0, 200)}`));
+                    }
+                });
+            });
+            req.on('error', e => { clearTimeout(timer); reject(e); });
+            req.write(bodyStr);
+            req.end();
+        });
+    }
+}
+
+module.exports = { Executor };

package/lib/registry.js

@@ -0,0 +1,217 @@
+// QLN — L2 tool index (memory cache + SQLite persistence)
+// Tool CRUD, batch registration, embedding precomputation, usage tracking
+const { createToolEntry, buildSearchText } = require('./schema');
+
+/**
+ * Tool registry — in-memory cache (Map) + SQLite persistence.
+ * Designed for up to 1000 tools.
+ */
+class Registry {
+    /**
+     * @param {import('./store').Store} store - SQLite store
+     * @param {import('./embedding').Embedding} [embedding] - Embedding engine (optional)
+     */
+    constructor(store, embedding = null) {
+        this._store = store;
+        this._embedding = embedding;
+        /** @type {Map<string, object>} name → tool entry */
+        this._cache = new Map();
+    }
+
+    /** Load all entries from SQLite into memory cache */
+    load() {
+        this._cache.clear();
+        const rows = this._store.loadAll();
+        for (const row of rows) {
+            this._cache.set(row.name, this._rowToEntry(row));
+        }
+    }
+
+    // ── CRUD ──
+
+    /**
+     * Register a tool (update if exists, preserve existing stats).
+     * @param {object} raw - Raw tool data
+     * @returns {object} Normalized tool entry
+     */
+    register(raw) {
+        const entry = createToolEntry(raw);
+        const existing = this._cache.get(entry.name);
+        if (existing) {
+            entry.usageCount = existing.usageCount || entry.usageCount;
+            entry.successRate = existing.successRate ?? entry.successRate;
+            if (existing.embedding && !entry.embedding) {
+                entry.embedding = existing.embedding;
+            }
+        }
+        entry.searchText = buildSearchText(entry);
+        this._cache.set(entry.name, entry);
+        this._store.upsert(entry);
+        return entry;
+    }
+
+    /**
+     * Batch register tools.
+     * @param {object[]} tools
+     * @returns {number} Number of registered tools
+     */
+    registerBatch(tools) {
+        let count = 0;
+        for (const raw of tools) {
+            try { this.register(raw); count++; }
+            catch { /* skip invalid */ }
+        }
+        return count;
+    }
+
+    /**
+     * Remove a tool.
+     * @param {string} name
+     * @returns {boolean}
+     */
+    remove(name) {
+        const had = this._cache.has(name);
+        this._cache.delete(name);
+        if (had) this._store.remove(name);
+        return had;
+    }
+
+    /**
+     * Purge all tools by source (for re-sync).
+     * @param {string} source
+     * @returns {number} Number deleted
+     */
+    purgeBySource(source) {
+        let deleted = 0;
+        for (const [name, entry] of this._cache) {
+            if (entry.source === source) {
+                this._cache.delete(name);
+                deleted++;
+            }
+        }
+        this._store.purgeBySource(source);
+        return deleted;
+    }
+
+    /** @param {string} name @returns {object|null} */
+    get(name) { return this._cache.get(name) || null; }
+
+    /** @returns {object[]} */
+    getAll() { return Array.from(this._cache.values()); }
+
+    /** @returns {number} */
+    get size() { return this._cache.size; }
+
+    /**
+     * Remove all tools by provider name.
+     * @param {string} providerName
+     * @returns {number} Number of tools removed
+     */
+    removeByProvider(providerName) {
+        const toRemove = [];
+        for (const [name, entry] of this._cache) {
+            if (entry.provider === providerName) toRemove.push(name);
+        }
+        for (const name of toRemove) {
+            this.remove(name);
+        }
+        return toRemove.length;
+    }
+
+    // ── Embeddings ──
+
+    /**
+     * Precompute embeddings for tools without one.
+     * @returns {Promise<{embedded: number, skipped: number, failed: number}>}
+     */
+    async precomputeEmbeddings() {
+        if (!this._embedding) return { embedded: 0, skipped: 0, failed: 0 };
+        const available = await this._embedding.isAvailable();
+        if (!available) return { embedded: 0, skipped: 0, failed: 0 };
+
+        let embedded = 0, skipped = 0, failed = 0;
+        for (const [, entry] of this._cache) {
+            if (entry.embedding) { skipped++; continue; }
+            try {
+                const text = entry.searchText || buildSearchText(entry);
+                const vec = await this._embedding.embed(text);
+                if (vec.length > 0) {
+                    entry.embedding = vec;
+                    this._store.upsert(entry);
+                    embedded++;
+                } else { failed++; }
+            } catch { failed++; }
+        }
+        return { embedded, skipped, failed };
+    }
+
+    // ── Usage tracking ──
+
+    /**
+     * Record tool usage.
+     * @param {string} name
+     * @param {boolean} success
+     */
+    recordUsage(name, success = true) {
+        const entry = this._cache.get(name);
+        if (!entry) return;
+        entry.usageCount++;
+        const alpha = 0.1;
+        entry.successRate = entry.successRate * (1 - alpha) + (success ? 1 : 0) * alpha;
+        entry.updatedAt = new Date().toISOString();
+        this._store.upsert(entry);
+    }
+
+    // ── Stats ──
+
+    /** @returns {object} */
+    stats() {
+        const bySource = {};
+        const byCategory = {};
+        let withEmbedding = 0;
+        for (const entry of this._cache.values()) {
+            bySource[entry.source] = (bySource[entry.source] || 0) + 1;
+            byCategory[entry.category] = (byCategory[entry.category] || 0) + 1;
+            if (entry.embedding) withEmbedding++;
+        }
+        return {
+            total: this._cache.size,
+            bySource,
+            byCategory,
+            withEmbedding,
+            embeddingCoverage: this._cache.size > 0
+                ? Math.round((withEmbedding / this._cache.size) * 100) + '%' : '0%',
+        };
+    }
+
+    // ── Internal ──
+
+    /** Convert SQLite row to tool entry */
+    _rowToEntry(row) {
+        return {
+            name: row.name,
+            description: row.description || '',
+            source: row.source || 'unknown',
+            category: row.category || 'misc',
+            provider: row.provider || row.plugin_name || '',
+            inputSchema: _parseJson(row.input_schema, null),
+            triggers: _parseJson(row.triggers, []),
+            tags: _parseJson(row.tags, []),
+            examples: _parseJson(row.examples, []),
+            endpoint: row.endpoint || '',
+            searchText: row.search_text || '',
+            embedding: _parseJson(row.embedding, null),
+            usageCount: row.usage_count || 0,
+            successRate: row.success_rate ?? 1.0,
+            registeredAt: row.registered_at,
+            updatedAt: row.updated_at,
+        };
+    }
+}
+
+function _parseJson(str, fallback) {
+    if (!str || str === '') return fallback;
+    try { return JSON.parse(str); } catch { return fallback; }
+}
+
+module.exports = { Registry };

package/lib/router.js

@@ -0,0 +1,160 @@
+// QLN — L1 Router (3-Stage parallel search engine)
+// Query → Stage1(Trigger) + Stage2(Keyword) + Stage3(Semantic) → Merge → Top-K
+const { buildSearchText } = require('./schema');
+
+/**
+ * 3-Stage search engine.
+ *
+ * Score formula:
+ *   final = trigger×3.0 + keyword×1.0 + semantic×2.0
+ *         + log2(usageCount+1)×0.5 + successRate×1.0
+ */
+class Router {
+    /**
+     * @param {import('./registry').Registry} registry
+     * @param {import('./vector-index').VectorIndex} vectorIndex
+     * @param {import('./embedding').Embedding} [embedding]
+     */
+    constructor(registry, vectorIndex, embedding = null) {
+        this._registry = registry;
+        this._vectorIndex = vectorIndex;
+        this._embedding = embedding;
+    }
+
+    /**
+     * Route natural language query to tools.
+     * @param {string} query - Natural language (e.g. "take a screenshot")
+     * @param {{topK?: number, threshold?: number}} [options]
+     * @returns {Promise<{results: object[], timing: object}>}
+     */
+    async route(query, options = {}) {
+        const topK = options.topK || 5;
+        const threshold = options.threshold || 0.1;
+        const scores = new Map();
+        const timing = { stage1: 0, stage2: 0, stage3: 0, merge: 0, total: 0 };
+        const t0 = Date.now();
+
+        // Stage 1: Trigger exact match (fastest)
+        const t1 = Date.now();
+        this._stage1TriggerMatch(query, scores);
+        timing.stage1 = Date.now() - t1;
+
+        // Stage 2: Keyword match (search_text LIKE)
+        const t2 = Date.now();
+        this._stage2KeywordMatch(query, scores);
+        timing.stage2 = Date.now() - t2;
+
+        // Stage 3: Semantic vector search (when embedding available)
+        const t3 = Date.now();
+        await this._stage3SemanticSearch(query, scores);
+        timing.stage3 = Date.now() - t3;
+
+        // Merge: Calculate final scores
+        const t4 = Date.now();
+        const results = this._mergeAndRank(scores, topK, threshold);
+        timing.merge = Date.now() - t4;
+        timing.total = Date.now() - t0;
+
+        return { results, timing };
+    }
+
+    /** Stage 1: Trigger word exact match. Weight: 3.0 */
+    _stage1TriggerMatch(query, scores) {
+        const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 1);
+        for (const tool of this._registry.getAll()) {
+            const triggers = tool.triggers || [];
+            let hits = 0;
+            for (const word of queryWords) {
+                if (triggers.some(t => t === word || t.includes(word))) hits++;
+            }
+            if (hits > 0) {
+                this._getOrCreate(scores, tool.name).stage1 = hits * 3.0;
+            }
+        }
+    }
+
+    /** Stage 2: search_text keyword match. Weight: 1.0 */
+    _stage2KeywordMatch(query, scores) {
+        const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2);
+        for (const tool of this._registry.getAll()) {
+            const text = (tool.searchText || buildSearchText(tool)).toLowerCase();
+            let matchCount = 0;
+            for (const word of queryWords) {
+                if (text.includes(word)) matchCount++;
+            }
+            if (matchCount > 0) {
+                this._getOrCreate(scores, tool.name).stage2 =
+                    (matchCount / Math.max(queryWords.length, 1)) * 1.0;
+            }
+        }
+    }
+
+    /** Stage 3: Semantic vector search. Weight: 2.0 */
+    async _stage3SemanticSearch(query, scores) {
+        if (!this._embedding || !this._vectorIndex) return;
+        try {
+            const available = await this._embedding.isAvailable();
+            if (!available) return;
+            const queryVec = await this._embedding.embed(query);
+            if (!queryVec || queryVec.length === 0) return;
+            const semanticResults = this._vectorIndex.search(queryVec, 20);
+            for (const r of semanticResults) {
+                this._getOrCreate(scores, r.name).stage3 = r.score * 2.0;
+            }
+        } catch { /* graceful degradation */ }
+    }
+
+    /** Merge all stage results + usage/success bonus → ranking */
+    _mergeAndRank(scores, topK, threshold) {
+        const results = [];
+        for (const [name, s] of scores) {
+            const tool = this._registry.get(name);
+            if (!tool) continue;
+            const usageBonus = Math.log2((tool.usageCount || 0) + 1) * 0.5;
+            const successBonus = (tool.successRate ?? 1.0) * 1.0;
+            const finalScore = (s.stage1 || 0) + (s.stage2 || 0) + (s.stage3 || 0)
+                + usageBonus + successBonus;
+            if (finalScore >= threshold) {
+                results.push({
+                    name,
+                    score: Math.round(finalScore * 100) / 100,
+                    stages: {
+                        trigger: s.stage1 || 0,
+                        keyword: s.stage2 || 0,
+                        semantic: s.stage3 || 0,
+                        usage: Math.round(usageBonus * 100) / 100,
+                        success: Math.round(successBonus * 100) / 100,
+                    },
+                    description: tool.description,
+                    source: tool.source,
+                    category: tool.category,
+                    inputSchema: tool.inputSchema,
+                });
+            }
+        }
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, topK);
+    }
+
+    /** Build vector index */
+    buildIndex() {
+        return this._vectorIndex.build(this._registry.getAll());
+    }
+
+    /** @private */
+    _getOrCreate(scores, name) {
+        if (!scores.has(name)) scores.set(name, { stage1: 0, stage2: 0, stage3: 0 });
+        return scores.get(name);
+    }
+
+    /** @returns {object} */
+    stats() {
+        return {
+            registrySize: this._registry.size,
+            vectorIndex: this._vectorIndex.stats(),
+            embeddingAvailable: !!this._embedding,
+        };
+    }
+}
+
+module.exports = { Router };