@vainplex/openclaw-knowledge-engine 0.1.0

package/dist/src/entity-extractor.d.ts

@@ -0,0 +1,30 @@
+import { Entity, Logger } from './types.js';
+/**
+ * Extracts entities from text using predefined regular expressions.
+ */
+export declare class EntityExtractor {
+    private readonly logger;
+    constructor(logger: Logger);
+    /**
+     * Extracts entities from a given text based on the regex patterns.
+     * @param text The input text to process.
+     * @returns An array of found entities.
+     */
+    extract(text: string): Entity[];
+    /**
+     * Processes a single regex match and upserts it into the entity map.
+     */
+    private processMatch;
+    /**
+     * Cleans and standardizes an entity value based on its type.
+     */
+    private canonicalize;
+    /**
+     * Calculates an initial importance score for an entity.
+     */
+    private calculateInitialImportance;
+    /**
+     * Merges two lists of entities by ID.
+     */
+    static mergeEntities(listA: Entity[], listB: Entity[]): Entity[];
+}

package/dist/src/entity-extractor.js

@@ -0,0 +1,123 @@
+// src/entity-extractor.ts
+import { REGEX_PATTERNS } from './patterns.js';
+// A map to associate regex pattern names with entity types.
+const PATTERN_TYPE_MAP = {
+    email: 'email',
+    url: 'url',
+    iso_date: 'date',
+    common_date: 'date',
+    german_date: 'date',
+    english_date: 'date',
+    proper_noun: 'unknown',
+    product_name: 'product',
+    organization_suffix: 'organization',
+};
+/**
+ * Extracts entities from text using predefined regular expressions.
+ */
+export class EntityExtractor {
+    logger;
+    constructor(logger) {
+        this.logger = logger;
+    }
+    /**
+     * Extracts entities from a given text based on the regex patterns.
+     * @param text The input text to process.
+     * @returns An array of found entities.
+     */
+    extract(text) {
+        const foundEntities = new Map();
+        for (const key in REGEX_PATTERNS) {
+            // Each access returns a fresh RegExp (via Proxy), avoiding /g state-bleed.
+            const regex = REGEX_PATTERNS[key];
+            if (!regex.global) {
+                this.logger.warn(`Regex for "${key}" is not global. Skipping.`);
+                continue;
+            }
+            const entityType = PATTERN_TYPE_MAP[key] || 'unknown';
+            let match;
+            while ((match = regex.exec(text)) !== null) {
+                const value = match[0].trim();
+                if (!value)
+                    continue;
+                this.processMatch(key, value, entityType, foundEntities);
+            }
+        }
+        return Array.from(foundEntities.values());
+    }
+    /**
+     * Processes a single regex match and upserts it into the entity map.
+     */
+    processMatch(_key, value, entityType, entities) {
+        const canonicalValue = this.canonicalize(value, entityType);
+        const id = `${entityType}:${canonicalValue.toLowerCase().replace(/\s+/g, '-')}`;
+        if (entities.has(id)) {
+            const existing = entities.get(id);
+            if (!existing.mentions.includes(value))
+                existing.mentions.push(value);
+            existing.count++;
+            if (!existing.source.includes('regex'))
+                existing.source.push('regex');
+        }
+        else {
+            entities.set(id, {
+                id,
+                type: entityType,
+                value: canonicalValue,
+                mentions: [value],
+                count: 1,
+                importance: this.calculateInitialImportance(entityType, value),
+                lastSeen: new Date().toISOString(),
+                source: ['regex'],
+            });
+        }
+    }
+    /**
+     * Cleans and standardizes an entity value based on its type.
+     */
+    canonicalize(value, type) {
+        if (type === 'organization') {
+            const suffixes = /,?\s?(?:Inc\.|LLC|Corp\.|GmbH|AG|Ltd\.)$/i;
+            return value.replace(suffixes, '').trim();
+        }
+        return value.replace(/[.,!?;:]$/, '').trim();
+    }
+    /**
+     * Calculates an initial importance score for an entity.
+     */
+    calculateInitialImportance(type, value) {
+        switch (type) {
+            case 'organization': return 0.8;
+            case 'person': return 0.7;
+            case 'product': return 0.6;
+            case 'location': return 0.5;
+            case 'date':
+            case 'email':
+            case 'url': return 0.4;
+            default: return value.split(/\s|-/).length > 1 ? 0.5 : 0.3;
+        }
+    }
+    /**
+     * Merges two lists of entities by ID.
+     */
+    static mergeEntities(listA, listB) {
+        const merged = new Map();
+        for (const e of listA)
+            merged.set(e.id, { ...e });
+        for (const entity of listB) {
+            if (merged.has(entity.id)) {
+                const ex = merged.get(entity.id);
+                ex.count += entity.count;
+                ex.mentions = [...new Set([...ex.mentions, ...entity.mentions])];
+                ex.source = [...new Set([...ex.source, ...entity.source])];
+                ex.lastSeen = new Date() > new Date(ex.lastSeen)
+                    ? new Date().toISOString() : ex.lastSeen;
+                ex.importance = Math.max(ex.importance, entity.importance);
+            }
+            else {
+                merged.set(entity.id, { ...entity });
+            }
+        }
+        return Array.from(merged.values());
+    }
+}

package/dist/src/fact-store.d.ts

@@ -0,0 +1,77 @@
+import type { Fact, KnowledgeConfig, Logger } from './types.js';
+/**
+ * Manages an in-memory and on-disk store of structured facts.
+ * Provides methods for loading, querying, modifying, and persisting facts.
+ */
+export declare class FactStore {
+    private readonly storage;
+    private readonly config;
+    private readonly logger;
+    private facts;
+    private isLoaded;
+    readonly commit: () => Promise<void>;
+    constructor(workspace: string, config: KnowledgeConfig['storage'], logger: Logger);
+    /**
+     * Immediately flushes any pending debounced writes.
+     * Useful in tests and before shutdown to ensure data is persisted.
+     */
+    flush(): Promise<void>;
+    /**
+     * Loads facts from the `facts.json` file into the in-memory store.
+     * If the file doesn't exist, it initializes an empty store.
+     */
+    load(): Promise<void>;
+    /**
+     * Adds a new fact to the store or updates an existing one based on content.
+     * @param newFactData The data for the new fact, excluding metadata fields.
+     * @returns The newly created or found Fact object.
+     */
+    addFact(newFactData: Omit<Fact, 'id' | 'createdAt' | 'lastAccessed' | 'relevance'>): Fact;
+    /**
+     * Retrieves a fact by its unique ID.
+     * @param id The UUID of the fact.
+     * @returns The Fact object, or undefined if not found.
+     */
+    getFact(id: string): Fact | undefined;
+    /**
+     * Queries the fact store based on subject, predicate, or object.
+     * @param query An object with optional subject, predicate, and/or object to match.
+     * @returns An array of matching facts, sorted by relevance.
+     */
+    query(query: {
+        subject?: string;
+        predicate?: string;
+        object?: string;
+    }): Fact[];
+    /**
+     * Applies a decay factor to the relevance score of all facts.
+     * @param rate The decay rate (e.g., 0.05 for 5%).
+     * @returns An object with the count of decayed facts.
+     */
+    decayFacts(rate: number): {
+        decayedCount: number;
+    };
+    /**
+     * Persists the current in-memory fact store to `facts.json`.
+     */
+    private persist;
+    /**
+     * Removes the least relevant facts if the store exceeds its configured max size.
+     */
+    private prune;
+    /**
+     * Boosts the relevance of a fact upon access.
+     * @param currentRelevance The current relevance score.
+     * @returns The new, boosted relevance score.
+     */
+    private boostRelevance;
+    /**
+     * Returns a list of all facts that have not been embedded yet.
+     */
+    getUnembeddedFacts(): Fact[];
+    /**
+     * Marks a list of facts as having been embedded.
+     * @param factIds An array of fact IDs to update.
+     */
+    markFactsAsEmbedded(factIds: string[]): void;
+}

package/dist/src/fact-store.js

@@ -0,0 +1,222 @@
+// src/fact-store.ts
+import { randomUUID } from 'node:crypto';
+import { AtomicStorage } from './storage.js';
+/**
+ * Manages an in-memory and on-disk store of structured facts.
+ * Provides methods for loading, querying, modifying, and persisting facts.
+ */
+export class FactStore {
+    storage;
+    config;
+    logger;
+    facts = new Map();
+    isLoaded = false;
+    commit;
+    constructor(workspace, config, logger) {
+        this.storage = new AtomicStorage(workspace, logger);
+        this.config = config;
+        this.logger = logger;
+        // Create a debounced version of the persist method.
+        this.commit = AtomicStorage.debounce(this.persist.bind(this), this.config.writeDebounceMs);
+    }
+    /**
+     * Immediately flushes any pending debounced writes.
+     * Useful in tests and before shutdown to ensure data is persisted.
+     */
+    async flush() {
+        if (this.isLoaded) {
+            await this.persist();
+        }
+    }
+    /**
+     * Loads facts from the `facts.json` file into the in-memory store.
+     * If the file doesn't exist, it initializes an empty store.
+     */
+    async load() {
+        if (this.isLoaded) {
+            this.logger.debug('Fact store is already loaded.');
+            return;
+        }
+        await this.storage.init();
+        const data = await this.storage.readJson('facts.json');
+        if (data && Array.isArray(data.facts)) {
+            this.facts = new Map(data.facts.map(fact => [fact.id, fact]));
+            this.logger.info(`Loaded ${this.facts.size} facts from storage.`);
+        }
+        else {
+            this.logger.info('No existing fact store found. Initializing a new one.');
+            this.facts = new Map();
+        }
+        this.isLoaded = true;
+    }
+    /**
+     * Adds a new fact to the store or updates an existing one based on content.
+     * @param newFactData The data for the new fact, excluding metadata fields.
+     * @returns The newly created or found Fact object.
+     */
+    addFact(newFactData) {
+        if (!this.isLoaded) {
+            throw new Error('FactStore has not been loaded yet. Call load() first.');
+        }
+        const now = new Date().toISOString();
+        // Check if a similar fact already exists to avoid duplicates
+        for (const existingFact of this.facts.values()) {
+            if (existingFact.subject === newFactData.subject &&
+                existingFact.predicate === newFactData.predicate &&
+                existingFact.object === newFactData.object) {
+                // Fact already exists, let's just boost its relevance and update timestamp
+                existingFact.relevance = this.boostRelevance(existingFact.relevance);
+                existingFact.lastAccessed = now;
+                this.commit();
+                return existingFact;
+            }
+        }
+        const newFact = {
+            ...newFactData,
+            id: randomUUID(),
+            createdAt: now,
+            lastAccessed: now,
+            relevance: 1.0, // New facts start with maximum relevance
+        };
+        this.facts.set(newFact.id, newFact);
+        this.prune(); // Check if we need to prune old facts
+        this.commit();
+        return newFact;
+    }
+    /**
+     * Retrieves a fact by its unique ID.
+     * @param id The UUID of the fact.
+     * @returns The Fact object, or undefined if not found.
+     */
+    getFact(id) {
+        const fact = this.facts.get(id);
+        if (fact) {
+            fact.lastAccessed = new Date().toISOString();
+            fact.relevance = this.boostRelevance(fact.relevance);
+            this.commit();
+        }
+        return fact;
+    }
+    /**
+     * Queries the fact store based on subject, predicate, or object.
+     * @param query An object with optional subject, predicate, and/or object to match.
+     * @returns An array of matching facts, sorted by relevance.
+     */
+    query(query) {
+        const results = [];
+        for (const fact of this.facts.values()) {
+            const subjectMatch = !query.subject || fact.subject === query.subject;
+            const predicateMatch = !query.predicate || fact.predicate === query.predicate;
+            const objectMatch = !query.object || fact.object === query.object;
+            if (subjectMatch && predicateMatch && objectMatch) {
+                results.push(fact);
+            }
+        }
+        // Sort by relevance, descending
+        return results.sort((a, b) => b.relevance - a.relevance);
+    }
+    /**
+     * Applies a decay factor to the relevance score of all facts.
+     * @param rate The decay rate (e.g., 0.05 for 5%).
+     * @returns An object with the count of decayed facts.
+     */
+    decayFacts(rate) {
+        let decayedCount = 0;
+        const minRelevance = 0.1; // Floor to prevent facts from disappearing completely
+        for (const fact of this.facts.values()) {
+            const newRelevance = fact.relevance * (1 - rate);
+            if (newRelevance !== fact.relevance) {
+                fact.relevance = Math.max(minRelevance, newRelevance);
+                decayedCount++;
+            }
+        }
+        if (decayedCount > 0) {
+            this.logger.info(`Applied decay rate of ${rate * 100}% to ${decayedCount} facts.`);
+            this.commit();
+        }
+        return { decayedCount };
+    }
+    /**
+     * Persists the current in-memory fact store to `facts.json`.
+     */
+    async persist() {
+        if (!this.isLoaded) {
+            this.logger.warn('Attempted to persist fact store before it was loaded. Aborting.');
+            return;
+        }
+        const data = {
+            updated: new Date().toISOString(),
+            facts: Array.from(this.facts.values()),
+        };
+        try {
+            await this.storage.writeJson('facts.json', data);
+            this.logger.debug(`Successfully persisted ${data.facts.length} facts.`);
+        }
+        catch (err) {
+            this.logger.error('Failed to persist fact store.', err);
+        }
+    }
+    /**
+     * Removes the least relevant facts if the store exceeds its configured max size.
+     */
+    prune() {
+        const factCount = this.facts.size;
+        if (factCount <= this.config.maxFacts) {
+            return;
+        }
+        const factsToPrune = factCount - this.config.maxFacts;
+        if (factsToPrune <= 0)
+            return;
+        // Get all facts, sort by relevance (ascending) and then by lastAccessed (ascending)
+        const sortedFacts = Array.from(this.facts.values()).sort((a, b) => {
+            if (a.relevance !== b.relevance) {
+                return a.relevance - b.relevance;
+            }
+            return new Date(a.lastAccessed).getTime() - new Date(b.lastAccessed).getTime();
+        });
+        for (let i = 0; i < factsToPrune; i++) {
+            this.facts.delete(sortedFacts[i].id);
+        }
+        this.logger.info(`Pruned ${factsToPrune} least relevant facts to maintain store size.`);
+    }
+    /**
+     * Boosts the relevance of a fact upon access.
+     * @param currentRelevance The current relevance score.
+     * @returns The new, boosted relevance score.
+     */
+    boostRelevance(currentRelevance) {
+        // Push the relevance 50% closer to 1.0
+        const boost = (1.0 - currentRelevance) * 0.5;
+        return Math.min(1.0, currentRelevance + boost);
+    }
+    /**
+     * Returns a list of all facts that have not been embedded yet.
+     */
+    getUnembeddedFacts() {
+        const results = [];
+        for (const fact of this.facts.values()) {
+            if (!fact.embedded) {
+                results.push(fact);
+            }
+        }
+        return results;
+    }
+    /**
+     * Marks a list of facts as having been embedded.
+     * @param factIds An array of fact IDs to update.
+     */
+    markFactsAsEmbedded(factIds) {
+        const now = new Date().toISOString();
+        let updatedCount = 0;
+        for (const id of factIds) {
+            const fact = this.facts.get(id);
+            if (fact) {
+                fact.embedded = now;
+                updatedCount++;
+            }
+        }
+        if (updatedCount > 0) {
+            this.commit();
+        }
+    }
+}

package/dist/src/hooks.d.ts

@@ -0,0 +1,24 @@
+import { OpenClawPluginApi, KnowledgeConfig } from './types.js';
+/**
+ * Manages the registration and orchestration of all plugin hooks.
+ */
+export declare class HookManager {
+    private readonly api;
+    private readonly config;
+    private readonly logger;
+    private entityExtractor;
+    private factStore;
+    private llmEnhancer?;
+    private maintenance?;
+    constructor(api: OpenClawPluginApi, config: KnowledgeConfig);
+    /** Registers all the necessary hooks with the OpenClaw host. */
+    registerHooks(): void;
+    /** Handler for the `session_start` hook. */
+    private onSessionStart;
+    /** Handler for `gateway_stop` — cleans up timers and flushes state. */
+    private onShutdown;
+    /** Handler for `message_received` and `message_sent` hooks. */
+    private onMessage;
+    /** Fire-and-forget: processes LLM batch results when available. */
+    private processLlmBatchWhenReady;
+}

package/dist/src/hooks.js

@@ -0,0 +1,94 @@
+// src/hooks.ts
+import { EntityExtractor } from './entity-extractor.js';
+import { FactStore } from './fact-store.js';
+import { LlmEnhancer } from './llm-enhancer.js';
+import { Maintenance } from './maintenance.js';
+import { Embeddings } from './embeddings.js';
+/**
+ * Manages the registration and orchestration of all plugin hooks.
+ */
+export class HookManager {
+    api;
+    config;
+    logger;
+    entityExtractor;
+    factStore;
+    llmEnhancer;
+    maintenance;
+    constructor(api, config) {
+        this.api = api;
+        this.config = config;
+        this.logger = api.logger;
+        this.entityExtractor = new EntityExtractor(this.logger);
+        this.factStore = new FactStore(this.config.workspace, this.config.storage, this.logger);
+        if (this.config.extraction.llm.enabled) {
+            this.llmEnhancer = new LlmEnhancer(this.config.extraction.llm, this.logger);
+        }
+    }
+    /** Registers all the necessary hooks with the OpenClaw host. */
+    registerHooks() {
+        if (!this.config.enabled) {
+            this.logger.info('Knowledge Engine is disabled. No hooks registered.');
+            return;
+        }
+        this.api.on('session_start', this.onSessionStart.bind(this), { priority: 200 });
+        this.api.on('message_received', this.onMessage.bind(this), { priority: 100 });
+        this.api.on('message_sent', this.onMessage.bind(this), { priority: 100 });
+        this.api.on('gateway_stop', this.onShutdown.bind(this), { priority: 900 });
+        this.logger.info('Registered all Knowledge Engine hooks.');
+    }
+    /** Handler for the `session_start` hook. */
+    async onSessionStart() {
+        this.logger.info('Knowledge Engine starting up...');
+        await this.factStore.load();
+        const embeddings = this.config.embeddings.enabled
+            ? new Embeddings(this.config.embeddings, this.logger)
+            : undefined;
+        this.maintenance = new Maintenance(this.config, this.logger, this.factStore, embeddings);
+        this.maintenance.start();
+    }
+    /** Handler for `gateway_stop` — cleans up timers and flushes state. */
+    async onShutdown() {
+        this.logger.info('Knowledge Engine shutting down...');
+        this.maintenance?.stop();
+        this.llmEnhancer?.clearTimers();
+        this.logger.info('Knowledge Engine shutdown complete.');
+    }
+    /** Handler for `message_received` and `message_sent` hooks. */
+    async onMessage(event) {
+        const text = event.content || event.message || event.text;
+        if (typeof text !== 'string' || text.trim() === '')
+            return;
+        this.logger.debug(`Processing message: "${text.substring(0, 50)}..."`);
+        if (this.config.extraction.regex.enabled) {
+            const entities = this.entityExtractor.extract(text);
+            if (entities.length > 0) {
+                this.logger.info(`Extracted ${entities.length} entities via regex.`);
+            }
+        }
+        if (this.llmEnhancer) {
+            const messageId = `msg-${Date.now()}`;
+            this.llmEnhancer.addToBatch({ id: messageId, text });
+            this.processLlmBatchWhenReady().catch(err => this.logger.error('LLM batch processing failed', err));
+        }
+    }
+    /** Fire-and-forget: processes LLM batch results when available. */
+    async processLlmBatchWhenReady() {
+        if (!this.llmEnhancer)
+            return;
+        const result = await this.llmEnhancer.sendBatch();
+        if (!result)
+            return;
+        if (result.facts.length > 0) {
+            this.logger.info(`Adding ${result.facts.length} LLM facts.`);
+            for (const f of result.facts) {
+                this.factStore.addFact({
+                    subject: f.subject,
+                    predicate: f.predicate,
+                    object: f.object,
+                    source: 'extracted-llm',
+                });
+            }
+        }
+    }
+}

package/dist/src/http-client.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Makes an HTTP or HTTPS POST request, auto-selecting the
+ * transport based on the URL's protocol.
+ *
+ * @param url  The full URL string to POST to.
+ * @param body The payload object to JSON-serialize and send.
+ * @returns A promise resolving with the response body string.
+ */
+export declare function httpPost(url: string, body: unknown): Promise<string>;

package/dist/src/http-client.js

@@ -0,0 +1,58 @@
+// src/http-client.ts
+import * as http from 'node:http';
+import * as https from 'node:https';
+/**
+ * Selects the correct HTTP/HTTPS module based on the URL protocol.
+ */
+function selectTransport(protocol) {
+    return protocol === 'https:' ? https : http;
+}
+/**
+ * Builds request options from a URL and payload.
+ */
+function buildRequestOptions(url, payload) {
+    return {
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname + url.search,
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Content-Length': Buffer.byteLength(payload),
+        },
+    };
+}
+/**
+ * Makes an HTTP or HTTPS POST request, auto-selecting the
+ * transport based on the URL's protocol.
+ *
+ * @param url  The full URL string to POST to.
+ * @param body The payload object to JSON-serialize and send.
+ * @returns A promise resolving with the response body string.
+ */
+export function httpPost(url, body) {
+    return new Promise((resolve, reject) => {
+        const parsed = new URL(url);
+        const payload = JSON.stringify(body);
+        const options = buildRequestOptions(parsed, payload);
+        const transport = selectTransport(parsed.protocol);
+        const req = transport.request(options, (res) => {
+            let data = '';
+            res.setEncoding('utf8');
+            res.on('data', (chunk) => { data += chunk; });
+            res.on('end', () => {
+                if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
+                    resolve(data);
+                }
+                else {
+                    reject(new Error(`HTTP request failed with status ${res.statusCode}: ${data}`));
+                }
+            });
+        });
+        req.on('error', (e) => {
+            reject(new Error(`HTTP request error: ${e.message}`));
+        });
+        req.write(payload);
+        req.end();
+    });
+}

package/dist/src/llm-enhancer.d.ts

@@ -0,0 +1,44 @@
+import { Entity, Fact, KnowledgeConfig, Logger } from './types.js';
+interface LlmBatchItem {
+    id: string;
+    text: string;
+}
+/**
+ * Manages batched requests to an external LLM for entity and fact extraction.
+ */
+export declare class LlmEnhancer {
+    private readonly config;
+    private readonly logger;
+    private batch;
+    private cooldownTimeout;
+    constructor(config: KnowledgeConfig['extraction']['llm'], logger: Logger);
+    /**
+     * Adds a message to the current batch.
+     * Triggers a batch send (with proper error handling) when the size is reached.
+     */
+    addToBatch(item: LlmBatchItem): void;
+    /** Resets the cooldown timer. When it expires the batch is sent. */
+    private resetCooldownTimer;
+    /**
+     * Clears all pending timers. Called during shutdown.
+     */
+    clearTimers(): void;
+    /**
+     * Sends the current batch to the LLM for processing.
+     */
+    sendBatch(): Promise<{
+        entities: Entity[];
+        facts: Fact[];
+    } | null>;
+    /** Constructs the prompt to be sent to the LLM. */
+    private constructPrompt;
+    /** Makes an HTTP(S) request to the configured LLM endpoint. */
+    private makeHttpRequest;
+    /** Parses and validates the JSON response from the LLM. */
+    private parseLlmResponse;
+    /** Transforms raw LLM entity data into the standard Entity format. */
+    private transformToEntities;
+    /** Transforms raw LLM fact data into partial Fact objects. */
+    private transformToFacts;
+}
+export {};