@vainplex/openclaw-knowledge-engine 0.1.0

package/src/fact-store.ts

@@ -0,0 +1,260 @@
+// src/fact-store.ts
+
+import { randomUUID } from 'node:crypto';
+import { AtomicStorage } from './storage.js';
+import type { Fact, FactsData, KnowledgeConfig, Logger } from './types.js';
+
+/**
+ * Manages an in-memory and on-disk store of structured facts.
+ * Provides methods for loading, querying, modifying, and persisting facts.
+ */
+export class FactStore {
+  private readonly storage: AtomicStorage;
+  private readonly config: KnowledgeConfig['storage'];
+  private readonly logger: Logger;
+  private facts: Map<string, Fact> = new Map();
+  private isLoaded: boolean = false;
+
+  public readonly commit: () => Promise<void>;
+
+  constructor(
+    workspace: string,
+    config: KnowledgeConfig['storage'],
+    logger: Logger
+  ) {
+    this.storage = new AtomicStorage(workspace, logger);
+    this.config = config;
+    this.logger = logger;
+
+    // Create a debounced version of the persist method.
+    this.commit = AtomicStorage.debounce(
+      this.persist.bind(this),
+      this.config.writeDebounceMs
+    );
+  }
+
+  /**
+   * Immediately flushes any pending debounced writes.
+   * Useful in tests and before shutdown to ensure data is persisted.
+   */
+  public async flush(): Promise<void> {
+    if (this.isLoaded) {
+      await this.persist();
+    }
+  }
+
+  /**
+   * Loads facts from the `facts.json` file into the in-memory store.
+   * If the file doesn't exist, it initializes an empty store.
+   */
+  public async load(): Promise<void> {
+    if (this.isLoaded) {
+      this.logger.debug('Fact store is already loaded.');
+      return;
+    }
+
+    await this.storage.init();
+    const data = await this.storage.readJson<FactsData>('facts.json');
+    if (data && Array.isArray(data.facts)) {
+      this.facts = new Map(data.facts.map(fact => [fact.id, fact]));
+      this.logger.info(`Loaded ${this.facts.size} facts from storage.`);
+    } else {
+      this.logger.info('No existing fact store found. Initializing a new one.');
+      this.facts = new Map();
+    }
+    this.isLoaded = true;
+  }
+
+  /**
+   * Adds a new fact to the store or updates an existing one based on content.
+   * @param newFactData The data for the new fact, excluding metadata fields.
+   * @returns The newly created or found Fact object.
+   */
+  public addFact(
+    newFactData: Omit<Fact, 'id' | 'createdAt' | 'lastAccessed' | 'relevance'>
+  ): Fact {
+    if (!this.isLoaded) {
+      throw new Error('FactStore has not been loaded yet. Call load() first.');
+    }
+    const now = new Date().toISOString();
+    
+    // Check if a similar fact already exists to avoid duplicates
+    for (const existingFact of this.facts.values()) {
+        if (
+            existingFact.subject === newFactData.subject &&
+            existingFact.predicate === newFactData.predicate &&
+            existingFact.object === newFactData.object
+        ) {
+            // Fact already exists, let's just boost its relevance and update timestamp
+            existingFact.relevance = this.boostRelevance(existingFact.relevance);
+            existingFact.lastAccessed = now;
+            this.commit();
+            return existingFact;
+        }
+    }
+
+    const newFact: Fact = {
+      ...newFactData,
+      id: randomUUID(),
+      createdAt: now,
+      lastAccessed: now,
+      relevance: 1.0, // New facts start with maximum relevance
+    };
+
+    this.facts.set(newFact.id, newFact);
+    this.prune(); // Check if we need to prune old facts
+    this.commit();
+    return newFact;
+  }
+
+  /**
+   * Retrieves a fact by its unique ID.
+   * @param id The UUID of the fact.
+   * @returns The Fact object, or undefined if not found.
+   */
+  public getFact(id: string): Fact | undefined {
+    const fact = this.facts.get(id);
+    if (fact) {
+        fact.lastAccessed = new Date().toISOString();
+        fact.relevance = this.boostRelevance(fact.relevance);
+        this.commit();
+    }
+    return fact;
+  }
+
+  /**
+   * Queries the fact store based on subject, predicate, or object.
+   * @param query An object with optional subject, predicate, and/or object to match.
+   * @returns An array of matching facts, sorted by relevance.
+   */
+  public query(query: { subject?: string; predicate?: string; object?: string }): Fact[] {
+    const results: Fact[] = [];
+    for (const fact of this.facts.values()) {
+      const subjectMatch = !query.subject || fact.subject === query.subject;
+      const predicateMatch = !query.predicate || fact.predicate === query.predicate;
+      const objectMatch = !query.object || fact.object === query.object;
+      
+      if (subjectMatch && predicateMatch && objectMatch) {
+        results.push(fact);
+      }
+    }
+    // Sort by relevance, descending
+    return results.sort((a, b) => b.relevance - a.relevance);
+  }
+
+  /**
+   * Applies a decay factor to the relevance score of all facts.
+   * @param rate The decay rate (e.g., 0.05 for 5%).
+   * @returns An object with the count of decayed facts.
+   */
+  public decayFacts(rate: number): { decayedCount: number } {
+    let decayedCount = 0;
+    const minRelevance = 0.1; // Floor to prevent facts from disappearing completely
+
+    for (const fact of this.facts.values()) {
+      const newRelevance = fact.relevance * (1 - rate);
+      if (newRelevance !== fact.relevance) {
+          fact.relevance = Math.max(minRelevance, newRelevance);
+          decayedCount++;
+      }
+    }
+    
+    if (decayedCount > 0) {
+      this.logger.info(`Applied decay rate of ${rate * 100}% to ${decayedCount} facts.`);
+      this.commit();
+    }
+    return { decayedCount };
+  }
+
+  /**
+   * Persists the current in-memory fact store to `facts.json`.
+   */
+  private async persist(): Promise<void> {
+    if (!this.isLoaded) {
+      this.logger.warn('Attempted to persist fact store before it was loaded. Aborting.');
+      return;
+    }
+    
+    const data: FactsData = {
+      updated: new Date().toISOString(),
+      facts: Array.from(this.facts.values()),
+    };
+    
+    try {
+      await this.storage.writeJson('facts.json', data);
+      this.logger.debug(`Successfully persisted ${data.facts.length} facts.`);
+    } catch (err) {
+      this.logger.error('Failed to persist fact store.', err as Error);
+    }
+  }
+
+  /**
+   * Removes the least relevant facts if the store exceeds its configured max size.
+   */
+  private prune(): void {
+    const factCount = this.facts.size;
+    if (factCount <= this.config.maxFacts) {
+      return;
+    }
+
+    const factsToPrune = factCount - this.config.maxFacts;
+    if (factsToPrune <= 0) return;
+
+    // Get all facts, sort by relevance (ascending) and then by lastAccessed (ascending)
+    const sortedFacts = Array.from(this.facts.values()).sort((a, b) => {
+        if (a.relevance !== b.relevance) {
+            return a.relevance - b.relevance;
+        }
+        return new Date(a.lastAccessed).getTime() - new Date(b.lastAccessed).getTime();
+    });
+
+    for (let i = 0; i < factsToPrune; i++) {
+        this.facts.delete(sortedFacts[i].id);
+    }
+    
+    this.logger.info(`Pruned ${factsToPrune} least relevant facts to maintain store size.`);
+  }
+
+  /**
+   * Boosts the relevance of a fact upon access.
+   * @param currentRelevance The current relevance score.
+   * @returns The new, boosted relevance score.
+   */
+  private boostRelevance(currentRelevance: number): number {
+    // Push the relevance 50% closer to 1.0
+    const boost = (1.0 - currentRelevance) * 0.5;
+    return Math.min(1.0, currentRelevance + boost);
+  }
+
+  /**
+   * Returns a list of all facts that have not been embedded yet.
+   */
+  public getUnembeddedFacts(): Fact[] {
+    const results: Fact[] = [];
+    for (const fact of this.facts.values()) {
+        if (!fact.embedded) {
+            results.push(fact);
+        }
+    }
+    return results;
+  }
+
+  /**
+   * Marks a list of facts as having been embedded.
+   * @param factIds An array of fact IDs to update.
+   */
+  public markFactsAsEmbedded(factIds: string[]): void {
+    const now = new Date().toISOString();
+    let updatedCount = 0;
+    for (const id of factIds) {
+        const fact = this.facts.get(id);
+        if (fact) {
+            fact.embedded = now;
+            updatedCount++;
+        }
+    }
+    if (updatedCount > 0) {
+        this.commit();
+    }
+  }
+}

package/src/hooks.ts

@@ -0,0 +1,125 @@
+// src/hooks.ts
+
+import {
+  OpenClawPluginApi,
+  HookEvent,
+  KnowledgeConfig,
+  Logger,
+} from './types.js';
+
+import { EntityExtractor } from './entity-extractor.js';
+import { FactStore } from './fact-store.js';
+import { LlmEnhancer } from './llm-enhancer.js';
+import { Maintenance } from './maintenance.js';
+import { Embeddings } from './embeddings.js';
+
+/**
+ * Manages the registration and orchestration of all plugin hooks.
+ */
+export class HookManager {
+  private readonly api: OpenClawPluginApi;
+  private readonly config: KnowledgeConfig;
+  private readonly logger: Logger;
+
+  private entityExtractor: EntityExtractor;
+  private factStore: FactStore;
+  private llmEnhancer?: LlmEnhancer;
+  private maintenance?: Maintenance;
+
+  constructor(api: OpenClawPluginApi, config: KnowledgeConfig) {
+    this.api = api;
+    this.config = config;
+    this.logger = api.logger;
+
+    this.entityExtractor = new EntityExtractor(this.logger);
+    this.factStore = new FactStore(
+      this.config.workspace,
+      this.config.storage,
+      this.logger
+    );
+
+    if (this.config.extraction.llm.enabled) {
+      this.llmEnhancer = new LlmEnhancer(this.config.extraction.llm, this.logger);
+    }
+  }
+
+  /** Registers all the necessary hooks with the OpenClaw host. */
+  public registerHooks(): void {
+    if (!this.config.enabled) {
+      this.logger.info('Knowledge Engine is disabled. No hooks registered.');
+      return;
+    }
+
+    this.api.on('session_start', this.onSessionStart.bind(this), { priority: 200 });
+    this.api.on('message_received', this.onMessage.bind(this), { priority: 100 });
+    this.api.on('message_sent', this.onMessage.bind(this), { priority: 100 });
+    this.api.on('gateway_stop', this.onShutdown.bind(this), { priority: 900 });
+
+    this.logger.info('Registered all Knowledge Engine hooks.');
+  }
+
+  /** Handler for the `session_start` hook. */
+  private async onSessionStart(): Promise<void> {
+    this.logger.info('Knowledge Engine starting up...');
+    await this.factStore.load();
+
+    const embeddings = this.config.embeddings.enabled
+      ? new Embeddings(this.config.embeddings, this.logger)
+      : undefined;
+
+    this.maintenance = new Maintenance(
+      this.config, this.logger, this.factStore, embeddings
+    );
+    this.maintenance.start();
+  }
+
+  /** Handler for `gateway_stop` — cleans up timers and flushes state. */
+  private async onShutdown(): Promise<void> {
+    this.logger.info('Knowledge Engine shutting down...');
+    this.maintenance?.stop();
+    this.llmEnhancer?.clearTimers();
+    this.logger.info('Knowledge Engine shutdown complete.');
+  }
+
+  /** Handler for `message_received` and `message_sent` hooks. */
+  private async onMessage(event: HookEvent): Promise<void> {
+    const text = event.content || event.message || event.text;
+    if (typeof text !== 'string' || text.trim() === '') return;
+
+    this.logger.debug(`Processing message: "${text.substring(0, 50)}..."`);
+
+    if (this.config.extraction.regex.enabled) {
+      const entities = this.entityExtractor.extract(text);
+      if (entities.length > 0) {
+        this.logger.info(`Extracted ${entities.length} entities via regex.`);
+      }
+    }
+
+    if (this.llmEnhancer) {
+      const messageId = `msg-${Date.now()}`;
+      this.llmEnhancer.addToBatch({ id: messageId, text });
+      this.processLlmBatchWhenReady().catch(err =>
+        this.logger.error('LLM batch processing failed', err as Error));
+    }
+  }
+
+  /** Fire-and-forget: processes LLM batch results when available. */
+  private async processLlmBatchWhenReady(): Promise<void> {
+    if (!this.llmEnhancer) return;
+
+    const result = await this.llmEnhancer.sendBatch();
+    if (!result) return;
+
+    if (result.facts.length > 0) {
+      this.logger.info(`Adding ${result.facts.length} LLM facts.`);
+      for (const f of result.facts) {
+        this.factStore.addFact({
+          subject: f.subject,
+          predicate: f.predicate,
+          object: f.object,
+          source: 'extracted-llm',
+        });
+      }
+    }
+  }
+}

package/src/http-client.ts

@@ -0,0 +1,74 @@
+// src/http-client.ts
+
+import * as http from 'node:http';
+import * as https from 'node:https';
+
+interface HttpPostOptions {
+  hostname: string;
+  port: string;
+  path: string;
+  method: 'POST';
+  headers: Record<string, string | number>;
+}
+
+/**
+ * Selects the correct HTTP/HTTPS module based on the URL protocol.
+ */
+function selectTransport(protocol: string): typeof http | typeof https {
+  return protocol === 'https:' ? https : http;
+}
+
+/**
+ * Builds request options from a URL and payload.
+ */
+function buildRequestOptions(url: URL, payload: string): HttpPostOptions {
+  return {
+    hostname: url.hostname,
+    port: url.port,
+    path: url.pathname + url.search,
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(payload),
+    },
+  };
+}
+
+/**
+ * Makes an HTTP or HTTPS POST request, auto-selecting the
+ * transport based on the URL's protocol.
+ *
+ * @param url  The full URL string to POST to.
+ * @param body The payload object to JSON-serialize and send.
+ * @returns A promise resolving with the response body string.
+ */
+export function httpPost(url: string, body: unknown): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const payload = JSON.stringify(body);
+    const options = buildRequestOptions(parsed, payload);
+    const transport = selectTransport(parsed.protocol);
+
+    const req = transport.request(options, (res) => {
+      let data = '';
+      res.setEncoding('utf8');
+      res.on('data', (chunk: string) => { data += chunk; });
+      res.on('end', () => {
+        if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
+          resolve(data);
+        } else {
+          reject(new Error(
+            `HTTP request failed with status ${res.statusCode}: ${data}`
+          ));
+        }
+      });
+    });
+
+    req.on('error', (e) => {
+      reject(new Error(`HTTP request error: ${e.message}`));
+    });
+
+    req.write(payload);
+    req.end();
+  });
+}

package/src/llm-enhancer.ts

@@ -0,0 +1,187 @@
+// src/llm-enhancer.ts
+
+import { Entity, Fact, KnowledgeConfig, Logger } from './types.js';
+import { httpPost } from './http-client.js';
+
+interface LlmBatchItem {
+  id: string;
+  text: string;
+}
+
+interface LlmExtractionResult {
+  entities: Omit<Entity, 'id' | 'mentions' | 'count' | 'lastSeen' | 'source'>[];
+  facts: Omit<Fact, 'id' | 'relevance' | 'createdAt' | 'lastAccessed' | 'source'>[];
+}
+
+/**
+ * Manages batched requests to an external LLM for entity and fact extraction.
+ */
+export class LlmEnhancer {
+  private readonly config: KnowledgeConfig['extraction']['llm'];
+  private readonly logger: Logger;
+  private batch: LlmBatchItem[] = [];
+  private cooldownTimeout: NodeJS.Timeout | null = null;
+
+  constructor(config: KnowledgeConfig['extraction']['llm'], logger: Logger) {
+    this.config = config;
+    this.logger = logger;
+  }
+
+  /**
+   * Adds a message to the current batch.
+   * Triggers a batch send (with proper error handling) when the size is reached.
+   */
+  public addToBatch(item: LlmBatchItem): void {
+    if (!this.config.enabled) return;
+
+    this.batch.push(item);
+    this.logger.debug(`Added message ${item.id} to LLM batch. Current size: ${this.batch.length}`);
+
+    if (this.batch.length >= this.config.batchSize) {
+      this.logger.info(`LLM batch size reached (${this.config.batchSize}). Sending immediately.`);
+      // S1: properly await and catch errors from sendBatch
+      this.sendBatch().catch(err => {
+        this.logger.error('Error sending LLM batch.', err as Error);
+      });
+    } else {
+      this.resetCooldownTimer();
+    }
+  }
+
+  /** Resets the cooldown timer. When it expires the batch is sent. */
+  private resetCooldownTimer(): void {
+    if (this.cooldownTimeout) clearTimeout(this.cooldownTimeout);
+    this.cooldownTimeout = setTimeout(() => {
+      if (this.batch.length > 0) {
+        this.logger.info('LLM cooldown expired. Sending batch.');
+        this.sendBatch().catch(err => {
+          this.logger.error('Error sending LLM batch on cooldown.', err as Error);
+        });
+      }
+    }, this.config.cooldownMs);
+    this.cooldownTimeout.unref();
+  }
+
+  /**
+   * Clears all pending timers. Called during shutdown.
+   */
+  public clearTimers(): void {
+    if (this.cooldownTimeout) {
+      clearTimeout(this.cooldownTimeout);
+      this.cooldownTimeout = null;
+    }
+  }
+
+  /**
+   * Sends the current batch to the LLM for processing.
+   */
+  public async sendBatch(): Promise<{ entities: Entity[]; facts: Fact[] } | null> {
+    this.clearTimers();
+
+    if (this.batch.length === 0) return null;
+
+    const currentBatch = [...this.batch];
+    this.batch = [];
+
+    const prompt = this.constructPrompt(currentBatch);
+
+    try {
+      const responseJson = await this.makeHttpRequest(prompt);
+      const result = this.parseLlmResponse(responseJson);
+      const entities = this.transformToEntities(result.entities);
+      const facts = this.transformToFacts(result.facts);
+      this.logger.info(`LLM extracted ${entities.length} entities and ${facts.length} facts.`);
+      return { entities, facts };
+    } catch (err) {
+      this.logger.error('Failed to send or process LLM batch.', err as Error);
+      return null;
+    }
+  }
+
+  /** Constructs the prompt to be sent to the LLM. */
+  private constructPrompt(batch: LlmBatchItem[]): string {
+    const conversation = batch.map(item => item.text).join('\n');
+    return [
+      'Analyze the following conversation and extract key entities and facts.',
+      'Respond with a single JSON object containing "entities" and "facts".',
+      '',
+      'For "entities", provide objects with "type", "value", and "importance".',
+      'Valid types: "person", "location", "organization", "product", "concept".',
+      '',
+      'For "facts", provide triples (subject, predicate, object).',
+      '',
+      'Conversation:',
+      '---',
+      conversation,
+      '---',
+      '',
+      'JSON Response:',
+    ].join('\n');
+  }
+
+  /** Makes an HTTP(S) request to the configured LLM endpoint. */
+  private makeHttpRequest(prompt: string): Promise<string> {
+    return httpPost(this.config.endpoint, {
+      model: this.config.model,
+      prompt,
+      stream: false,
+      format: 'json',
+    });
+  }
+
+  /** Parses and validates the JSON response from the LLM. */
+  private parseLlmResponse(responseJson: string): LlmExtractionResult {
+    try {
+      const outer = JSON.parse(responseJson) as Record<string, unknown>;
+      const inner = typeof outer.response === 'string'
+        ? outer.response : JSON.stringify(outer);
+      const data = JSON.parse(inner) as Record<string, unknown>;
+      if (!data || typeof data !== 'object') {
+        throw new Error('LLM response is not a valid object.');
+      }
+      return {
+        entities: Array.isArray(data.entities) ? data.entities : [],
+        facts: Array.isArray(data.facts) ? data.facts : [],
+      };
+    } catch (err) {
+      this.logger.error(`Failed to parse LLM response: ${responseJson}`, err as Error);
+      throw new Error('Invalid JSON response from LLM.');
+    }
+  }
+
+  /** Transforms raw LLM entity data into the standard Entity format. */
+  private transformToEntities(rawEntities: unknown[]): Entity[] {
+    const entities: Entity[] = [];
+    for (const raw of rawEntities) {
+      const r = raw as Record<string, unknown>;
+      if (typeof r.value !== 'string' || typeof r.type !== 'string') continue;
+      const value = r.value.trim();
+      const type = r.type.toLowerCase();
+      const id = `${type}:${value.toLowerCase().replace(/\s+/g, '-')}`;
+      const imp = typeof r.importance === 'number'
+        ? Math.max(0, Math.min(1, r.importance)) : 0.7;
+      entities.push({
+        id, value, type: type as Entity['type'],
+        mentions: [value], count: 1, importance: imp,
+        lastSeen: new Date().toISOString(), source: ['llm'],
+      });
+    }
+    return entities;
+  }
+
+  /** Transforms raw LLM fact data into partial Fact objects. */
+  private transformToFacts(rawFacts: unknown[]): Fact[] {
+    const facts: Fact[] = [];
+    for (const raw of rawFacts) {
+      const r = raw as Record<string, unknown>;
+      if (typeof r.subject !== 'string' || typeof r.predicate !== 'string' || typeof r.object !== 'string') continue;
+      facts.push({
+        subject: r.subject.trim(),
+        predicate: r.predicate.trim().toLowerCase().replace(/\s+/g, '-'),
+        object: r.object.trim(),
+        source: 'extracted-llm',
+      } as Fact);
+    }
+    return facts;
+  }
+}