@vainplex/openclaw-knowledge-engine 0.1.1 → 0.1.3

package/src/entity-extractor.ts

@@ -1,137 +0,0 @@
-// src/entity-extractor.ts
-
-import { Entity, Logger } from './types.js';
-import { REGEX_PATTERNS } from './patterns.js';
-
-// A map to associate regex pattern names with entity types.
-const PATTERN_TYPE_MAP: Record<string, Entity['type']> = {
-  email: 'email',
-  url: 'url',
-  iso_date: 'date',
-  common_date: 'date',
-  german_date: 'date',
-  english_date: 'date',
-  proper_noun: 'unknown',
-  product_name: 'product',
-  organization_suffix: 'organization',
-};
-
-/**
- * Extracts entities from text using predefined regular expressions.
- */
-export class EntityExtractor {
-  private readonly logger: Logger;
-
-  constructor(logger: Logger) {
-    this.logger = logger;
-  }
-
-  /**
-   * Extracts entities from a given text based on the regex patterns.
-   * @param text The input text to process.
-   * @returns An array of found entities.
-   */
-  public extract(text: string): Entity[] {
-    const foundEntities: Map<string, Entity> = new Map();
-
-    for (const key in REGEX_PATTERNS) {
-      // Each access returns a fresh RegExp (via Proxy), avoiding /g state-bleed.
-      const regex = REGEX_PATTERNS[key];
-      if (!regex.global) {
-        this.logger.warn(`Regex for "${key}" is not global. Skipping.`);
-        continue;
-      }
-      const entityType = PATTERN_TYPE_MAP[key] || 'unknown';
-      let match;
-      while ((match = regex.exec(text)) !== null) {
-        const value = match[0].trim();
-        if (!value) continue;
-        this.processMatch(key, value, entityType, foundEntities);
-      }
-    }
-
-    return Array.from(foundEntities.values());
-  }
-
-  /**
-   * Processes a single regex match and upserts it into the entity map.
-   */
-  private processMatch(
-    _key: string,
-    value: string,
-    entityType: Entity['type'],
-    entities: Map<string, Entity>
-  ): void {
-    const canonicalValue = this.canonicalize(value, entityType);
-    const id = `${entityType}:${canonicalValue.toLowerCase().replace(/\s+/g, '-')}`;
-
-    if (entities.has(id)) {
-      const existing = entities.get(id)!;
-      if (!existing.mentions.includes(value)) existing.mentions.push(value);
-      existing.count++;
-      if (!existing.source.includes('regex')) existing.source.push('regex');
-    } else {
-      entities.set(id, {
-        id,
-        type: entityType,
-        value: canonicalValue,
-        mentions: [value],
-        count: 1,
-        importance: this.calculateInitialImportance(entityType, value),
-        lastSeen: new Date().toISOString(),
-        source: ['regex'],
-      });
-    }
-  }
-
-  /**
-   * Cleans and standardizes an entity value based on its type.
-   */
-  private canonicalize(value: string, type: Entity['type']): string {
-    if (type === 'organization') {
-      const suffixes = /,?\s?(?:Inc\.|LLC|Corp\.|GmbH|AG|Ltd\.)$/i;
-      return value.replace(suffixes, '').trim();
-    }
-    return value.replace(/[.,!?;:]$/, '').trim();
-  }
-
-  /**
-   * Calculates an initial importance score for an entity.
-   */
-  private calculateInitialImportance(type: Entity['type'], value: string): number {
-    switch (type) {
-      case 'organization': return 0.8;
-      case 'person':       return 0.7;
-      case 'product':      return 0.6;
-      case 'location':     return 0.5;
-      case 'date':
-      case 'email':
-      case 'url':          return 0.4;
-      default:             return value.split(/\s|-/).length > 1 ? 0.5 : 0.3;
-    }
-  }
-
-  /**
-   * Merges two lists of entities by ID.
-   */
-  public static mergeEntities(listA: Entity[], listB: Entity[]): Entity[] {
-    const merged: Map<string, Entity> = new Map();
-    for (const e of listA) merged.set(e.id, { ...e });
-
-    for (const entity of listB) {
-      if (merged.has(entity.id)) {
-        const ex = merged.get(entity.id)!;
-        ex.count += entity.count;
-        ex.mentions = [...new Set([...ex.mentions, ...entity.mentions])];
-        ex.source = [...new Set([...ex.source, ...entity.source])];
-        ex.lastSeen = new Date() > new Date(ex.lastSeen)
-          ? new Date().toISOString() : ex.lastSeen;
-        ex.importance = Math.max(ex.importance, entity.importance);
-      } else {
-        merged.set(entity.id, { ...entity });
-      }
-    }
-
-    return Array.from(merged.values());
-  }
-}

package/src/fact-store.ts

@@ -1,260 +0,0 @@
-// src/fact-store.ts
-
-import { randomUUID } from 'node:crypto';
-import { AtomicStorage } from './storage.js';
-import type { Fact, FactsData, KnowledgeConfig, Logger } from './types.js';
-
-/**
- * Manages an in-memory and on-disk store of structured facts.
- * Provides methods for loading, querying, modifying, and persisting facts.
- */
-export class FactStore {
-  private readonly storage: AtomicStorage;
-  private readonly config: KnowledgeConfig['storage'];
-  private readonly logger: Logger;
-  private facts: Map<string, Fact> = new Map();
-  private isLoaded: boolean = false;
-
-  public readonly commit: () => Promise<void>;
-
-  constructor(
-    workspace: string,
-    config: KnowledgeConfig['storage'],
-    logger: Logger
-  ) {
-    this.storage = new AtomicStorage(workspace, logger);
-    this.config = config;
-    this.logger = logger;
-
-    // Create a debounced version of the persist method.
-    this.commit = AtomicStorage.debounce(
-      this.persist.bind(this),
-      this.config.writeDebounceMs
-    );
-  }
-
-  /**
-   * Immediately flushes any pending debounced writes.
-   * Useful in tests and before shutdown to ensure data is persisted.
-   */
-  public async flush(): Promise<void> {
-    if (this.isLoaded) {
-      await this.persist();
-    }
-  }
-
-  /**
-   * Loads facts from the `facts.json` file into the in-memory store.
-   * If the file doesn't exist, it initializes an empty store.
-   */
-  public async load(): Promise<void> {
-    if (this.isLoaded) {
-      this.logger.debug('Fact store is already loaded.');
-      return;
-    }
-
-    await this.storage.init();
-    const data = await this.storage.readJson<FactsData>('facts.json');
-    if (data && Array.isArray(data.facts)) {
-      this.facts = new Map(data.facts.map(fact => [fact.id, fact]));
-      this.logger.info(`Loaded ${this.facts.size} facts from storage.`);
-    } else {
-      this.logger.info('No existing fact store found. Initializing a new one.');
-      this.facts = new Map();
-    }
-    this.isLoaded = true;
-  }
-
-  /**
-   * Adds a new fact to the store or updates an existing one based on content.
-   * @param newFactData The data for the new fact, excluding metadata fields.
-   * @returns The newly created or found Fact object.
-   */
-  public addFact(
-    newFactData: Omit<Fact, 'id' | 'createdAt' | 'lastAccessed' | 'relevance'>
-  ): Fact {
-    if (!this.isLoaded) {
-      throw new Error('FactStore has not been loaded yet. Call load() first.');
-    }
-    const now = new Date().toISOString();
-    
-    // Check if a similar fact already exists to avoid duplicates
-    for (const existingFact of this.facts.values()) {
-        if (
-            existingFact.subject === newFactData.subject &&
-            existingFact.predicate === newFactData.predicate &&
-            existingFact.object === newFactData.object
-        ) {
-            // Fact already exists, let's just boost its relevance and update timestamp
-            existingFact.relevance = this.boostRelevance(existingFact.relevance);
-            existingFact.lastAccessed = now;
-            this.commit();
-            return existingFact;
-        }
-    }
-
-    const newFact: Fact = {
-      ...newFactData,
-      id: randomUUID(),
-      createdAt: now,
-      lastAccessed: now,
-      relevance: 1.0, // New facts start with maximum relevance
-    };
-
-    this.facts.set(newFact.id, newFact);
-    this.prune(); // Check if we need to prune old facts
-    this.commit();
-    return newFact;
-  }
-
-  /**
-   * Retrieves a fact by its unique ID.
-   * @param id The UUID of the fact.
-   * @returns The Fact object, or undefined if not found.
-   */
-  public getFact(id: string): Fact | undefined {
-    const fact = this.facts.get(id);
-    if (fact) {
-        fact.lastAccessed = new Date().toISOString();
-        fact.relevance = this.boostRelevance(fact.relevance);
-        this.commit();
-    }
-    return fact;
-  }
-
-  /**
-   * Queries the fact store based on subject, predicate, or object.
-   * @param query An object with optional subject, predicate, and/or object to match.
-   * @returns An array of matching facts, sorted by relevance.
-   */
-  public query(query: { subject?: string; predicate?: string; object?: string }): Fact[] {
-    const results: Fact[] = [];
-    for (const fact of this.facts.values()) {
-      const subjectMatch = !query.subject || fact.subject === query.subject;
-      const predicateMatch = !query.predicate || fact.predicate === query.predicate;
-      const objectMatch = !query.object || fact.object === query.object;
-      
-      if (subjectMatch && predicateMatch && objectMatch) {
-        results.push(fact);
-      }
-    }
-    // Sort by relevance, descending
-    return results.sort((a, b) => b.relevance - a.relevance);
-  }
-
-  /**
-   * Applies a decay factor to the relevance score of all facts.
-   * @param rate The decay rate (e.g., 0.05 for 5%).
-   * @returns An object with the count of decayed facts.
-   */
-  public decayFacts(rate: number): { decayedCount: number } {
-    let decayedCount = 0;
-    const minRelevance = 0.1; // Floor to prevent facts from disappearing completely
-
-    for (const fact of this.facts.values()) {
-      const newRelevance = fact.relevance * (1 - rate);
-      if (newRelevance !== fact.relevance) {
-          fact.relevance = Math.max(minRelevance, newRelevance);
-          decayedCount++;
-      }
-    }
-    
-    if (decayedCount > 0) {
-      this.logger.info(`Applied decay rate of ${rate * 100}% to ${decayedCount} facts.`);
-      this.commit();
-    }
-    return { decayedCount };
-  }
-
-  /**
-   * Persists the current in-memory fact store to `facts.json`.
-   */
-  private async persist(): Promise<void> {
-    if (!this.isLoaded) {
-      this.logger.warn('Attempted to persist fact store before it was loaded. Aborting.');
-      return;
-    }
-    
-    const data: FactsData = {
-      updated: new Date().toISOString(),
-      facts: Array.from(this.facts.values()),
-    };
-    
-    try {
-      await this.storage.writeJson('facts.json', data);
-      this.logger.debug(`Successfully persisted ${data.facts.length} facts.`);
-    } catch (err) {
-      this.logger.error('Failed to persist fact store.', err as Error);
-    }
-  }
-
-  /**
-   * Removes the least relevant facts if the store exceeds its configured max size.
-   */
-  private prune(): void {
-    const factCount = this.facts.size;
-    if (factCount <= this.config.maxFacts) {
-      return;
-    }
-
-    const factsToPrune = factCount - this.config.maxFacts;
-    if (factsToPrune <= 0) return;
-
-    // Get all facts, sort by relevance (ascending) and then by lastAccessed (ascending)
-    const sortedFacts = Array.from(this.facts.values()).sort((a, b) => {
-        if (a.relevance !== b.relevance) {
-            return a.relevance - b.relevance;
-        }
-        return new Date(a.lastAccessed).getTime() - new Date(b.lastAccessed).getTime();
-    });
-
-    for (let i = 0; i < factsToPrune; i++) {
-        this.facts.delete(sortedFacts[i].id);
-    }
-    
-    this.logger.info(`Pruned ${factsToPrune} least relevant facts to maintain store size.`);
-  }
-
-  /**
-   * Boosts the relevance of a fact upon access.
-   * @param currentRelevance The current relevance score.
-   * @returns The new, boosted relevance score.
-   */
-  private boostRelevance(currentRelevance: number): number {
-    // Push the relevance 50% closer to 1.0
-    const boost = (1.0 - currentRelevance) * 0.5;
-    return Math.min(1.0, currentRelevance + boost);
-  }
-
-  /**
-   * Returns a list of all facts that have not been embedded yet.
-   */
-  public getUnembeddedFacts(): Fact[] {
-    const results: Fact[] = [];
-    for (const fact of this.facts.values()) {
-        if (!fact.embedded) {
-            results.push(fact);
-        }
-    }
-    return results;
-  }
-
-  /**
-   * Marks a list of facts as having been embedded.
-   * @param factIds An array of fact IDs to update.
-   */
-  public markFactsAsEmbedded(factIds: string[]): void {
-    const now = new Date().toISOString();
-    let updatedCount = 0;
-    for (const id of factIds) {
-        const fact = this.facts.get(id);
-        if (fact) {
-            fact.embedded = now;
-            updatedCount++;
-        }
-    }
-    if (updatedCount > 0) {
-        this.commit();
-    }
-  }
-}

package/src/hooks.ts

@@ -1,125 +0,0 @@
-// src/hooks.ts
-
-import {
-  OpenClawPluginApi,
-  HookEvent,
-  KnowledgeConfig,
-  Logger,
-} from './types.js';
-
-import { EntityExtractor } from './entity-extractor.js';
-import { FactStore } from './fact-store.js';
-import { LlmEnhancer } from './llm-enhancer.js';
-import { Maintenance } from './maintenance.js';
-import { Embeddings } from './embeddings.js';
-
-/**
- * Manages the registration and orchestration of all plugin hooks.
- */
-export class HookManager {
-  private readonly api: OpenClawPluginApi;
-  private readonly config: KnowledgeConfig;
-  private readonly logger: Logger;
-
-  private entityExtractor: EntityExtractor;
-  private factStore: FactStore;
-  private llmEnhancer?: LlmEnhancer;
-  private maintenance?: Maintenance;
-
-  constructor(api: OpenClawPluginApi, config: KnowledgeConfig) {
-    this.api = api;
-    this.config = config;
-    this.logger = api.logger;
-
-    this.entityExtractor = new EntityExtractor(this.logger);
-    this.factStore = new FactStore(
-      this.config.workspace,
-      this.config.storage,
-      this.logger
-    );
-
-    if (this.config.extraction.llm.enabled) {
-      this.llmEnhancer = new LlmEnhancer(this.config.extraction.llm, this.logger);
-    }
-  }
-
-  /** Registers all the necessary hooks with the OpenClaw host. */
-  public registerHooks(): void {
-    if (!this.config.enabled) {
-      this.logger.info('Knowledge Engine is disabled. No hooks registered.');
-      return;
-    }
-
-    this.api.on('session_start', this.onSessionStart.bind(this), { priority: 200 });
-    this.api.on('message_received', this.onMessage.bind(this), { priority: 100 });
-    this.api.on('message_sent', this.onMessage.bind(this), { priority: 100 });
-    this.api.on('gateway_stop', this.onShutdown.bind(this), { priority: 900 });
-
-    this.logger.info('Registered all Knowledge Engine hooks.');
-  }
-
-  /** Handler for the `session_start` hook. */
-  private async onSessionStart(): Promise<void> {
-    this.logger.info('Knowledge Engine starting up...');
-    await this.factStore.load();
-
-    const embeddings = this.config.embeddings.enabled
-      ? new Embeddings(this.config.embeddings, this.logger)
-      : undefined;
-
-    this.maintenance = new Maintenance(
-      this.config, this.logger, this.factStore, embeddings
-    );
-    this.maintenance.start();
-  }
-
-  /** Handler for `gateway_stop` — cleans up timers and flushes state. */
-  private async onShutdown(): Promise<void> {
-    this.logger.info('Knowledge Engine shutting down...');
-    this.maintenance?.stop();
-    this.llmEnhancer?.clearTimers();
-    this.logger.info('Knowledge Engine shutdown complete.');
-  }
-
-  /** Handler for `message_received` and `message_sent` hooks. */
-  private async onMessage(event: HookEvent): Promise<void> {
-    const text = event.content || event.message || event.text;
-    if (typeof text !== 'string' || text.trim() === '') return;
-
-    this.logger.debug(`Processing message: "${text.substring(0, 50)}..."`);
-
-    if (this.config.extraction.regex.enabled) {
-      const entities = this.entityExtractor.extract(text);
-      if (entities.length > 0) {
-        this.logger.info(`Extracted ${entities.length} entities via regex.`);
-      }
-    }
-
-    if (this.llmEnhancer) {
-      const messageId = `msg-${Date.now()}`;
-      this.llmEnhancer.addToBatch({ id: messageId, text });
-      this.processLlmBatchWhenReady().catch(err =>
-        this.logger.error('LLM batch processing failed', err as Error));
-    }
-  }
-
-  /** Fire-and-forget: processes LLM batch results when available. */
-  private async processLlmBatchWhenReady(): Promise<void> {
-    if (!this.llmEnhancer) return;
-
-    const result = await this.llmEnhancer.sendBatch();
-    if (!result) return;
-
-    if (result.facts.length > 0) {
-      this.logger.info(`Adding ${result.facts.length} LLM facts.`);
-      for (const f of result.facts) {
-        this.factStore.addFact({
-          subject: f.subject,
-          predicate: f.predicate,
-          object: f.object,
-          source: 'extracted-llm',
-        });
-      }
-    }
-  }
-}

package/src/http-client.ts

@@ -1,74 +0,0 @@
-// src/http-client.ts
-
-import * as http from 'node:http';
-import * as https from 'node:https';
-
-interface HttpPostOptions {
-  hostname: string;
-  port: string;
-  path: string;
-  method: 'POST';
-  headers: Record<string, string | number>;
-}
-
-/**
- * Selects the correct HTTP/HTTPS module based on the URL protocol.
- */
-function selectTransport(protocol: string): typeof http | typeof https {
-  return protocol === 'https:' ? https : http;
-}
-
-/**
- * Builds request options from a URL and payload.
- */
-function buildRequestOptions(url: URL, payload: string): HttpPostOptions {
-  return {
-    hostname: url.hostname,
-    port: url.port,
-    path: url.pathname + url.search,
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'Content-Length': Buffer.byteLength(payload),
-    },
-  };
-}
-
-/**
- * Makes an HTTP or HTTPS POST request, auto-selecting the
- * transport based on the URL's protocol.
- *
- * @param url  The full URL string to POST to.
- * @param body The payload object to JSON-serialize and send.
- * @returns A promise resolving with the response body string.
- */
-export function httpPost(url: string, body: unknown): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const parsed = new URL(url);
-    const payload = JSON.stringify(body);
-    const options = buildRequestOptions(parsed, payload);
-    const transport = selectTransport(parsed.protocol);
-
-    const req = transport.request(options, (res) => {
-      let data = '';
-      res.setEncoding('utf8');
-      res.on('data', (chunk: string) => { data += chunk; });
-      res.on('end', () => {
-        if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
-          resolve(data);
-        } else {
-          reject(new Error(
-            `HTTP request failed with status ${res.statusCode}: ${data}`
-          ));
-        }
-      });
-    });
-
-    req.on('error', (e) => {
-      reject(new Error(`HTTP request error: ${e.message}`));
-    });
-
-    req.write(payload);
-    req.end();
-  });
-}