@vainplex/openclaw-knowledge-engine 0.1.0

package/openclaw.plugin.json

@@ -0,0 +1,125 @@
+{
+  "id": "@vainplex/openclaw-knowledge-engine",
+  "config": {
+    "enabled": {
+      "type": "boolean",
+      "default": true,
+      "description": "Whether the knowledge engine plugin is enabled."
+    },
+    "workspace": {
+      "type": "string",
+      "default": "~/.clawd/plugins/knowledge-engine",
+      "description": "The directory to store knowledge files (entities.json, facts.json)."
+    },
+    "extraction": {
+      "type": "object",
+      "properties": {
+        "regex": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true,
+              "description": "Enable or disable high-speed regex-based entity extraction."
+            }
+          }
+        },
+        "llm": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true,
+              "description": "Enable or disable high-fidelity LLM-based entity and fact extraction."
+            },
+            "model": {
+              "type": "string",
+              "default": "mistral:7b",
+              "description": "The model name to use for the LLM API call (e.g., Ollama model)."
+            },
+            "endpoint": {
+              "type": "string",
+              "default": "http://localhost:11434/api/generate",
+              "description": "The HTTP endpoint for the LLM generation API."
+            },
+            "batchSize": {
+              "type": "number",
+              "default": 10,
+              "description": "Number of messages to batch together before sending to the LLM."
+            },
+            "cooldownMs": {
+              "type": "number",
+              "default": 30000,
+              "description": "Milliseconds to wait after the last message before sending a batch to the LLM."
+            }
+          }
+        }
+      }
+    },
+    "decay": {
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "Enable or disable the periodic decay of fact relevance."
+        },
+        "intervalHours": {
+          "type": "number",
+          "default": 24,
+          "description": "How often (in hours) to run the decay process."
+        },
+        "rate": {
+          "type": "number",
+          "default": 0.02,
+          "description": "The percentage of relevance to decay in each interval (e.g., 0.02 is 2%)."
+        }
+      }
+    },
+    "embeddings": {
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": false,
+          "description": "Enable or disable syncing of facts to a vector database."
+        },
+        "endpoint": {
+          "type": "string",
+          "default": "http://localhost:8000/api/v1/collections/facts/add",
+          "description": "The HTTP endpoint for the vector database's add API (ChromaDB compatible)."
+        },
+        "collectionName": {
+          "type": "string",
+          "default": "openclaw-facts",
+          "description": "The name of the collection to use in the vector database."
+        },
+        "syncIntervalMinutes": {
+          "type": "number",
+          "default": 15,
+          "description": "How often (in minutes) to sync new facts to the vector database."
+        }
+      }
+    },
+    "storage": {
+      "type": "object",
+      "properties": {
+        "maxEntities": {
+          "type": "number",
+          "default": 5000,
+          "description": "The maximum number of entities to store before pruning."
+        },
+        "maxFacts": {
+          "type": "number",
+          "default": 10000,
+          "description": "The maximum number of facts to store before pruning."
+        },
+        "writeDebounceMs": {
+          "type": "number",
+          "default": 15000,
+          "description": "Milliseconds to wait after a change before writing data to disk."
+        }
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@vainplex/openclaw-knowledge-engine",
+  "version": "0.1.0",
+  "description": "An OpenClaw plugin for real-time and batch knowledge extraction from conversational data.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "tsx --test test/*.test.ts",
+    "lint": "eslint . --ext .ts",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "openclaw",
+    "plugin",
+    "knowledge-extraction",
+    "nlp",
+    "entity-extraction"
+  ],
+  "author": "Vainplex",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/alberthild/openclaw-knowledge-engine.git"
+  },
+  "openclaw": {
+    "id": "@vainplex/openclaw-knowledge-engine"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.24",
+    "eslint": "^8.57.0",
+    "typescript": "^5.3.3",
+    "tsx": "^4.7.1"
+  },
+  "type": "module"
+}

package/src/config.ts

@@ -0,0 +1,180 @@
+// src/config.ts
+
+import * as path from 'node:path';
+import { KnowledgeConfig, Logger } from './types.js';
+
+/**
+ * The default configuration values for the plugin.
+ * These are merged with the user-provided configuration.
+ */
+export const DEFAULT_CONFIG: Omit<KnowledgeConfig, 'workspace'> = {
+  enabled: true,
+  extraction: {
+    regex: { enabled: true },
+    llm: {
+      enabled: true,
+      model: 'mistral:7b',
+      endpoint: 'http://localhost:11434/api/generate',
+      batchSize: 10,
+      cooldownMs: 30000,
+    },
+  },
+  decay: {
+    enabled: true,
+    intervalHours: 24,
+    rate: 0.02,
+  },
+  embeddings: {
+    enabled: false,
+    endpoint: 'http://localhost:8000/api/v1/collections/facts/add',
+    collectionName: 'openclaw-facts',
+    syncIntervalMinutes: 15,
+  },
+  storage: {
+    maxEntities: 5000,
+    maxFacts: 10000,
+    writeDebounceMs: 15000,
+  },
+};
+
+/** Type-safe deep merge: spread source into target for Record values. */
+function deepMerge<T extends Record<string, unknown>>(
+  target: T,
+  source: Record<string, unknown>
+): T {
+  const result = { ...target } as Record<string, unknown>;
+  for (const key of Object.keys(source)) {
+    const srcVal = source[key];
+    const tgtVal = result[key];
+    if (isPlainObject(srcVal) && isPlainObject(tgtVal)) {
+      result[key] = deepMerge(
+        tgtVal as Record<string, unknown>,
+        srcVal as Record<string, unknown>
+      );
+    } else if (srcVal !== undefined) {
+      result[key] = srcVal;
+    }
+  }
+  return result as T;
+}
+
+function isPlainObject(val: unknown): val is Record<string, unknown> {
+  return typeof val === 'object' && val !== null && !Array.isArray(val);
+}
+
+/** Merge user config over defaults and resolve workspace. */
+function mergeConfigDefaults(
+  userConfig: Record<string, unknown>,
+  openClawWorkspace: string
+): KnowledgeConfig {
+  const merged = deepMerge(
+    DEFAULT_CONFIG as unknown as Record<string, unknown>,
+    userConfig
+  );
+  const ws = typeof userConfig.workspace === 'string' && userConfig.workspace
+    ? userConfig.workspace
+    : path.join(openClawWorkspace, 'knowledge-engine');
+  return { ...merged, workspace: ws } as KnowledgeConfig;
+}
+
+/** Replace a leading tilde with the user's home directory. */
+function resolveTilde(ws: string, logger: Logger, fallback: string): string {
+  if (!ws.startsWith('~')) return ws;
+  const homeDir = process.env.HOME || process.env.USERPROFILE;
+  if (homeDir) return path.join(homeDir, ws.slice(1));
+  logger.warn('Could not resolve home directory for workspace path.');
+  return fallback;
+}
+
+/**
+ * Resolves and validates the plugin's configuration.
+ *
+ * @param userConfig The user-provided configuration from OpenClaw's pluginConfig.
+ * @param logger A logger instance for logging warnings or errors.
+ * @param openClawWorkspace The root workspace directory provided by OpenClaw.
+ * @returns A fully resolved KnowledgeConfig, or null if validation fails.
+ */
+export function resolveConfig(
+  userConfig: Record<string, unknown>,
+  logger: Logger,
+  openClawWorkspace: string
+): KnowledgeConfig | null {
+  const config = mergeConfigDefaults(userConfig, openClawWorkspace);
+  const fallbackWs = path.join(openClawWorkspace, 'knowledge-engine');
+  config.workspace = resolveTilde(config.workspace, logger, fallbackWs);
+
+  const errors = validateConfig(config);
+  if (errors.length > 0) {
+    errors.forEach(e => logger.error(`Invalid configuration: ${e}`));
+    return null;
+  }
+
+  logger.info('Knowledge Engine configuration resolved successfully.');
+  return config;
+}
+
+// ── Validation ──────────────────────────────────────────────
+
+function validateConfig(config: KnowledgeConfig): string[] {
+  return [
+    ...validateRoot(config),
+    ...validateExtraction(config.extraction),
+    ...validateDecay(config.decay),
+    ...validateEmbeddings(config.embeddings),
+    ...validateStorage(config.storage),
+  ];
+}
+
+function validateRoot(c: KnowledgeConfig): string[] {
+  const errs: string[] = [];
+  if (typeof c.enabled !== 'boolean') errs.push('"enabled" must be a boolean.');
+  if (typeof c.workspace !== 'string' || c.workspace.trim() === '') {
+    errs.push('"workspace" must be a non-empty string.');
+  }
+  return errs;
+}
+
+function validateExtraction(ext: KnowledgeConfig['extraction']): string[] {
+  const errs: string[] = [];
+  if (ext.llm.enabled) {
+    if (!isValidHttpUrl(ext.llm.endpoint)) {
+      errs.push('"extraction.llm.endpoint" must be a valid HTTP/S URL.');
+    }
+    if ((ext.llm.batchSize ?? 0) < 1) {
+      errs.push('"extraction.llm.batchSize" must be at least 1.');
+    }
+  }
+  return errs;
+}
+
+function validateDecay(d: KnowledgeConfig['decay']): string[] {
+  const errs: string[] = [];
+  if (d.rate < 0 || d.rate > 1) errs.push('"decay.rate" must be between 0 and 1.');
+  if ((d.intervalHours ?? 0) <= 0) errs.push('"decay.intervalHours" must be greater than 0.');
+  return errs;
+}
+
+function validateEmbeddings(e: KnowledgeConfig['embeddings']): string[] {
+  const errs: string[] = [];
+  if (e.enabled && !isValidHttpUrl(e.endpoint)) {
+    errs.push('"embeddings.endpoint" must be a valid HTTP/S URL.');
+  }
+  return errs;
+}
+
+function validateStorage(s: KnowledgeConfig['storage']): string[] {
+  const errs: string[] = [];
+  if ((s.writeDebounceMs ?? 0) < 0) {
+    errs.push('"storage.writeDebounceMs" must be a non-negative number.');
+  }
+  return errs;
+}
+
+function isValidHttpUrl(str: string): boolean {
+  try {
+    const url = new URL(str);
+    return url.protocol === 'http:' || url.protocol === 'https:';
+  } catch {
+    return false;
+  }
+}

package/src/embeddings.ts

@@ -0,0 +1,82 @@
+// src/embeddings.ts
+
+import { Fact, KnowledgeConfig, Logger } from './types.js';
+import { httpPost } from './http-client.js';
+
+/** ChromaDB v2 API payload format. */
+interface ChromaPayload {
+  ids: string[];
+  documents: string[];
+  metadatas: Record<string, string>[];
+}
+
+/**
+ * Manages optional integration with a ChromaDB-compatible vector database.
+ */
+export class Embeddings {
+  private readonly config: KnowledgeConfig['embeddings'];
+  private readonly logger: Logger;
+
+  constructor(config: KnowledgeConfig['embeddings'], logger: Logger) {
+    this.config = config;
+    this.logger = logger;
+  }
+
+  /** Checks if the embeddings service is enabled. */
+  public isEnabled(): boolean {
+    return this.config.enabled;
+  }
+
+  /**
+   * Syncs a batch of facts to the vector database.
+   * @returns The number of successfully synced facts.
+   */
+  public async sync(facts: Fact[]): Promise<number> {
+    if (!this.isEnabled() || facts.length === 0) return 0;
+
+    this.logger.info(`Starting embedding sync for ${facts.length} facts.`);
+    const payload = this.constructChromaPayload(facts);
+    const url = this.buildEndpointUrl();
+
+    try {
+      await httpPost(url, payload);
+      this.logger.info(`Successfully synced ${facts.length} facts to ChromaDB.`);
+      return facts.length;
+    } catch (err) {
+      this.logger.error('Failed to sync embeddings to ChromaDB.', err as Error);
+      return 0;
+    }
+  }
+
+  /** Builds the full endpoint URL with collection name substituted. */
+  private buildEndpointUrl(): string {
+    return this.config.endpoint
+      .replace('{name}', this.config.collectionName)
+      .replace('//', '//')  // preserve protocol double-slash
+      .replace(/([^:])\/\//g, '$1/');  // collapse any other double-slashes
+  }
+
+  /**
+   * Constructs the payload for ChromaDB v2 API.
+   * Metadata values are all strings (v2 requirement).
+   */
+  private constructChromaPayload(facts: Fact[]): ChromaPayload {
+    const payload: ChromaPayload = { ids: [], documents: [], metadatas: [] };
+
+    for (const fact of facts) {
+      payload.ids.push(fact.id);
+      payload.documents.push(
+        `${fact.subject} ${fact.predicate.replace(/-/g, ' ')} ${fact.object}.`
+      );
+      payload.metadatas.push({
+        subject: fact.subject,
+        predicate: fact.predicate,
+        object: fact.object,
+        source: fact.source,
+        createdAt: fact.createdAt,
+      });
+    }
+
+    return payload;
+  }
+}

package/src/entity-extractor.ts

@@ -0,0 +1,137 @@
+// src/entity-extractor.ts
+
+import { Entity, Logger } from './types.js';
+import { REGEX_PATTERNS } from './patterns.js';
+
+// A map to associate regex pattern names with entity types.
+const PATTERN_TYPE_MAP: Record<string, Entity['type']> = {
+  email: 'email',
+  url: 'url',
+  iso_date: 'date',
+  common_date: 'date',
+  german_date: 'date',
+  english_date: 'date',
+  proper_noun: 'unknown',
+  product_name: 'product',
+  organization_suffix: 'organization',
+};
+
+/**
+ * Extracts entities from text using predefined regular expressions.
+ */
+export class EntityExtractor {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
+  /**
+   * Extracts entities from a given text based on the regex patterns.
+   * @param text The input text to process.
+   * @returns An array of found entities.
+   */
+  public extract(text: string): Entity[] {
+    const foundEntities: Map<string, Entity> = new Map();
+
+    for (const key in REGEX_PATTERNS) {
+      // Each access returns a fresh RegExp (via Proxy), avoiding /g state-bleed.
+      const regex = REGEX_PATTERNS[key];
+      if (!regex.global) {
+        this.logger.warn(`Regex for "${key}" is not global. Skipping.`);
+        continue;
+      }
+      const entityType = PATTERN_TYPE_MAP[key] || 'unknown';
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        const value = match[0].trim();
+        if (!value) continue;
+        this.processMatch(key, value, entityType, foundEntities);
+      }
+    }
+
+    return Array.from(foundEntities.values());
+  }
+
+  /**
+   * Processes a single regex match and upserts it into the entity map.
+   */
+  private processMatch(
+    _key: string,
+    value: string,
+    entityType: Entity['type'],
+    entities: Map<string, Entity>
+  ): void {
+    const canonicalValue = this.canonicalize(value, entityType);
+    const id = `${entityType}:${canonicalValue.toLowerCase().replace(/\s+/g, '-')}`;
+
+    if (entities.has(id)) {
+      const existing = entities.get(id)!;
+      if (!existing.mentions.includes(value)) existing.mentions.push(value);
+      existing.count++;
+      if (!existing.source.includes('regex')) existing.source.push('regex');
+    } else {
+      entities.set(id, {
+        id,
+        type: entityType,
+        value: canonicalValue,
+        mentions: [value],
+        count: 1,
+        importance: this.calculateInitialImportance(entityType, value),
+        lastSeen: new Date().toISOString(),
+        source: ['regex'],
+      });
+    }
+  }
+
+  /**
+   * Cleans and standardizes an entity value based on its type.
+   */
+  private canonicalize(value: string, type: Entity['type']): string {
+    if (type === 'organization') {
+      const suffixes = /,?\s?(?:Inc\.|LLC|Corp\.|GmbH|AG|Ltd\.)$/i;
+      return value.replace(suffixes, '').trim();
+    }
+    return value.replace(/[.,!?;:]$/, '').trim();
+  }
+
+  /**
+   * Calculates an initial importance score for an entity.
+   */
+  private calculateInitialImportance(type: Entity['type'], value: string): number {
+    switch (type) {
+      case 'organization': return 0.8;
+      case 'person':       return 0.7;
+      case 'product':      return 0.6;
+      case 'location':     return 0.5;
+      case 'date':
+      case 'email':
+      case 'url':          return 0.4;
+      default:             return value.split(/\s|-/).length > 1 ? 0.5 : 0.3;
+    }
+  }
+
+  /**
+   * Merges two lists of entities by ID.
+   */
+  public static mergeEntities(listA: Entity[], listB: Entity[]): Entity[] {
+    const merged: Map<string, Entity> = new Map();
+    for (const e of listA) merged.set(e.id, { ...e });
+
+    for (const entity of listB) {
+      if (merged.has(entity.id)) {
+        const ex = merged.get(entity.id)!;
+        ex.count += entity.count;
+        ex.mentions = [...new Set([...ex.mentions, ...entity.mentions])];
+        ex.source = [...new Set([...ex.source, ...entity.source])];
+        ex.lastSeen = new Date() > new Date(ex.lastSeen)
+          ? new Date().toISOString() : ex.lastSeen;
+        ex.importance = Math.max(ex.importance, entity.importance);
+      } else {
+        merged.set(entity.id, { ...entity });
+      }
+    }
+
+    return Array.from(merged.values());
+  }
+}