@vainplex/openclaw-knowledge-engine 0.1.0

package/src/maintenance.ts

@@ -0,0 +1,102 @@
+// src/maintenance.ts
+
+import { Embeddings } from './embeddings.js';
+import { FactStore } from './fact-store.js';
+import { KnowledgeConfig, Logger } from './types.js';
+
+/**
+ * Manages background maintenance tasks for the knowledge engine,
+ * such as decaying fact relevance and syncing embeddings.
+ */
+export class Maintenance {
+  private readonly config: KnowledgeConfig;
+  private readonly logger: Logger;
+  private readonly factStore: FactStore;
+  private readonly embeddings?: Embeddings;
+
+  private decayTimer: NodeJS.Timeout | null = null;
+  private embeddingsTimer: NodeJS.Timeout | null = null;
+
+  constructor(
+    config: KnowledgeConfig,
+    logger: Logger,
+    factStore: FactStore,
+    embeddings?: Embeddings
+  ) {
+    this.config = config;
+    this.logger = logger;
+    this.factStore = factStore;
+    this.embeddings = embeddings;
+  }
+
+  /** Starts all configured maintenance timers. */
+  public start(): void {
+    this.logger.info('Starting maintenance service...');
+    this.stop();
+    this.startDecayTimer();
+    this.startEmbeddingsTimer();
+  }
+
+  /** Stops all running maintenance timers. */
+  public stop(): void {
+    if (this.decayTimer) {
+      clearInterval(this.decayTimer);
+      this.decayTimer = null;
+    }
+    if (this.embeddingsTimer) {
+      clearInterval(this.embeddingsTimer);
+      this.embeddingsTimer = null;
+    }
+    this.logger.info('Stopped maintenance service.');
+  }
+
+  private startDecayTimer(): void {
+    if (!this.config.decay.enabled) return;
+    const ms = this.config.decay.intervalHours * 60 * 60 * 1000;
+    this.decayTimer = setInterval(() => this.runDecay(), ms);
+    this.decayTimer.unref();
+    this.logger.info(`Scheduled fact decay every ${this.config.decay.intervalHours} hours.`);
+  }
+
+  private startEmbeddingsTimer(): void {
+    if (!this.embeddings?.isEnabled()) return;
+    const ms = this.config.embeddings.syncIntervalMinutes * 60 * 1000;
+    this.embeddingsTimer = setInterval(() => this.runEmbeddingsSync(), ms);
+    this.embeddingsTimer.unref();
+    this.logger.info(`Scheduled embeddings sync every ${this.config.embeddings.syncIntervalMinutes} min.`);
+  }
+
+  /** Executes the fact decay process. */
+  public runDecay(): void {
+    this.logger.info('Running scheduled fact decay...');
+    try {
+      const { decayedCount } = this.factStore.decayFacts(this.config.decay.rate);
+      this.logger.info(`Fact decay complete. Decayed ${decayedCount} facts.`);
+    } catch (err) {
+      this.logger.error('Error during fact decay.', err as Error);
+    }
+  }
+
+  /** Executes the embeddings synchronization process. */
+  public async runEmbeddingsSync(): Promise<void> {
+    if (!this.embeddings?.isEnabled()) return;
+
+    this.logger.info('Running scheduled embeddings sync...');
+    try {
+      const unembedded = this.factStore.getUnembeddedFacts();
+      if (unembedded.length === 0) {
+        this.logger.info('No new facts to sync for embeddings.');
+        return;
+      }
+
+      const synced = await this.embeddings.sync(unembedded);
+      if (synced > 0) {
+        const ids = unembedded.slice(0, synced).map(f => f.id);
+        this.factStore.markFactsAsEmbedded(ids);
+        this.logger.info(`Embeddings sync complete. Synced ${synced} facts.`);
+      }
+    } catch (err) {
+      this.logger.error('Error during embeddings sync.', err as Error);
+    }
+  }
+}

package/src/patterns.ts

@@ -0,0 +1,90 @@
+// src/patterns.ts
+
+/**
+ * Common words that look like proper nouns (start of sentence) but are not.
+ */
+const EXCLUDED_WORDS = [
+  'A', 'An', 'The', 'Hello', 'My', 'This', 'Contact', 'He', 'She',
+  'It', 'We', 'They', 'I', 'You', 'His', 'Her', 'Our', 'Your',
+  'Their', 'Its', 'That', 'These', 'Those', 'What', 'Which', 'Who',
+  'How', 'When', 'Where', 'Why', 'But', 'And', 'Or', 'So', 'Not',
+  'No', 'Yes', 'Also', 'Just', 'For', 'From', 'With', 'About',
+  'After', 'Before', 'Between', 'During', 'Into', 'Through',
+  'Event', 'Talk', 'Project', 'Multiple', 'German',
+  'Am', 'Are', 'Is', 'Was', 'Were', 'Has', 'Have',
+  'Had', 'Do', 'Does', 'Did', 'Will', 'Would', 'Could', 'Should',
+  'May', 'Might', 'Must', 'Can', 'Shall', 'If', 'Then',
+];
+
+const EXCL = EXCLUDED_WORDS.map(w => `${w}\\b`).join('|');
+
+/** Capitalized word: handles O'Malley, McDonald's, acronyms like USS */
+const CAP = `(?:[A-Z][a-z']*(?:[A-Z][a-z']+)*|[A-Z]{2,})`;
+
+const DE_MONTHS =
+  'Januar|Februar|März|Mar|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember';
+
+const EN_MONTHS =
+  'January|February|March|April|May|June|July|August|September|October|November|December';
+
+/** Proper noun: one or more cap-words with exclusion list applied per word. */
+function properNounFactory(): RegExp {
+  return new RegExp(
+    `\\b(?!${EXCL})${CAP}(?:(?:-|\\s)(?!${EXCL})${CAP})*\\b`, 'g'
+  );
+}
+
+/** Product name: three branches for multi-word+Roman, word+version, camelCase. */
+function productNameFactory(): RegExp {
+  return new RegExp(
+    `\\b(?:(?!${EXCL})[A-Z][a-zA-Z0-9]{2,}(?:\\s[a-zA-Z]+)*\\s[IVXLCDM]+` +
+    `|[a-zA-Z][a-zA-Z0-9-]{2,}[\\s-]v?\\d+(?:\\.\\d+)?` +
+    `|[a-zA-Z][a-zA-Z0-9]+[IVXLCDM]+)\\b`, 'g'
+  );
+}
+
+/** Creates a fresh RegExp factory for each pattern key. */
+function buildPatterns(): Record<string, () => RegExp> {
+  return {
+    email:       () => /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b/g,
+    url:         () => /\bhttps?:\/\/[^\s/$.?#].[^\s]*\b/g,
+    iso_date:    () => /\b\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d+)?Z?)?\b/g,
+    common_date: () => /\b(?:\d{1,2}\/\d{1,2}\/\d{2,4})|(?:\d{1,2}\.\d{1,2}\.\d{2,4})\b/g,
+    german_date: () => new RegExp(`\\b\\d{1,2}\\.\\s(?:${DE_MONTHS})\\s+\\d{4}\\b`, 'gi'),
+    english_date: () => new RegExp(`\\b(?:${EN_MONTHS})\\s+\\d{1,2}(?:st|nd|rd|th)?,\\s+\\d{4}\\b`, 'gi'),
+    proper_noun: properNounFactory,
+    product_name: productNameFactory,
+    organization_suffix: () => new RegExp(
+      '\\b(?:[A-Z][A-Za-z0-9]+(?:\\s[A-Z][A-Za-z0-9]+)*),?\\s?' +
+      '(?:Inc\\.|LLC|Corp\\.|GmbH|AG|Ltd\\.)', 'g'
+    ),
+  };
+}
+
+const PATTERN_FACTORIES = buildPatterns();
+
+/**
+ * A collection of regular expression factories for extracting entities.
+ * Each property access creates a fresh RegExp to avoid /g state-bleed.
+ */
+export const REGEX_PATTERNS: Record<string, RegExp> = new Proxy(
+  {} as Record<string, RegExp>,
+  {
+    get(_target, prop: string): RegExp | undefined {
+      const factory = PATTERN_FACTORIES[prop];
+      return factory ? factory() : undefined;
+    },
+    ownKeys(): string[] {
+      return Object.keys(PATTERN_FACTORIES);
+    },
+    getOwnPropertyDescriptor(_target, prop: string) {
+      if (prop in PATTERN_FACTORIES) {
+        return { configurable: true, enumerable: true, writable: false };
+      }
+      return undefined;
+    },
+    has(_target, prop: string): boolean {
+      return prop in PATTERN_FACTORIES;
+    },
+  }
+);

package/src/storage.ts

@@ -0,0 +1,122 @@
+// src/storage.ts
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { IStorage, Logger } from './types.js';
+
+/** Type guard for Node.js system errors with a `code` property. */
+function isNodeError(err: unknown): err is NodeJS.ErrnoException {
+  return err instanceof Error && 'code' in err;
+}
+
+/**
+ * A utility class for performing atomic and durable file I/O operations.
+ * It writes to a temporary file first, then renames it to the final destination,
+ * which prevents data corruption in case of a crash during the write.
+ */
+export class AtomicStorage implements IStorage {
+  private readonly storagePath: string;
+  private readonly logger: Logger;
+
+  /**
+   * Creates an instance of AtomicStorage.
+   * @param storagePath The base directory where files will be stored.
+   * @param logger A logger instance for logging errors.
+   */
+  constructor(storagePath: string, logger: Logger) {
+    this.storagePath = storagePath;
+    this.logger = logger;
+  }
+
+  /**
+   * Ensures that the storage directory exists.
+   */
+  public async init(): Promise<void> {
+    try {
+      await fs.mkdir(this.storagePath, { recursive: true });
+    } catch (err) {
+      this.logger.error(`Failed to create storage directory: ${this.storagePath}`, err as Error);
+      throw err;
+    }
+  }
+
+  /**
+   * Reads and parses a JSON file from the storage path.
+   * @param fileName The name of the file to read (e.g., "facts.json").
+   * @returns The parsed JSON object, or null if the file doesn't exist or is invalid.
+   */
+  async readJson<T>(fileName: string): Promise<T | null> {
+    const filePath = path.join(this.storagePath, fileName);
+    try {
+      const content = await fs.readFile(filePath, 'utf-8');
+      return JSON.parse(content) as T;
+    } catch (err) {
+      if (isNodeError(err) && err.code === 'ENOENT') {
+        return null;
+      }
+      this.logger.error(`Failed to read or parse JSON file: ${filePath}`, err as Error);
+      return null;
+    }
+  }
+
+  /**
+   * Writes a JSON object to a file atomically.
+   * @param fileName The name of the file to write (e.g., "facts.json").
+   * @param data The JSON object to serialize and write.
+   */
+  async writeJson<T>(fileName: string, data: T): Promise<void> {
+    const filePath = path.join(this.storagePath, fileName);
+    const tempFilePath = `${filePath}.${Date.now()}.tmp`;
+
+    try {
+      const jsonString = JSON.stringify(data, null, 2);
+      await fs.writeFile(tempFilePath, jsonString, 'utf-8');
+      await fs.rename(tempFilePath, filePath);
+    } catch (err) {
+      this.logger.error(`Failed to write JSON file atomically: ${filePath}`, err as Error);
+      // Attempt to clean up the temporary file if it exists
+      try {
+        await fs.unlink(tempFilePath);
+      } catch (cleanupErr) {
+        if (!isNodeError(cleanupErr) || cleanupErr.code !== 'ENOENT') {
+          this.logger.warn(`Failed to clean up temporary file: ${tempFilePath}`);
+        }
+      }
+      throw err; // Re-throw the original error
+    }
+  }
+
+  /**
+   * A debouncer function to limit the rate at which a function is executed.
+   * This version is designed for async functions and returns a promise that
+   * resolves with the result of the last invocation.
+   * @param func The async function to debounce.
+   * @param delay The debounce delay in milliseconds.
+   * @returns A debounced version of the function that returns a promise.
+   */
+  static debounce<A extends unknown[], R>(
+    func: (...args: A) => Promise<R>,
+    delay: number
+  ): (...args: A) => Promise<R> {
+    let timeoutId: NodeJS.Timeout | null = null;
+    let resolvers: { resolve: (v: R) => void; reject: (e: unknown) => void }[] = [];
+
+    return (...args: A): Promise<R> => {
+      if (timeoutId) clearTimeout(timeoutId);
+
+      const promise = new Promise<R>((resolve, reject) => {
+        resolvers.push({ resolve, reject });
+      });
+
+      timeoutId = setTimeout(() => {
+        const current = resolvers;
+        resolvers = [];
+        func(...args)
+          .then(result => current.forEach(r => r.resolve(result)))
+          .catch(err => current.forEach(r => r.reject(err)));
+      }, delay);
+
+      return promise;
+    };
+  }
+}

package/src/types.ts

@@ -0,0 +1,144 @@
+// src/types.ts
+
+/**
+ * The public API exposed by the OpenClaw host to the plugin.
+ * This is a subset of the full API, containing only what this plugin needs.
+ */
+export interface OpenClawPluginApi {
+  pluginConfig: Record<string, unknown>;
+  logger: Logger;
+  on: (
+    event: string,
+    handler: (event: HookEvent, ctx: HookContext) => void,
+    options?: { priority: number }
+  ) => void;
+}
+
+/**
+ * A generic logger interface compatible with OpenClaw's logger.
+ */
+export interface Logger {
+  info: (msg: string) => void;
+  warn: (msg: string) => void;
+  error: (msg: string, err?: Error) => void;
+  debug: (msg: string) => void;
+}
+
+/**
+ * Represents the data payload for an OpenClaw hook.
+ * It's a generic shape, as different hooks have different payloads.
+ */
+export interface HookEvent {
+  content?: string;
+  message?: string;
+  text?: string;
+  from?: string;
+  sender?: string;
+  role?: "user" | "assistant";
+  [key: string]: unknown;
+}
+
+/**
+ * Represents the context object passed with each hook event.
+ */
+export interface HookContext {
+  workspace: string; // Absolute path to the OpenClaw workspace
+}
+
+/**
+ * The fully resolved and validated plugin configuration object.
+ */
+export interface KnowledgeConfig {
+  enabled: boolean;
+  workspace: string;
+  extraction: {
+    regex: {
+      enabled: boolean;
+    };
+    llm: {
+      enabled: boolean;
+      model: string;
+      endpoint: string;
+      batchSize: number;
+      cooldownMs: number;
+    };
+  };
+  decay: {
+    enabled: boolean;
+    intervalHours: number;
+    rate: number; // e.g., 0.05 for 5% decay per interval
+  };
+  embeddings: {
+    enabled: boolean;
+    endpoint: string;
+    collectionName: string;
+    syncIntervalMinutes: number;
+  };
+  storage: {
+    maxEntities: number;
+    maxFacts: number;
+    writeDebounceMs: number;
+  };
+}
+
+/**
+ * Represents an extracted entity.
+ */
+export interface Entity {
+  id: string; // e.g., "person:claude"
+  type:
+    | "person"
+    | "location"
+    | "organization"
+    | "date"
+    | "product"
+    | "concept"
+    | "email"
+    | "url"
+    | "unknown";
+  value: string; // The canonical value, e.g., "Claude"
+  mentions: string[]; // Different ways it was mentioned, e.g., ["claude", "Claude's"]
+  count: number;
+  importance: number; // 0.0 to 1.0
+  lastSeen: string; // ISO 8601 timestamp
+  source: ("regex" | "llm")[];
+}
+
+/**
+ * Represents a structured fact (a triple).
+ */
+export interface Fact {
+  id: string; // UUID v4
+  subject: string; // Entity ID
+  predicate: string; // e.g., "is-a", "has-property", "works-at"
+  object: string; // Entity ID or literal value
+  relevance: number; // 0.0 to 1.0, subject to decay
+  createdAt: string; // ISO 8601 timestamp
+  lastAccessed: string; // ISO 8601 timestamp
+  source: "ingested" | "extracted-regex" | "extracted-llm";
+  embedded?: string; // ISO 8601 timestamp of last embedding sync
+}
+
+/**
+ * The data structure for the entities.json file.
+ */
+export interface EntitiesData {
+  updated: string;
+  entities: Entity[];
+}
+
+/**
+ * The data structure for the facts.json file.
+ */
+export interface FactsData {
+  updated: string;
+  facts: Fact[];
+}
+
+/**
+ * Interface for a generic file storage utility.
+ */
+export interface IStorage {
+  readJson<T>(fileName: string): Promise<T | null>;
+  writeJson<T>(fileName: string, data: T): Promise<void>;
+}

package/test/config.test.ts

@@ -0,0 +1,152 @@
+// test/config.test.ts
+
+import { describe, it, beforeEach } from 'node:test';
+import * as assert from 'node:assert';
+import * as path from 'node:path';
+import { resolveConfig, DEFAULT_CONFIG } from '../src/config.js';
+import type { Logger, KnowledgeConfig } from '../src/types.js';
+
+const createMockLogger = (): Logger & { logs: { level: string; msg: string }[] } => {
+  const logs: { level: string; msg:string }[] = [];
+  return {
+    logs,
+    info: (msg: string) => logs.push({ level: 'info', msg }),
+    warn: (msg: string) => logs.push({ level: 'warn', msg }),
+    error: (msg: string) => logs.push({ level: 'error', msg }),
+    debug: (msg: string) => logs.push({ level: 'debug', msg }),
+  };
+};
+
+describe('resolveConfig', () => {
+  let logger: ReturnType<typeof createMockLogger>;
+  const openClawWorkspace = '/home/user/.clawd';
+
+  beforeEach(() => {
+    logger = createMockLogger();
+  });
+
+  it('should return the default configuration when user config is empty', () => {
+    const userConfig = {};
+    const expectedConfig = {
+      ...DEFAULT_CONFIG,
+      workspace: path.join(openClawWorkspace, 'knowledge-engine'),
+    };
+    const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+    assert.deepStrictEqual(resolved, expectedConfig);
+  });
+
+  it('should merge user-provided values with defaults', () => {
+    const userConfig = {
+      extraction: {
+        llm: {
+          enabled: false,
+          model: 'custom-model',
+        },
+      },
+      storage: {
+        writeDebounceMs: 5000,
+      },
+    };
+    const resolved = resolveConfig(userConfig, logger, openClawWorkspace) as KnowledgeConfig;
+    assert.strictEqual(resolved.extraction.llm.enabled, false);
+    assert.strictEqual(resolved.extraction.llm.model, 'custom-model');
+    assert.strictEqual(resolved.extraction.llm.batchSize, DEFAULT_CONFIG.extraction.llm.batchSize); // Should remain default
+    assert.strictEqual(resolved.storage.writeDebounceMs, 5000);
+    assert.strictEqual(resolved.decay.rate, DEFAULT_CONFIG.decay.rate); // Should remain default
+  });
+
+  it('should resolve the workspace path correctly', () => {
+    const userConfig = { workspace: '/custom/path' };
+    const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+    assert.strictEqual(resolved?.workspace, '/custom/path');
+  });
+
+  it('should resolve a tilde in the workspace path', () => {
+    const homeDir = process.env.HOME || '/home/user';
+    process.env.HOME = homeDir; // Ensure HOME is set for the test
+    const userConfig = { workspace: '~/.my-knowledge' };
+    const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+    assert.strictEqual(resolved?.workspace, path.join(homeDir, '.my-knowledge'));
+  });
+
+  it('should use the default workspace path if user path is not provided', () => {
+    const userConfig = {};
+    const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+    assert.strictEqual(resolved?.workspace, path.join(openClawWorkspace, 'knowledge-engine'));
+  });
+
+  describe('Validation', () => {
+    it('should return null and log errors for an invalid LLM endpoint URL', () => {
+      const userConfig = {
+        extraction: {
+          llm: {
+            enabled: true,
+            endpoint: 'not-a-valid-url',
+          },
+        },
+      };
+      const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+      assert.strictEqual(resolved, null);
+      assert.strictEqual(logger.logs.length, 1);
+      assert.strictEqual(logger.logs[0].level, 'error');
+      assert.ok(logger.logs[0].msg.includes('"extraction.llm.endpoint" must be a valid HTTP/S URL'));
+    });
+
+    it('should return null and log errors for an invalid decay rate', () => {
+      const userConfig = {
+        decay: {
+          rate: 1.5, // > 1
+        },
+      };
+      const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+      assert.strictEqual(resolved, null);
+      assert.strictEqual(logger.logs.length, 1);
+      assert.ok(logger.logs[0].msg.includes('"decay.rate" must be between 0 and 1'));
+    });
+
+    it('should return null and log errors for a non-positive decay interval', () => {
+        const userConfig = {
+          decay: {
+            intervalHours: 0,
+          },
+        };
+        const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+        assert.strictEqual(resolved, null);
+        assert.strictEqual(logger.logs.length, 1);
+        assert.ok(logger.logs[0].msg.includes('"decay.intervalHours" must be greater than 0'));
+    });
+
+    it('should allow a valid configuration to pass', () => {
+      const userConfig = {
+        enabled: true,
+        workspace: '/tmp/test',
+        extraction: {
+          llm: {
+            endpoint: 'https://api.example.com',
+          },
+        },
+        decay: {
+          rate: 0.1,
+        },
+        embeddings: {
+          enabled: true,
+          endpoint: 'http://localhost:8000',
+        },
+      };
+      const resolved = resolveConfig(userConfig, logger, openClawWorkspace);
+      assert.ok(resolved);
+      assert.strictEqual(logger.logs.filter(l => l.level === 'error').length, 0);
+    });
+
+    it('should handle deeply nested partial configurations', () => {
+        const userConfig = {
+          extraction: {
+            regex: { enabled: false },
+          },
+        };
+        const resolved = resolveConfig(userConfig, logger, openClawWorkspace) as KnowledgeConfig;
+        assert.strictEqual(resolved.extraction.regex.enabled, false);
+        assert.strictEqual(resolved.extraction.llm.enabled, DEFAULT_CONFIG.extraction.llm.enabled);
+    });
+  });
+});