@ekai/contexto 0.1.0 → 0.1.1

package/README.md

@@ -0,0 +1,102 @@
+# @ekai/contexto
+
+OpenClaw plugin that provides local-first memory — ingest conversation turns and recall relevant context automatically.
+
+Uses [`@ekai/memory`](../../memory/) for semantic extraction, embedding, and SQLite storage.
+
+## Install
+
+```bash
+openclaw plugins install @ekai/contexto
+```
+
+Or from source:
+```bash
+openclaw plugins install ./integrations/openclaw
+```
+
+## Configure
+
+In your OpenClaw config:
+
+```json5
+{
+  plugins: {
+    allow: ["@ekai/contexto"],
+    entries: {
+      "@ekai/contexto": {
+        enabled: true,
+        config: {
+          "dbPath": "~/.openclaw/ekai/memory.db",
+          "provider": "openai",
+          "apiKey": "sk-..."
+        }
+      }
+    }
+  }
+}
+```
+
+| Setting | Default | Description |
+|---------|---------|-------------|
+| `dbPath` | `~/.openclaw/ekai/memory.db` | Path to SQLite memory file |
+| `provider` | (auto-detected) | LLM provider for extraction/embedding (`openai`, `gemini`, `openrouter`) |
+| `apiKey` | (auto-detected) | API key for the selected provider |
+| `bootstrapDelayMs` | `1000` | Milliseconds to wait between sessions during bootstrap backfill |
+
+### Provider auto-detection
+
+When `provider` and `apiKey` are not explicitly configured, the plugin auto-detects from environment variables:
+
+1. **Both `provider` + `apiKey` in config** — used as-is
+2. **Only `provider` in config** — API key resolved from the provider's env var (e.g. `OPENAI_API_KEY`)
+3. **Only `apiKey` in config** — ignored with a warning (ambiguous without provider)
+4. **`MEMORY_EMBED_PROVIDER` or `MEMORY_EXTRACT_PROVIDER` set** — defers to `@ekai/memory` core
+5. **Auto-detect from env** — checks `OPENAI_API_KEY` → `GOOGLE_API_KEY` → `OPENROUTER_API_KEY` (first match wins)
+6. **Nothing found** — passes no provider, lets core handle the error
+
+## Verify
+
+```bash
+openclaw plugins list       # should show @ekai/contexto
+openclaw hooks list         # should show plugin:@ekai/contexto:* hooks
+```
+
+## Bootstrap
+
+If the plugin is installed on an existing OpenClaw instance with historical conversations, use the `/memory-bootstrap` slash command to backfill all session transcripts into memory:
+
+```
+/memory-bootstrap
+```
+
+Bootstrap scans `{stateDir}/agents/*/sessions/*.jsonl`, parses each session, and ingests the messages. Progress is tracked per-session so it can resume if interrupted. Running the command again after completion returns immediately. Configure `bootstrapDelayMs` to control pacing.
+
+## How It Works
+
+Two hooks and a slash command:
+
+1. **`agent_end`** — Ingests new conversation turns into memory (ongoing). Normalizes messages (user + assistant only), redacts secrets, extracts semantic memories via `@ekai/memory`. Only processes the delta since the last ingestion.
+
+2. **`before_prompt_build`** — Recalls relevant memories for the current query and prepends them as context (capped at 2000 chars).
+
+3. **`/memory-bootstrap`** — One-time backfill of all existing session transcripts. Scans the OpenClaw state directory for historical JSONL session files and ingests them into memory. Runs in the background with configurable delay between sessions. Idempotent — safe to re-run.
+
+Delta tracking is persisted to `{dbPath}.progress.json` using composite keys (`agentId:sessionId`) so only new messages are ingested, even across restarts. Both ongoing ingestion and bootstrap share the same progress file.
+
+## Development
+
+```bash
+# Type-check (no build needed -- OpenClaw loads .ts via jiti)
+npm run type-check --workspace=integrations/openclaw
+
+# Run tests
+npm test --workspace=integrations/openclaw
+
+# Local dev install (symlink)
+openclaw plugins install -l ./integrations/openclaw
+```
+
+## License
+
+MIT

package/openclaw.plugin.json

@@ -1,16 +1,32 @@
 {
-  "id": "ekai-contexto",
+  "id": "@ekai/contexto",
   "configSchema": {
     "type": "object",
     "additionalProperties": false,
     "properties": {
-      "logPath": { "type": "string" }
+      "dbPath": { "type": "string" },
+      "provider": { "type": "string" },
+      "apiKey": { "type": "string" },
+      "bootstrapDelayMs": { "type": "number" }
     }
   },
   "uiHints": {
-    "logPath": {
-      "label": "Event log path",
-      "description": "Path to the JSONL event log file"
+    "dbPath": {
+      "label": "Memory database path",
+      "description": "Path to SQLite memory file (default: ~/.openclaw/ekai/memory.db)"
+    },
+    "provider": {
+      "label": "LLM provider",
+      "description": "Provider for extraction and embedding (openai, gemini, openrouter)"
+    },
+    "apiKey": {
+      "label": "API key",
+      "description": "API key for the selected provider"
+    },
+    "bootstrapDelayMs": {
+      "label": "Bootstrap delay",
+      "description": "ms between sessions during backfill (default: 1000)",
+      "advanced": true
     }
   }
 }

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "@ekai/contexto",
-  "version": "0.1.0",
-  "description": "OpenClaw plugin — captures all lifecycle events to JSONL for context, memory, and analytics",
+  "version": "0.1.1",
+  "description": "OpenClaw plugin — local-first memory for context recall and ingest",
   "type": "module",
   "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
   "main": "src/index.ts",
   "files": [
     "src/",
@@ -15,20 +18,25 @@
     "ai",
     "context",
     "memory",
-    "lifecycle",
-    "events"
+    "recall",
+    "ingest"
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/ekai-labs/ekai-gateway",
+    "url": "https://github.com/ekailabs/contexto",
     "directory": "integrations/openclaw"
   },
   "author": "Ekai Labs",
   "scripts": {
-    "type-check": "tsc --noEmit"
+    "type-check": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@ekai/memory": "^0.0.1"
   },
   "devDependencies": {
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.2",
+    "vitest": "^3.0.0"
   },
   "openclaw": {
     "extensions": ["./src/index.ts"]

package/src/bootstrap.ts

@@ -0,0 +1,120 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import type { Memory } from '@ekai/memory';
+import { normalizeMessages, redact } from './index.js';
+
+export type BootstrapStatus = {
+  status: 'running' | 'done';
+  startedAt?: number;
+  completedAt?: number;
+  sessionsProcessed?: number;
+};
+
+export type BootstrapProgress = {
+  __bootstrap?: BootstrapStatus;
+  [key: string]: number | BootstrapStatus | undefined;
+};
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function listDirs(dir: string): string[] {
+  try {
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+  } catch {
+    return [];
+  }
+}
+
+function listJsonl(dir: string): string[] {
+  try {
+    return readdirSync(dir)
+      .filter((f) => f.endsWith('.jsonl') && !f.includes('.reset.'));
+  } catch {
+    return [];
+  }
+}
+
+export async function runBootstrap(opts: {
+  stateDir: string;
+  mem: Memory;
+  progress: BootstrapProgress;
+  saveProgress: () => Promise<void>;
+  logger: { info(m: string): void; warn(m: string): void };
+  delayMs?: number;
+  ensureAgent: (id: string) => void;
+}): Promise<{ sessionsProcessed: number }> {
+  const { stateDir, mem, progress, saveProgress, logger, ensureAgent } = opts;
+  const delayMs = Math.max(0, opts.delayMs ?? 1000);
+
+  if (progress.__bootstrap?.status === 'done') {
+    return { sessionsProcessed: 0 };
+  }
+
+  progress.__bootstrap = { status: 'running', startedAt: Date.now() };
+  await saveProgress();
+
+  const agentsDir = join(stateDir, 'agents');
+  const agentDirs = listDirs(agentsDir);
+
+  if (agentDirs.length === 0) {
+    progress.__bootstrap = { status: 'done', completedAt: Date.now(), sessionsProcessed: 0 };
+    await saveProgress();
+    return { sessionsProcessed: 0 };
+  }
+
+  let count = 0;
+
+  for (const agentId of agentDirs) {
+    const sessionsDir = join(agentsDir, agentId, 'sessions');
+    const files = listJsonl(sessionsDir);
+
+    for (const file of files) {
+      const sessionId = basename(file, '.jsonl');
+      const compositeKey = `${agentId}:${sessionId}`;
+      if (compositeKey in progress) continue;
+
+      const filePath = join(sessionsDir, file);
+      const raw = readFileSync(filePath, 'utf-8');
+      const lines = raw.split('\n');
+      const messages: unknown[] = [];
+
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (!line.trim()) continue;
+
+        let entry: any;
+        try {
+          entry = JSON.parse(line);
+        } catch {
+          logger.warn(`@ekai/contexto: malformed JSON in ${file} line ${i + 1}`);
+          continue;
+        }
+
+        if (entry.type === 'message' && entry.message) {
+          messages.push(entry.message);
+        }
+      }
+
+      const turns = normalizeMessages(messages);
+      if (turns.length > 0) {
+        const redacted = turns.map((t) => ({ role: t.role, content: redact(t.content) }));
+        ensureAgent(agentId);
+        await mem.agent(agentId).add(redacted);
+      }
+
+      progress[compositeKey] = messages.length;
+      await saveProgress();
+      count++;
+
+      if (delayMs > 0) await delay(delayMs);
+    }
+  }
+
+  progress.__bootstrap = { status: 'done', completedAt: Date.now(), sessionsProcessed: count };
+  await saveProgress();
+  return { sessionsProcessed: count };
+}

package/src/index.ts

@@ -1,47 +1,270 @@
-import { EventLog } from './store.js';
-
-const HOOKS = [
-  { name: 'session_start', description: 'Log session start' },
-  { name: 'session_end', description: 'Log session end' },
-  { name: 'message_received', description: 'Log inbound message' },
-  { name: 'message_sent', description: 'Log outbound message' },
-  { name: 'before_prompt_build', description: 'Log pre-prompt state' },
-  { name: 'llm_input', description: 'Log LLM request' },
-  { name: 'llm_output', description: 'Log LLM response' },
-  { name: 'before_tool_call', description: 'Log pre-tool invocation' },
-  { name: 'after_tool_call', description: 'Log tool result' },
-  { name: 'tool_result_persist', description: 'Log tool result persistence' },
-  { name: 'agent_end', description: 'Log agent completion' },
-  { name: 'before_compaction', description: 'Log pre-compaction state' },
-  { name: 'after_compaction', description: 'Log post-compaction state' },
-] as const;
+import { Memory, type ProviderName } from '@ekai/memory';
+import { readFileSync, mkdirSync } from 'node:fs';
+import { writeFile, rename } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import os from 'node:os';
+import { runBootstrap, type BootstrapProgress } from './bootstrap.js';
+
+// --- Inline helpers ---
+
+export function extractText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content))
+    return content
+      .filter((c: any) => c?.type === 'text' && c?.text)
+      .map((c: any) => c.text)
+      .join('\n');
+  if (content && typeof content === 'object' && 'text' in content) return String((content as any).text);
+  return '';
+}
+
+export function normalizeMessages(raw: unknown[]): Array<{ role: string; content: string }> {
+  return raw
+    .filter((m: any) => m?.role === 'user' || m?.role === 'assistant')
+    .map((m: any) => ({ role: m.role, content: extractText(m.content) }))
+    .filter((m) => m.content.trim().length > 0);
+}
+
+export const REDACT_PATTERNS = [
+  /Bearer\s+[A-Za-z0-9._~+\/=-]+/g,
+  /\b(sk|pk|api)[_-][A-Za-z0-9]{20,}\b/g,
+  /\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/g,
+];
+
+export function redact(text: string): string {
+  let out = text;
+  for (const p of REDACT_PATTERNS) out = out.replace(p, '[REDACTED]');
+  return out;
+}
+
+export function lastUserMessage(messages: unknown[]): string | undefined {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const m = messages[i] as any;
+    if (m?.role === 'user') {
+      const t = extractText(m.content);
+      if (t.trim()) return t;
+    }
+  }
+}
+
+export function loadProgress(path: string): BootstrapProgress {
+  try {
+    return JSON.parse(readFileSync(path, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+// --- Provider auto-detection ---
+
+const PROVIDER_ENV_KEYS: Record<ProviderName, string> = {
+  openai: 'OPENAI_API_KEY',
+  gemini: 'GOOGLE_API_KEY',
+  openrouter: 'OPENROUTER_API_KEY',
+};
+
+const AUTO_DETECT_ORDER: Array<{ env: string; provider: ProviderName }> = [
+  { env: 'OPENAI_API_KEY', provider: 'openai' },
+  { env: 'GOOGLE_API_KEY', provider: 'gemini' },
+  { env: 'OPENROUTER_API_KEY', provider: 'openrouter' },
+];
+
+export function resolveMemoryProvider(
+  pluginConfig: any,
+  logger: { info(m: string): void; warn(m: string): void },
+): { provider: ProviderName; apiKey: string; source: string } | undefined {
+  const cfgProvider = pluginConfig?.provider;
+  const cfgApiKey = pluginConfig?.apiKey;
+
+  // Case 1: both explicit
+  if (cfgProvider && cfgApiKey) {
+    return { provider: cfgProvider as ProviderName, apiKey: cfgApiKey, source: 'config' };
+  }
+
+  // Case 2: provider only → resolve key from env
+  if (cfgProvider && !cfgApiKey) {
+    const envVar = PROVIDER_ENV_KEYS[cfgProvider];
+    const key = envVar && process.env[envVar];
+    if (key) {
+      return { provider: cfgProvider as ProviderName, apiKey: key, source: 'config+env' };
+    }
+    logger.warn(`@ekai/contexto: provider '${cfgProvider}' configured but ${envVar ?? 'API key env var'} not set`);
+    return undefined;
+  }
+
+  // Case 3: apiKey only → ambiguous, warn and ignore
+  if (!cfgProvider && cfgApiKey) {
+    logger.warn('@ekai/contexto: apiKey configured without provider — ignoring (set provider to use it)');
+  }
+
+  // Case 4: defer to core if MEMORY_*_PROVIDER is set
+  if (process.env.MEMORY_EMBED_PROVIDER || process.env.MEMORY_EXTRACT_PROVIDER) {
+    return undefined;
+  }
+
+  // Case 5: auto-detect from env keys
+  for (const { env, provider } of AUTO_DETECT_ORDER) {
+    const key = process.env[env];
+    if (key) {
+      return { provider, apiKey: key, source: 'env' };
+    }
+  }
+
+  // Case 6: nothing found, let core handle
+  return undefined;
+}
+
+// --- Plugin definition ---
 
 export default {
-  id: 'ekai-contexto',
+  id: '@ekai/contexto',
   name: 'Ekai Contexto',
-  description: 'Context engine for OpenClaw — captures lifecycle events, extensible to memory injection',
-  configSchema: {},
+  description: 'Local-first memory for OpenClaw',
+  configSchema: {
+    type: 'object',
+    additionalProperties: false,
+    properties: {
+      dbPath: { type: 'string' },
+      provider: { type: 'string' },
+      apiKey: { type: 'string' },
+    },
+  },
 
   register(api: any) {
-    const logPath = api.resolvePath(api.pluginConfig?.logPath ?? '~/.openclaw/ekai/events.jsonl');
-    const log = new EventLog(logPath);
-
-    for (const hook of HOOKS) {
-      api.registerHook({
-        name: `contexto:${hook.name}`,
-        description: hook.description,
-        hook: hook.name,
-        handler: (event: unknown, ctx: unknown) => {
-          try {
-            log.append(hook.name, event, ctx);
-          } catch (err) {
-            // Never crash OpenClaw — log the failure and move on
-            api.logger.warn(`ekai-contexto: failed to log ${hook.name}: ${String(err)}`);
-          }
-        },
-      });
+    const dbPath = api.resolvePath(api.pluginConfig?.dbPath ?? '~/.openclaw/ekai/memory.db');
+    mkdirSync(dirname(dbPath), { recursive: true });
+    const resolved = resolveMemoryProvider(api.pluginConfig, api.logger);
+    const mem = new Memory({
+      ...(resolved ? { provider: resolved.provider, apiKey: resolved.apiKey } : {}),
+      dbPath,
+    });
+
+    // --- Agent management ---
+    const knownAgents = new Set<string>();
+    function ensureAgent(agentId: string) {
+      if (knownAgents.has(agentId)) return;
+      const exists = mem.getAgents().some((a) => a.id === agentId);
+      if (!exists) mem.addAgent(agentId, { name: agentId });
+      knownAgents.add(agentId);
+    }
+    // Seed from existing agents in DB (survives restarts)
+    for (const a of mem.getAgents()) knownAgents.add(a.id);
+    ensureAgent('main');
+
+    // --- Delta tracking (persisted) ---
+    const progressPath = dbPath.replace(/\.db$/, '') + '.progress.json';
+    const progress: BootstrapProgress = loadProgress(progressPath);
+
+    async function saveProgress() {
+      const tmp = progressPath + '.tmp';
+      await writeFile(tmp, JSON.stringify(progress), 'utf-8');
+      await rename(tmp, progressPath);
     }
 
-    api.logger.info(`ekai-contexto: logging to ${logPath}`);
+    const MAX_PREPEND_CHARS = 2000;
+
+    // --- agent_end: ingest delta ---
+    api.on('agent_end', async (event: any, ctx: any) => {
+      const sessionId = ctx?.sessionId ?? ctx?.sessionKey;
+      if (!sessionId || !event?.messages?.length) return;
+
+      const agentId = ctx?.agentId ?? 'main';
+      const progressKey = `${agentId}:${sessionId}`;
+      const lastCount = (progress[progressKey] as number) ?? 0;
+      // Handle count shrink (e.g. compaction) — re-ingest from start
+      const startIdx = event.messages.length < lastCount ? 0 : lastCount;
+      if (startIdx >= event.messages.length) return;
+
+      try {
+        ensureAgent(agentId);
+
+        const delta = event.messages.slice(startIdx);
+        const turns = normalizeMessages(delta);
+        if (turns.length === 0) {
+          progress[progressKey] = event.messages.length;
+          await saveProgress();
+          return;
+        }
+
+        const redacted = turns.map((t) => ({ role: t.role, content: redact(t.content) }));
+        await mem.agent(agentId).add(redacted, { userId: ctx?.userId });
+        progress[progressKey] = event.messages.length;
+        await saveProgress();
+
+        api.logger.info(`@ekai/contexto: ingested ${redacted.length} turns`);
+      } catch (err) {
+        api.logger.warn(`@ekai/contexto: ingest failed: ${String(err)}`);
+      }
+    });
+
+    // --- before_prompt_build: recall ---
+    api.on('before_prompt_build', async (event: any, ctx: any) => {
+      try {
+        const agentId = ctx?.agentId ?? 'main';
+        if (!knownAgents.has(agentId)) return;
+
+        const query = lastUserMessage(event?.messages ?? []);
+        if (!query) return;
+
+        const results = await mem.agent(agentId).search(query, { userId: ctx?.userId });
+        if (results.length === 0) return;
+
+        let block = results
+          .slice(0, 5)
+          .map((r: any) => `- ${r.content}`)
+          .join('\n');
+
+        if (block.length > MAX_PREPEND_CHARS) block = block.slice(0, MAX_PREPEND_CHARS) + '…';
+
+        return { prependContext: `## Relevant memories\n${block}` };
+      } catch (err) {
+        api.logger.warn(`@ekai/contexto: recall failed: ${String(err)}`);
+      }
+    });
+
+    // --- /memory-bootstrap command ---
+    api.registerCommand({
+      name: 'memory-bootstrap',
+      description: 'Backfill existing session history into memory',
+      acceptsArgs: false,
+      requireAuth: true,
+      handler: async () => {
+        if (progress.__bootstrap?.status === 'done') {
+          return { text: 'Bootstrap already completed.' };
+        }
+        if (progress.__bootstrap?.status === 'running') {
+          return { text: 'Bootstrap already in progress.' };
+        }
+
+        const stateDir = api.runtime?.state?.resolveStateDir?.(process.env, os.homedir());
+        if (!stateDir) {
+          return { text: 'Could not resolve state directory.' };
+        }
+
+        progress.__bootstrap = { status: 'running', startedAt: Date.now() };
+        await saveProgress();
+
+        runBootstrap({
+          stateDir,
+          mem,
+          progress,
+          saveProgress,
+          logger: api.logger,
+          ensureAgent,
+          delayMs: Math.max(0, Number(api.pluginConfig?.bootstrapDelayMs) || 1000),
+        })
+          .then((r) => api.logger.info(`@ekai/contexto: bootstrap done — ${r.sessionsProcessed} sessions`))
+          .catch((err) => {
+            api.logger.warn(`@ekai/contexto: bootstrap failed: ${err}`);
+            progress.__bootstrap = undefined;
+            saveProgress();
+          });
+
+        return { text: 'Memory bootstrap started. Check logs for progress.' };
+      },
+    });
+
+    const providerInfo = resolved ? ` (${resolved.provider} via ${resolved.source})` : '';
+    api.logger.info(`@ekai/contexto: memory at ${dbPath}${providerInfo}`);
   },
 };

package/src/store.ts

@@ -1,56 +0,0 @@
-import { appendFileSync, mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
-
-/**
- * Safe JSON replacer — handles circular refs, BigInt, undefined, errors.
- * No external deps needed.
- */
-function safeReplacer() {
-  const seen = new WeakSet();
-  return (_key: string, value: unknown): unknown => {
-    if (typeof value === 'bigint') return value.toString();
-    if (value instanceof Error) return { message: value.message, stack: value.stack };
-    if (value !== null && typeof value === 'object') {
-      if (seen.has(value)) return '[Circular]';
-      seen.add(value);
-    }
-    return value;
-  };
-}
-
-function safeStringify(obj: unknown): string {
-  try {
-    return JSON.stringify(obj, safeReplacer());
-  } catch {
-    return JSON.stringify({ _error: 'serialization failed' });
-  }
-}
-
-/**
- * Extract sessionId and agentId from event or ctx (whichever has them).
- */
-function extractIds(event: any, ctx: any): { sessionId?: string; agentId?: string } {
-  return {
-    sessionId: event?.sessionId ?? ctx?.sessionId ?? ctx?.sessionKey ?? undefined,
-    agentId: event?.agentId ?? ctx?.agentId ?? undefined,
-  };
-}
-
-export class EventLog {
-  constructor(private path: string) {
-    mkdirSync(dirname(path), { recursive: true });
-  }
-
-  /**
-   * Append a hook event to the JSONL log.
-   * Uses appendFileSync for tool_result_persist sync compatibility.
-   *
-   * NOTE (v0): appendFileSync blocks the event loop. Acceptable at low volume.
-   * Future: async write with buffering for high-throughput hooks.
-   */
-  append(hook: string, event: unknown, ctx: unknown): void {
-    const { sessionId, agentId } = extractIds(event, ctx);
-    const line = safeStringify({ ts: Date.now(), hook, sessionId, agentId, event, ctx });
-    appendFileSync(this.path, line + '\n');
-  }
-}