kiku-agent 1.2.0 → 1.3.1

package/server/agent/memory/hierarchicalStore.js

@@ -0,0 +1,523 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { HIERARCHICAL_MEMORY_DIR, ARTIFACTS_STORE_DIR } from '../appPaths.js';
+
+export const POINTER_THRESHOLD_CHARS = 1000;
+
+export const VALID_RUN_ID_REGEX = /^[a-zA-Z0-9_-]{3,64}$/;
+export const VALID_DOMAIN_REGEX = /^[a-zA-Z0-9_-]{2,64}$/;
+
+
+export function parseAndValidatePath(rawPath) {
+  if (typeof rawPath !== 'string') {
+    throw new Error('Memory path must be a string.');
+  }
+
+  const normalized = rawPath.trim().replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
+  if (normalized.includes('..') || /[\0\r\n]/.test(normalized)) {
+    throw new Error(`Invalid path traversal or characters detected: "${rawPath}"`);
+  }
+
+  const parts = normalized.split('/');
+  if (parts.length < 2 || parts[0] !== 'kiku') {
+    throw new Error(`Path must start with "kiku/". Received: "${rawPath}"`);
+  }
+
+  const section = parts[1];
+  if (section === 'runs') {
+    if (parts.length !== 4) {
+      throw new Error(`Run paths must follow "kiku/runs/{run_id}/{subpath}". Received: "${rawPath}"`);
+    }
+    const runId = parts[2];
+    const subpath = parts[3];
+    if (!VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Invalid run_id "${runId}". Must match ${VALID_RUN_ID_REGEX}`);
+    }
+    if (!['state', 'plan', 'artifacts'].includes(subpath)) {
+      throw new Error(`Invalid run subpath "${subpath}". Allowed: state, plan, artifacts`);
+    }
+    return { section: 'runs', runId, subpath, path: normalized, type: 'context' };
+  }
+
+  if (section === 'reflections') {
+    if (parts.length !== 3) {
+      throw new Error(`Reflection paths must follow "kiku/reflections/{tool_or_domain}". Received: "${rawPath}"`);
+    }
+    const domain = parts[2];
+    if (!VALID_DOMAIN_REGEX.test(domain)) {
+      throw new Error(`Invalid reflection domain/tool "${domain}". Must match ${VALID_DOMAIN_REGEX}`);
+    }
+    return { section: 'reflections', domain, path: normalized, type: 'context' };
+  }
+
+  if (section === 'guidelines') {
+    if (parts.length !== 3) {
+      throw new Error(`Guideline paths must follow "kiku/guidelines/{domain}". Received: "${rawPath}"`);
+    }
+    const domain = parts[2];
+    if (!VALID_DOMAIN_REGEX.test(domain)) {
+      throw new Error(`Invalid guideline domain "${domain}". Must match ${VALID_DOMAIN_REGEX}`);
+    }
+    return { section: 'guidelines', domain, path: normalized, type: 'user' };
+  }
+
+  if (section === 'archive') {
+    if (parts.length !== 3) {
+      throw new Error(`Archive paths must follow "kiku/archive/{run_id}". Received: "${rawPath}"`);
+    }
+    const runId = parts[2];
+    if (!VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Invalid archive run_id "${runId}". Must match ${VALID_RUN_ID_REGEX}`);
+    }
+    return { section: 'archive', runId, path: normalized, type: 'archive' };
+  }
+
+  throw new Error(`Unrecognized memory section "${section}" in path "${rawPath}". Allowed sections: runs, reflections, guidelines, archive.`);
+}
+
+export function computeContentHash(content) {
+  const json = JSON.stringify(content, (k, v) => {
+    if (v && typeof v === 'object' && !Array.isArray(v)) {
+      const sorted = {};
+      for (const key of Object.keys(v).sort()) {
+        sorted[key] = v[key];
+      }
+      return sorted;
+    }
+    return v;
+  });
+  return crypto.createHash('sha256').update(json || '').digest('hex');
+}
+
+export class HierarchicalStore {
+  constructor(options = {}) {
+    this.persistenceEnabled = options.persistence !== false;
+    this.baseDir = options.baseDir || HIERARCHICAL_MEMORY_DIR;
+    this.artifactsDir = options.artifactsDir || ARTIFACTS_STORE_DIR;
+    this.nodes = new Map();
+    this._inMemoryArtifacts = new Map();
+    this._pointerCache = new Map();
+    this._indexDirty = false;
+    this._indexFlushTimer = null;
+    this._indexFlushChain = Promise.resolve();
+    this._initialized = false;
+  }
+
+  async init() {
+    if (this._initialized) return true;
+    if (this.persistenceEnabled) {
+      try {
+        await fs.mkdir(this.baseDir, { recursive: true, mode: 0o700 });
+        await fs.mkdir(this.artifactsDir, { recursive: true, mode: 0o700 });
+        await this._loadAllFromDisk();
+      } catch (err) {
+
+      }
+    }
+    this._initialized = true;
+    return true;
+  }
+
+  async _loadAllFromDisk() {
+    try {
+      const indexFile = path.join(this.baseDir, 'index.json');
+      const raw = await fs.readFile(indexFile, 'utf-8');
+      const items = JSON.parse(raw);
+      if (Array.isArray(items)) {
+        for (const item of items) {
+          if (item?.path) {
+            this.nodes.set(item.path, item);
+          }
+        }
+      }
+    } catch {
+    
+    }
+  }
+
+  async _saveIndexToDisk() {
+    if (!this.persistenceEnabled) return;
+    try {
+      const indexFile = path.join(this.baseDir, 'index.json');
+      const tmpFile = `${indexFile}.${Date.now()}.${Math.random().toString(36).slice(2, 6)}.tmp`;
+      const items = Array.from(this.nodes.values());
+      await fs.writeFile(tmpFile, JSON.stringify(items), { encoding: 'utf-8', mode: 0o600 });
+      await fs.rename(tmpFile, indexFile);
+    } catch {
+
+    }
+  }
+
+  _markIndexDirty() {
+    if (!this.persistenceEnabled) return;
+    this._indexDirty = true;
+    if (this._indexFlushTimer) return;
+    this._indexFlushTimer = setTimeout(() => {
+      this._indexFlushTimer = null;
+      this.flushIndex().catch(() => {});
+    }, 150);
+  }
+
+  async flushIndex() {
+    if (this._indexFlushTimer) {
+      clearTimeout(this._indexFlushTimer);
+      this._indexFlushTimer = null;
+    }
+    if (!this._indexDirty) return true;
+    const next = this._indexFlushChain.then(() => {
+      this._indexDirty = false;
+      return this._saveIndexToDisk();
+    });
+    this._indexFlushChain = next.catch(() => {});
+    await next;
+    return true;
+  }
+
+ 
+  async _applyPointerOverData(data) {
+    if (data === null || data === undefined) return data;
+
+    let serialized;
+    let isString = typeof data === 'string';
+
+    if (isString) {
+      serialized = data;
+    } else if (typeof data === 'object') {
+      serialized = JSON.stringify(data);
+    } else {
+      return data;
+    }
+
+    if (serialized.length <= POINTER_THRESHOLD_CHARS) {
+      return data;
+    }
+
+    
+    const hash = crypto.createHash('sha256').update(serialized).digest('hex');
+    const filename = `${hash}.json`;
+    const artifactPath = path.join(this.artifactsDir, filename);
+
+    if (this.persistenceEnabled) {
+      try {
+        await fs.mkdir(this.artifactsDir, { recursive: true, mode: 0o700 });
+        await fs.writeFile(artifactPath, serialized, { encoding: 'utf-8', mode: 0o600 });
+      } catch (err) {
+        console.error(`[HierarchicalStore] Failed to write pointer artifact: ${err.message}`);
+      }
+    } else {
+      this._inMemoryArtifacts.set(hash, serialized);
+    }
+
+    const preview = isString
+      ? serialized.slice(0, 150) + '...'
+      : (typeof data === 'object' ? Object.keys(data).join(', ').slice(0, 120) : '');
+
+    return {
+      $type: 'pointer',
+      uri: `file://${artifactPath.replace(/\\/g, '/')}`,
+      sizeBytes: Buffer.byteLength(serialized, 'utf-8'),
+      charCount: serialized.length,
+      sha256: hash,
+      preview,
+      isString,
+      createdAt: new Date().toISOString()
+    };
+  }
+
+  
+  async resolvePointer(pointer) {
+    if (!pointer || pointer.$type !== 'pointer' || !pointer.uri) {
+      return pointer;
+    }
+    if (!this.persistenceEnabled && pointer.sha256 && this._inMemoryArtifacts.has(pointer.sha256)) {
+      const raw = this._inMemoryArtifacts.get(pointer.sha256);
+      return pointer.isString ? raw : JSON.parse(raw);
+    }
+    let rawPath = String(pointer.uri).replace(/^file:\/\//, '');
+    if (process.platform === 'win32' && /^\/[a-zA-Z]:/.test(rawPath)) {
+      rawPath = rawPath.slice(1);
+    }
+    const resolvedPath = path.resolve(rawPath);
+    const allowedDir = path.resolve(this.artifactsDir);
+    const allowedBase = path.resolve(this.baseDir);
+    const inArtifacts = resolvedPath === allowedDir || resolvedPath.startsWith(allowedDir + path.sep);
+    const inBase = resolvedPath === allowedBase || resolvedPath.startsWith(allowedBase + path.sep);
+    if (!inArtifacts && !inBase) {
+      return { error: 'Failed to resolve pointer artifact: Access denied (path outside storage directory)' };
+    }
+    const cacheKey = pointer.sha256 || null;
+    if (cacheKey) {
+      const cached = this._pointerCache.get(cacheKey);
+      if (cached) {
+        if (cached.expires > Date.now()) {
+          return pointer.isString ? cached.parsed : structuredClone(cached.parsed);
+        }
+        this._pointerCache.delete(cacheKey);
+      }
+    }
+    try {
+      const raw = await fs.readFile(resolvedPath, 'utf-8');
+      const parsed = pointer.isString ? raw : JSON.parse(raw);
+      if (cacheKey) {
+        this._pointerCache.set(cacheKey, { expires: Date.now() + 60000, parsed });
+        if (this._pointerCache.size > 200) {
+          this._pointerCache.delete(this._pointerCache.keys().next().value);
+        }
+      }
+      return pointer.isString ? parsed : structuredClone(parsed);
+    } catch (err) {
+      return { error: `Failed to resolve pointer artifact: ${err.message}` };
+    }
+  }
+
+ 
+  _assertRunScope(parsed, scopedRunId) {
+    if (scopedRunId && parsed.section === 'runs') {
+      if (parsed.runId !== scopedRunId) {
+        throw new Error(`Run scoping violation: Active run "${scopedRunId}" cannot mutate run "${parsed.runId}"`);
+      }
+    }
+  }
+
+  
+  async setNode(rawPath, content, options = {}) {
+    await this.init();
+    const parsed = parseAndValidatePath(rawPath);
+    this._assertRunScope(parsed, options.scopedRunId);
+
+    const processedContent = await this._applyPointerOverData(content);
+    const contentHash = computeContentHash(processedContent);
+
+    const existing = this.nodes.get(parsed.path);
+    if (existing && existing.contentHash === contentHash && !options.force) {
+      return { status: 'unchanged', path: parsed.path, contentHash, node: existing };
+    }
+
+    const now = new Date().toISOString();
+    const node = {
+      path: parsed.path,
+      section: parsed.section,
+      type: options.type || parsed.type,
+      runId: parsed.runId || null,
+      domain: parsed.domain || null,
+      content: processedContent,
+      contentHash,
+      createdAt: existing ? existing.createdAt : now,
+      updatedAt: now,
+      metadata: options.metadata || (existing ? existing.metadata : {})
+    };
+
+    this.nodes.set(parsed.path, node);
+    this._markIndexDirty();
+    return { status: 'saved', path: parsed.path, contentHash, node };
+  }
+
+  
+  async replaceNode(rawPath, content, options = {}) {
+    return this.setNode(rawPath, content, { ...options, replace: true });
+  }
+
+  
+  async getNode(rawPath) {
+    await this.init();
+    const parsed = parseAndValidatePath(rawPath);
+    return this.nodes.get(parsed.path) || null;
+  }
+
+  
+  async deleteNode(rawPath, options = {}) {
+    await this.init();
+    const parsed = parseAndValidatePath(rawPath);
+    this._assertRunScope(parsed, options.scopedRunId);
+
+    const existed = this.nodes.delete(parsed.path);
+    if (existed) {
+      this._markIndexDirty();
+    }
+    return existed;
+  }
+
+  
+  async listNodes(prefix = '') {
+    await this.init();
+    const normPrefix = String(prefix || '').trim().replace(/\\/g, '/');
+    const matched = [];
+    for (const [key, node] of this.nodes.entries()) {
+      if (!normPrefix || key.startsWith(normPrefix)) {
+        matched.push(node);
+      }
+    }
+    return matched;
+  }
+
+ 
+  async checkpointStep({ runId, stepName, status, summary, metadata = {} }) {
+    if (!runId || !VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Valid runId is required for checkpointing. Received: "${runId}"`);
+    }
+    if (!stepName) {
+      throw new Error('stepName is required for checkpointing.');
+    }
+    const cleanStatus = ['pending', 'in_progress', 'completed', 'failed', 'paused', 'waiting_user'].includes(status)
+      ? status
+      : 'in_progress';
+
+  
+    const statePath = `kiku/runs/${runId}/state`;
+    const currentStateNode = await this.getNode(statePath);
+    const previousState = currentStateNode?.content || {
+      runId,
+      status: 'IN_PROGRESS',
+      currentStep: null,
+      metadata: {}
+    };
+
+    const newState = {
+      runId,
+      status: cleanStatus === 'failed' ? 'FAILED' : (cleanStatus === 'completed' ? 'IN_PROGRESS' : cleanStatus.toUpperCase()),
+      currentStep: stepName,
+      lastSummary: summary || '',
+      updatedAt: new Date().toISOString(),
+      metadata: { ...(previousState.metadata || {}), ...metadata }
+    };
+    await this.setNode(statePath, newState, { scopedRunId: runId });
+
+  
+    const planPath = `kiku/runs/${runId}/plan`;
+    const currentPlanNode = await this.getNode(planPath);
+    const plan = currentPlanNode?.content || {
+      runId,
+      subtasks: []
+    };
+
+    if (!Array.isArray(plan.subtasks)) plan.subtasks = [];
+    const existingTask = plan.subtasks.find(t => t.name === stepName);
+    if (existingTask) {
+      existingTask.status = cleanStatus;
+      existingTask.summary = summary || existingTask.summary;
+      existingTask.updatedAt = new Date().toISOString();
+    } else {
+      plan.subtasks.push({
+        id: `step_${plan.subtasks.length + 1}`,
+        name: stepName,
+        status: cleanStatus,
+        summary: summary || '',
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString()
+      });
+    }
+
+    await this.setNode(planPath, plan, { scopedRunId: runId });
+    await this.flushIndex();
+
+    return {
+      runId,
+      currentStep: stepName,
+      status: cleanStatus,
+      summary,
+      totalSubtasks: plan.subtasks.length
+    };
+  }
+
+  
+  async recordFailure({ runId, errorTrace, recoveryAction, toolName = null }) {
+    if (!runId || !VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Valid runId is required. Received: "${runId}"`);
+    }
+
+    const statePath = `kiku/runs/${runId}/state`;
+    const currentStateNode = await this.getNode(statePath);
+    const currentState = currentStateNode?.content || { runId, failures: [] };
+    const failures = Array.isArray(currentState.failures) ? currentState.failures : [];
+
+    const failureRecord = {
+      id: `fail_${Date.now()}`,
+      toolName: toolName || 'unknown',
+      errorTrace: String(errorTrace || '').slice(0, 1500),
+      recoveryAction: String(recoveryAction || '').slice(0, 1000),
+      recordedAt: new Date().toISOString()
+    };
+
+    failures.push(failureRecord);
+    currentState.failures = failures;
+    currentState.lastFailure = failureRecord;
+
+    await this.setNode(statePath, currentState, { scopedRunId: runId });
+    return { status: 'recorded', runId, failure: failureRecord };
+  }
+
+  
+  async loadActiveRun(runId) {
+    if (!runId || !VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Valid runId is required. Received: "${runId}"`);
+    }
+    const [stateNode, planNode, artifactsNode] = await Promise.all([
+      this.getNode(`kiku/runs/${runId}/state`),
+      this.getNode(`kiku/runs/${runId}/plan`),
+      this.getNode(`kiku/runs/${runId}/artifacts`)
+    ]);
+
+    return {
+      runId,
+      state: stateNode?.content || null,
+      plan: planNode?.content || null,
+      artifacts: artifactsNode?.content || null,
+      exists: Boolean(stateNode || planNode)
+    };
+  }
+
+  
+  async archiveRun(runId, ttlDays = 7) {
+    if (!runId || !VALID_RUN_ID_REGEX.test(runId)) {
+      throw new Error(`Valid runId is required. Received: "${runId}"`);
+    }
+    const runData = await this.loadActiveRun(runId);
+    if (!runData.exists) {
+      return { status: 'not_found', runId };
+    }
+
+    const now = Date.now();
+    const expiresAt = new Date(now + ttlDays * 24 * 60 * 60 * 1000).toISOString();
+
+    const archiveContent = {
+      runId,
+      archivedAt: new Date(now).toISOString(),
+      expiresAt,
+      ttlDays,
+      state: runData.state,
+      plan: runData.plan,
+      artifacts: runData.artifacts
+    };
+
+    const archivePath = `kiku/archive/${runId}`;
+    await this.setNode(archivePath, archiveContent, { type: 'archive' });
+
+   
+    await Promise.all([
+      this.deleteNode(`kiku/runs/${runId}/state`),
+      this.deleteNode(`kiku/runs/${runId}/plan`),
+      this.deleteNode(`kiku/runs/${runId}/artifacts`)
+    ]);
+    await this.flushIndex();
+
+    return { status: 'archived', runId, archivePath, expiresAt };
+  }
+
+  
+  async purgeExpiredArchives() {
+    await this.init();
+    const archives = await this.listNodes('kiku/archive/');
+    const now = Date.now();
+    let purgedCount = 0;
+
+    for (const arch of archives) {
+      const expiresAt = arch.content?.expiresAt;
+      if (expiresAt && new Date(expiresAt).getTime() < now) {
+        await this.deleteNode(arch.path);
+        purgedCount++;
+      }
+    }
+    return { purgedCount, remaining: archives.length - purgedCount };
+  }
+}

package/server/agent/memory/reflectionWorker.js

@@ -0,0 +1,110 @@
+export class ReflectionWorker {
+  constructor(options = {}) {
+    this.store = options.store;
+    this.memory = options.memory;
+    this.defaultTtlDays = options.ttlDays || 7;
+    this._isProcessing = false;
+  }
+
+
+  async processPostRunCompaction(runId, finalStatus = 'COMPLETED', metadata = {}) {
+    if (!this.store) {
+      return { success: false, error: 'HierarchicalStore instance required' };
+    }
+
+    try {
+      const runData = await this.store.loadActiveRun(runId);
+      if (!runData.exists) {
+        return { success: false, error: `Run "${runId}" does not exist in active memory.` };
+      }
+
+      const state = runData.state || {};
+      const plan = runData.plan || {};
+      const subtasks = Array.isArray(plan.subtasks) ? plan.subtasks : [];
+      const failures = Array.isArray(state.failures) ? state.failures : [];
+
+
+      const completedCount = subtasks.filter(t => t.status === 'completed').length;
+      const failedCount = subtasks.filter(t => t.status === 'failed').length;
+      const stepSummaries = subtasks
+        .filter(t => t.summary)
+        .map(t => `- Step "${t.name}" [${t.status}]: ${t.summary}`)
+        .join('\n');
+
+      const executiveSummary = {
+        runId,
+        finalStatus,
+        totalSteps: subtasks.length,
+        completedSteps: completedCount,
+        failedSteps: failedCount,
+        stepHighlights: stepSummaries || (state.lastSummary ? `- ${state.lastSummary}` : 'No step summaries recorded.'),
+        compiledAt: new Date().toISOString(),
+        metadata: { ...(state.metadata || {}), ...metadata }
+      };
+
+   
+      let reflectionsCount = 0;
+      for (const fail of failures) {
+        const domain = (fail.toolName && fail.toolName !== 'unknown')
+          ? fail.toolName.replace(/[^a-zA-Z0-9_-]/g, '_').toLowerCase()
+          : 'general_execution';
+
+        const reflectionPath = `kiku/reflections/${domain}`;
+        const existingNode = await this.store.getNode(reflectionPath);
+        const existing = existingNode?.content || {
+          domain,
+          occurrences: 0,
+          patterns: [],
+          lessons: []
+        };
+
+        const patterns = Array.isArray(existing.patterns) ? existing.patterns : [];
+        const isDuplicate = patterns.some(p => p.errorTrace === fail.errorTrace);
+
+        if (!isDuplicate) {
+          patterns.push({
+            errorTrace: fail.errorTrace,
+            recoveryAction: fail.recoveryAction,
+            recordedAt: fail.recordedAt || new Date().toISOString(),
+            runId
+          });
+        }
+
+        const updatedReflection = {
+          domain,
+          issuePattern: fail.errorTrace,
+          recoveryAction: fail.recoveryAction,
+          occurrences: (existing.occurrences || 0) + 1,
+          confidence: Math.min(1.5, 0.8 + (existing.occurrences || 0) * 0.1),
+          patterns: patterns.slice(-10),
+          lastUpdated: new Date().toISOString()
+        };
+
+        await this.store.setNode(reflectionPath, updatedReflection, { type: 'context' });
+        reflectionsCount++;
+      }
+
+    
+      state.executiveSummary = executiveSummary;
+      state.status = finalStatus;
+      await this.store.setNode(`kiku/runs/${runId}/state`, state, { scopedRunId: runId });
+
+     
+      const archiveResult = await this.store.archiveRun(runId, this.defaultTtlDays);
+
+    
+      this.store.purgeExpiredArchives().catch(() => {});
+
+      return {
+        success: true,
+        runId,
+        finalStatus,
+        executiveSummary,
+        reflectionsExtracted: reflectionsCount,
+        archive: archiveResult
+      };
+    } catch (err) {
+      return { success: false, error: err.message };
+    }
+  }
+}