forge-server 0.1.0

package/src/memory/memory-compat.ts

@@ -0,0 +1,233 @@
+/**
+ * Claude-flow / Ruflo memory compatibility layer.
+ * Translates between the two memory schemas and the unified ObservationPayload format.
+ * Ensures memory_store, memory_retrieve, memory_search work with both parameter shapes.
+ *
+ * Claude-flow schema: { content, type, category, tags, metadata, importance, ttl }
+ * Ruflo schema:       { key, value, namespace, tags, ttl }
+ */
+
+import type { ObservationPayload } from '../util/types.js';
+
+// ============================================================
+// Parameter Shape Detection
+// ============================================================
+
+export type MemorySource = 'forge' | 'claude-flow' | 'ruflo';
+
+export interface NormalizedMemoryInput {
+  content: string;
+  key: string | null;
+  namespace: string;
+  type: ObservationPayload['type'];
+  tags: string[];
+  importance: number;
+  ttl?: number;
+  metadata: Record<string, unknown>;
+  source: MemorySource;
+  // Ruflo-style original value (preserved for retrieval)
+  originalValue?: unknown;
+}
+
+/**
+ * Detect whether input follows ruflo-style (key+value) or claude-flow-style (content).
+ */
+export function detectMemorySource(input: {
+  key?: string;
+  value?: unknown;
+  content?: string;
+  type?: string;
+  category?: string;
+  namespace?: string;
+}): MemorySource {
+  if (input.key !== undefined && input.value !== undefined) {
+    return 'ruflo';
+  }
+  if (input.content !== undefined || input.type !== undefined || input.category !== undefined) {
+    return 'claude-flow';
+  }
+  // Ambiguous - use key presence as tiebreaker
+  if (input.key !== undefined) {
+    return 'ruflo';
+  }
+  return 'claude-flow';
+}
+
+/**
+ * Normalize a memory_store input into unified format regardless of schema origin.
+ */
+export function normalizeMemoryInput(input: {
+  // Ruflo-style
+  key?: string;
+  value?: unknown;
+  // Claude-flow-style
+  content?: string;
+  type?: 'episodic' | 'semantic' | 'procedural' | 'working';
+  category?: string;
+  // Shared
+  namespace?: string;
+  tags?: string[];
+  metadata?: Record<string, unknown>;
+  importance?: number;
+  ttl?: number;
+}): NormalizedMemoryInput {
+  const source = detectMemorySource(input);
+
+  let content: string;
+  let key: string | null;
+  let originalValue: unknown;
+
+  if (source === 'ruflo') {
+    key = input.key ?? `mem-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
+    originalValue = input.value;
+    content = typeof input.value === 'string'
+      ? input.value
+      : JSON.stringify(input.value ?? '');
+  } else {
+    // claude-flow style
+    key = input.key ?? null;
+    content = input.content ?? '';
+    originalValue = undefined;
+  }
+
+  // category is an alias for namespace in claude-flow
+  const namespace = input.namespace ?? input.category ?? 'default';
+
+  return {
+    content,
+    key,
+    namespace,
+    type: input.type ?? 'episodic',
+    tags: input.tags ?? [],
+    importance: input.importance ?? 0.5,
+    ttl: input.ttl,
+    metadata: input.metadata ?? {},
+    source,
+    originalValue,
+  };
+}
+
+/**
+ * Format an ObservationPayload for ruflo-style retrieve response.
+ * Parses content back to original value if it was JSON.
+ */
+export function formatRufloRetrieve(payload: ObservationPayload): {
+  key: string;
+  value: unknown;
+  content: string;
+  namespace: string;
+  type: string;
+  tags: string[];
+  importance: number;
+  created_at: string;
+  accessed_at: string;
+  found: boolean;
+} {
+  let value: unknown = payload.content;
+  // Try to parse back to original type
+  try {
+    value = JSON.parse(payload.content);
+  } catch {
+    value = payload.content;
+  }
+
+  return {
+    key: payload.key ?? '',
+    value,
+    content: payload.content,
+    namespace: payload.namespace,
+    type: payload.type,
+    tags: payload.tags,
+    importance: payload.importance,
+    created_at: new Date(payload.created_at).toISOString(),
+    accessed_at: new Date(payload.accessed_at).toISOString(),
+    found: true,
+  };
+}
+
+/**
+ * Format an ObservationPayload for memory_search response (compatible with both schemas).
+ */
+export function formatMemorySearchResult(payload: ObservationPayload, relevanceScore: number): {
+  id: string;
+  key: string | null;
+  content: string;
+  value: unknown;
+  type: string;
+  namespace: string;
+  tags: string[];
+  importance: number;
+  relevance: number;
+  created_at: string;
+  accessed_at: string;
+  metadata: Record<string, unknown> | undefined;
+  source: MemorySource;
+} {
+  let value: unknown = payload.content;
+  // Try to parse back to original type for ruflo compat
+  try {
+    value = JSON.parse(payload.content);
+  } catch {
+    value = payload.content;
+  }
+
+  return {
+    id: payload.id,
+    key: payload.key,
+    content: payload.content,
+    value,
+    type: payload.type,
+    namespace: payload.namespace,
+    tags: payload.tags,
+    importance: payload.importance,
+    relevance: relevanceScore,
+    created_at: new Date(payload.created_at).toISOString(),
+    accessed_at: new Date(payload.accessed_at).toISOString(),
+    metadata: Object.keys(payload.metadata).length > 0 ? payload.metadata : undefined,
+    source: payload.source,
+  };
+}
+
+/**
+ * Build a Qdrant filter for memory_search based on the input parameters.
+ */
+export function buildMemorySearchFilter(params: {
+  namespace?: string;
+  category?: string;
+  type?: string;
+  tags?: string[];
+  source?: MemorySource;
+  includeStale?: boolean;
+}): Record<string, unknown> | undefined {
+  const must: Array<{ key: string; match: { value: unknown } }> = [];
+
+  // Namespace / category filter
+  const ns = params.namespace ?? params.category;
+  if (ns && ns !== 'default') {
+    must.push({ key: 'namespace', match: { value: ns } });
+  }
+
+  // Type filter
+  if (params.type && params.type !== 'all') {
+    must.push({ key: 'type', match: { value: params.type } });
+  }
+
+  // Source filter
+  if (params.source) {
+    must.push({ key: 'source', match: { value: params.source } });
+  }
+
+  // Tags filter - must match ALL tags (AND logic per architecture spec)
+  // Qdrant doesn't support AND on array elements natively, so we filter post-search
+  // The tags param is handled in post-processing.
+
+  return must.length > 0 ? { must } : undefined;
+}
+
+/**
+ * Post-filter results by tags (AND logic: result must have ALL requested tags).
+ */
+export function filterByTags<T extends { tags: string[] }>(results: T[], tags: string[]): T[] {
+  if (!tags || tags.length === 0) return results;
+  return results.filter(r => tags.every(tag => r.tags.includes(tag)));
+}

package/src/memory/observation-store.ts

@@ -0,0 +1,224 @@
+/**
+ * Observation store - CRUD for observations with symbol linking.
+ * Observations are stored in Qdrant (vector search) and FalkorDB (graph links).
+ * Implements the save_observation and related memory tool semantics from architecture.md.
+ *
+ * Note: ObservationRoute is defined inline here rather than imported from
+ * knowledge/routing-engine.ts (that module is not merged from forge-graph-rag;
+ * the pipeline server has its own knowledge/ modules).
+ */
+
+import { randomUUID } from 'crypto';
+import type { VectorStore, GraphStore } from '../storage/interfaces.js';
+import type { ObservationPayload } from '../util/types.js';
+import { embedText } from '../ingestion/embedder.js';
+import { LINK_OBSERVATION_TO_SYMBOL, UPSERT_OBSERVATION } from '../query/graph-queries.js';
+import { incrementObservationCount } from './session-tracker.js';
+import { logger } from '../util/logger.js';
+
+// Inline type — matches the shape from forge-graph-rag's routing-engine.ts
+export interface ObservationRoute {
+  writeVector: boolean;
+  writeGraphNode: boolean;
+  defaultNamespace: string;
+}
+
+export interface SaveObservationOptions {
+  content: string;
+  symbols?: string[];
+  repoId?: string;
+  importance?: number;
+  tags?: string[];
+  category?: string;
+  sessionId: string;
+  type?: ObservationPayload['type'];
+  source?: ObservationPayload['source'];
+  // Ruflo-style key-value support
+  key?: string;
+  namespace?: string;
+  value?: unknown;
+  metadata?: Record<string, unknown>;
+  ttl?: number;
+}
+
+export interface SaveObservationResult {
+  id: string;
+  stored: boolean;
+  linkedSymbols: string[];
+  unresolvedSymbols: string[];
+}
+
+/**
+ * Save an observation to Qdrant (vector) and optionally FalkorDB (graph node + symbol links).
+ * The optional `route` parameter controls whether graph operations are executed.
+ * When `route.writeGraphNode` is false (external repo), only Qdrant is written.
+ */
+export async function saveObservation(
+  opts: SaveObservationOptions,
+  vectorStore: VectorStore,
+  graphStore: GraphStore,
+  route?: ObservationRoute
+): Promise<SaveObservationResult> {
+  // Default route: write everywhere (backward-compatible when caller doesn't supply route)
+  const effectiveRoute: ObservationRoute = route ?? {
+    writeVector: true,
+    writeGraphNode: true,
+    defaultNamespace: 'default',
+  };
+  const {
+    content,
+    symbols = [],
+    repoId = null,
+    importance = 0.5,
+    tags = [],
+    category = null,
+    sessionId,
+    type = 'episodic',
+    source = 'forge',
+    key = null,
+    namespace = 'default',
+    value,
+    metadata = {},
+    ttl,
+  } = opts;
+
+  const id = randomUUID();
+  const now = Date.now();
+
+  // Build the content string - for ruflo-style key/value, serialize value
+  let finalContent = content;
+  if (!finalContent && value !== undefined) {
+    finalContent = typeof value === 'string' ? value : JSON.stringify(value);
+  }
+  if (!finalContent) {
+    finalContent = key ? `[key: ${key}]` : '[empty observation]';
+  }
+
+  // Build the observation payload (stored in Qdrant)
+  const payload: ObservationPayload = {
+    id,
+    repo_id: repoId ?? null,
+    session_id: sessionId,
+    content: finalContent,
+    type,
+    category,
+    tags,
+    importance,
+    created_at: now,
+    accessed_at: now,
+    access_count: 0,
+    is_stale: false,
+    linked_symbols: [],
+    source,
+    metadata: ttl ? { ...metadata, ttl, expires_at: now + ttl * 1000 } : metadata,
+    key,
+    namespace,
+  };
+
+  // Embed the content
+  let vector: number[];
+  try {
+    vector = await embedText(finalContent);
+  } catch (err) {
+    logger.warn('Failed to embed observation content', { error: String(err) });
+    // Fallback: zero vector (won't match anything but stores the payload)
+    vector = new Array(384).fill(0);
+  }
+
+  // Store in Qdrant
+  try {
+    await vectorStore.upsertObservation(id, vector, payload);
+  } catch (err) {
+    logger.error('Failed to store observation in Qdrant', { id, error: String(err) });
+    return { id, stored: false, linkedSymbols: [], unresolvedSymbols: symbols };
+  }
+
+  // Store Observation node in FalkorDB and link to symbols (owned repos only)
+  const linkedSymbols: string[] = [];
+  const unresolvedSymbols: string[] = [];
+
+  if (effectiveRoute.writeGraphNode) {
+    try {
+      await graphStore.query(UPSERT_OBSERVATION(id, finalContent, sessionId, importance));
+
+      // Attempt to link each symbol
+      for (const symbolName of symbols) {
+        try {
+          const effectiveRepoId = repoId ?? '';
+          await graphStore.query(LINK_OBSERVATION_TO_SYMBOL(id, symbolName, effectiveRepoId));
+          linkedSymbols.push(symbolName);
+          logger.debug('Linked observation to symbol', { observationId: id, symbol: symbolName });
+        } catch {
+          unresolvedSymbols.push(symbolName);
+          logger.debug('Symbol not found in graph for observation link', { symbol: symbolName });
+        }
+      }
+
+      // Update payload with resolved symbols
+      if (linkedSymbols.length > 0) {
+        await vectorStore.updateObservation(id, { linked_symbols: linkedSymbols });
+      }
+    } catch (err) {
+      // Graph operations failing is non-fatal - Qdrant is the source of truth
+      logger.warn('FalkorDB observation node creation failed (non-fatal)', { id, error: String(err) });
+      unresolvedSymbols.push(...symbols);
+    }
+  } else {
+    // External repo: all symbols are unresolved (no graph to link against)
+    unresolvedSymbols.push(...symbols);
+    logger.debug('External repo observation — skipping graph node creation', { id, repoId });
+  }
+
+  // Increment the in-memory session observation count
+  incrementObservationCount();
+
+  logger.info('Observation saved', {
+    id,
+    contentLength: finalContent.length,
+    linkedSymbols: linkedSymbols.length,
+    unresolvedSymbols: unresolvedSymbols.length,
+  });
+
+  return { id, stored: true, linkedSymbols, unresolvedSymbols };
+}
+
+/**
+ * Retrieve an observation by ID.
+ */
+export async function getObservation(
+  id: string,
+  vectorStore: VectorStore
+): Promise<ObservationPayload | null> {
+  return vectorStore.getObservation(id);
+}
+
+/**
+ * Update access tracking for an observation (access_count, accessed_at).
+ */
+export async function touchObservation(
+  id: string,
+  vectorStore: VectorStore
+): Promise<void> {
+  try {
+    const obs = await vectorStore.getObservation(id);
+    if (obs) {
+      await vectorStore.updateObservation(id, {
+        accessed_at: Date.now(),
+        access_count: (obs.access_count ?? 0) + 1,
+      });
+    }
+  } catch (err) {
+    logger.debug('Failed to update observation access tracking', { id, error: String(err) });
+  }
+}
+
+/**
+ * Check if an observation has expired (TTL expired).
+ */
+export function isExpired(payload: ObservationPayload): boolean {
+  const expiresAt = payload.metadata?.expires_at;
+  if (typeof expiresAt === 'number') {
+    return Date.now() > expiresAt;
+  }
+  return false;
+}