kernl 0.6.2 → 0.6.3

package/src/memory/index.ts

@@ -0,0 +1,49 @@
+/**
+ * Memory module.
+ */
+
+export { Memory } from "./memory";
+export { MemoryByteEncoder } from "./encoder";
+export { buildMemoryIndexSchema } from "./schema";
+export { MemoryIndexHandle } from "./handle";
+export type { MemoryIndexHandleConfig } from "./handle";
+
+export type {
+  // Byte types
+  TextByte,
+  ImageByte,
+  AudioByte,
+  VideoByte,
+  MemoryByte,
+  IndexableByte,
+  MemoryByteCodec,
+  // Core types
+  MemoryScope,
+  MemoryKind,
+  NewMemory,
+  AgentMemoryCreate,
+  MemoryConfig,
+  MemoryReindexParams,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemoryFilter,
+  MemoryListOptions,
+  MemorySearchQuery,
+  IndexMemoryRecord,
+  IndexMemoryRecordPatch,
+  WorkingMemorySnapshot,
+  ShortTermMemorySnapshot,
+} from "./types";
+
+export type { MemoryStore } from "./store";
+
+export type {
+  MemoryIndexBase,
+  MemorySearchIndex,
+  MemoryGraphIndex,
+  MemoryArchiveIndex,
+  GraphTraversalQuery,
+  GraphTraversalResult,
+  ArchiveQuery,
+  ArchiveResult,
+} from "./indexes";

package/src/memory/indexes.ts

@@ -0,0 +1,108 @@
+/**
+ * Memory index interfaces.
+ *
+ * Indexes are projections of the primary store (DB) that enable
+ * specialized query patterns (vector search, graph traversal, archival).
+ */
+
+import type { SearchHit } from "@kernl-sdk/retrieval";
+
+import type {
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemorySearchQuery,
+  IndexMemoryRecord,
+} from "./types";
+
+/**
+ * Base interface for memory indexes.
+ *
+ * All indexes share common lifecycle operations (index, patch, delete)
+ * but differ in their query interface.
+ */
+export interface MemoryIndexBase<TQuery, TResult> {
+  readonly id: string /* provider id - "pgvector" | "turbopuffer", ... */;
+
+  /**
+   * Query the index.
+   */
+  query(query: TQuery): Promise<TResult>;
+
+  /**
+   * Index one or more memory records (idempotent upsert).
+   */
+  index(memories: MemoryRecord | MemoryRecord[]): Promise<void>;
+
+  /**
+   * Partially update one or more records' projections.
+   */
+  update(updates: MemoryRecordUpdate | MemoryRecordUpdate[]): Promise<void>;
+
+  /**
+   * Remove one or more records from this index (DB row remains).
+   */
+  delete(ids: string | string[]): Promise<void>;
+
+  /**
+   * Index warming (optional).
+   */
+  warm(index: string): Promise<void>;
+}
+
+/**
+ * Memory search index - vector/semantic search over memories.
+ */
+export interface MemorySearchIndex
+  extends MemoryIndexBase<MemorySearchQuery, SearchHit<IndexMemoryRecord>[]> {}
+
+/**
+ * Graph traversal query (stub).
+ */
+export interface GraphTraversalQuery {
+  // TODO: define graph query params
+  depth?: number;
+}
+
+/**
+ * Graph traversal result (stub).
+ */
+export interface GraphTraversalResult {
+  // TODO: define graph result shape
+  nodes: Array<{ id: string; record?: MemoryRecord }>;
+  edges: Array<{ from: string; to: string; relation: string }>;
+}
+
+/**
+ * Memory graph index - relationship/graph traversal over memories (stub).
+ */
+export interface MemoryGraphIndex
+  extends MemoryIndexBase<GraphTraversalQuery, GraphTraversalResult> {
+  /**
+   * Explicit traversal API (alias for query).
+   */
+  traverse(query: GraphTraversalQuery): Promise<GraphTraversalResult>;
+}
+
+/**
+ * Archive query (stub).
+ */
+export interface ArchiveQuery {
+  // TODO: define archive query params
+  before?: number;
+  collections?: string[];
+}
+
+/**
+ * Archive result (stub).
+ */
+export interface ArchiveResult {
+  // TODO: define archive result shape
+  id: string;
+  uri: string;
+}
+
+/**
+ * Memory archive index - cold storage/archival backend (stub).
+ */
+export interface MemoryArchiveIndex
+  extends MemoryIndexBase<ArchiveQuery, ArchiveResult[]> {}

package/src/memory/memory.ts

@@ -0,0 +1,143 @@
+import type { IndexHandle, SearchHit } from "@kernl-sdk/retrieval";
+import type { AsyncCodec } from "@kernl-sdk/shared/lib";
+
+import type { MemoryStore } from "./store";
+import type {
+  NewMemory,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemoryScope,
+  MemoryConfig,
+  MemorySearchQuery,
+  MemoryByteCodec,
+  IndexMemoryRecord,
+  WorkingMemorySnapshot,
+  ShortTermMemorySnapshot,
+  MemoryReindexParams,
+} from "./types";
+import { MEMORY_FILTER, PATCH_CODEC, recordCodec } from "./codecs";
+
+/**
+ * Memory is the primary memory abstraction for agents.
+ *
+ * Sits above storage/index layers + owns cognitive policy, eviction/TTL, consolidation.
+ *
+ *  - L1 / wmem: active working set exposed to the model
+ *  - L2 / smem: bounded recent context with a TTL
+ *  - L3 / lmem: durable, structured long-term store
+ *
+ * Delegates persistence to storage adapters and optional indexes as
+ * _projections_ of the primary memory store.
+ */
+export class Memory {
+  private readonly store: MemoryStore;
+  private readonly _search: IndexHandle<IndexMemoryRecord> | null;
+
+  private readonly encoder: MemoryByteCodec;
+  private readonly rcodec: AsyncCodec<MemoryRecord, IndexMemoryRecord>;
+
+  constructor(config: MemoryConfig) {
+    this.store = config.store;
+    this._search = config.search ?? null;
+
+    // TODO: default encoder using text-embedding-3-small
+    this.encoder = config.encoder;
+    this.rcodec = recordCodec(config.encoder);
+  }
+
+  /**
+   * Create a new memory record.
+   * Writes to primary store first, then indexes if configured.
+   */
+  async create(memory: NewMemory): Promise<MemoryRecord> {
+    const record = await this.store.create(memory);
+
+    // index into search if avail
+    if (this._search) {
+      const indexed = await this.rcodec.encode(record);
+      await this._search.upsert(indexed);
+    }
+
+    return record;
+  }
+
+  /**
+   * Update an existing memory record.
+   * Updates primary store, then re-indexes or patches search index.
+   */
+  async update(update: MemoryRecordUpdate): Promise<MemoryRecord> {
+    const record = await this.store.update(update.id, update);
+    if (!this._search) return record;
+
+    if (update.content) {
+      const indexed = await this.rcodec.encode(record); // content changed → full re-index with new embeddings
+      await this._search.upsert(indexed);
+    } else {
+      const patch = PATCH_CODEC.encode(update); // metadata only → cheap patch
+      await this._search.patch(patch);
+    }
+
+    return record;
+  }
+
+  /**
+   * Semantic/metadata search across memories.
+   *
+   * Sends rich query with both text and vector - the index handle
+   * adapts based on backend capabilities (e.g. drops text for pgvector).
+   */
+  async search(q: MemorySearchQuery): Promise<SearchHit<IndexMemoryRecord>[]> {
+    if (!this._search) {
+      throw new Error("search index not configured");
+    }
+
+    const tvec = await this.encoder.embed(q.query);
+
+    return this._search.query({
+      query: [{ text: q.query, tvec }],
+      filter: q.filter ? MEMORY_FILTER.encode(q.filter) : undefined,
+      topK: q.limit ?? 20,
+    });
+  }
+
+  /**
+   * Repair indexing for a memory without modifying the DB row.
+   */
+  async reindex(params: MemoryReindexParams): Promise<void> {
+    const record = await this.store.get(params.id);
+    if (!record) {
+      throw new Error(`memory not found: ${params.id}`);
+    }
+
+    const indexes = params.indexes ?? ["search", "graph", "archive"];
+
+    if (indexes.includes("search") && this._search) {
+      const indexed = await this.rcodec.encode(record);
+      await this._search.upsert(indexed);
+    }
+  }
+
+  /* --- (TODO): unclear what the shape of these should be.. --- */
+
+  /**
+   * Load working memory (L1) - wmem-pinned memories for the scope.
+   */
+  async loadWorkingMemory(scope: MemoryScope): Promise<WorkingMemorySnapshot> {
+    const records = await this.store.list({
+      filter: { scope, wmem: true },
+    });
+    return { scope, records };
+  }
+
+  /**
+   * Load short-term memory (L2) - active smem for the scope.
+   */
+  async loadShortTermMemory(
+    scope: MemoryScope,
+  ): Promise<ShortTermMemorySnapshot> {
+    const records = await this.store.list({
+      filter: { scope, smem: true },
+    });
+    return { scope, records };
+  }
+}

package/src/memory/schema.ts

@@ -0,0 +1,142 @@
+/**
+ * Memory index schema builder.
+ */
+
+import type { FieldSchema } from "@kernl-sdk/retrieval";
+
+/**
+ * Options for building memory index schema.
+ */
+export interface MemoryIndexSchemaOptions {
+  /**
+   * Vector dimensions for embeddings.
+   * @default 1536 (OpenAI text-embedding-3-small)
+   */
+  dimensions?: number;
+
+  /**
+   * Similarity metric for vector search.
+   * @default "cosine"
+   */
+  similarity?: "cosine" | "euclidean" | "dot_product";
+}
+
+/**
+ * Build a canonical memory index schema for vector search.
+ *
+ * Returns a schema Record that can be used with SearchIndex.createIndex().
+ *
+ * Schema includes all fields from IndexMemoryRecord:
+ * - id: primary key
+ * - namespace, entityId, agentId: scope fields (filterable, nullable)
+ * - kind, collection: memory attributes (filterable)
+ * - timestamp, createdAt, updatedAt: timestamps (filterable + sortable)
+ * - text: content text (full-text search)
+ * - tvec, ivec, avec, vvec: vector embeddings for text/image/audio/video
+ * - metadata: structured metadata (object)
+ *
+ * @example
+ * ```ts
+ * const schema = buildMemoryIndexSchema({ dimensions: 1536 });
+ * const handle = new MemoryIndexHandle({ index: vector, indexId: "kernl_memories", schema });
+ * ```
+ */
+export function buildMemoryIndexSchema(
+  options: MemoryIndexSchemaOptions = {},
+): Record<string, FieldSchema> {
+  const dimensions = options.dimensions ?? 1536;
+  const similarity = options.similarity ?? "cosine";
+
+  const schema: Record<string, FieldSchema> = {
+    id: {
+      type: "string",
+      pk: true,
+      sortable: true,
+    },
+
+    // scope fields (filterable, nullable)
+    namespace: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+    entityId: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+    agentId: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+
+    // memory attributes (filterable)
+    kind: {
+      type: "string",
+      filterable: true,
+    },
+    collection: {
+      type: "string",
+      filterable: true,
+    },
+
+    // timestamps (filterable + sortable) - store as bigint to safely handle
+    // millisecond Unix epoch values without overflow in SQL backends.
+    timestamp: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+    createdAt: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+    updatedAt: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+
+    // content fields
+    text: {
+      type: "string",
+      fts: true,
+      optional: true,
+    },
+
+    // vector fields for different modalities
+    tvec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    ivec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    avec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    vvec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+
+    metadata: {
+      type: "object",
+      optional: true,
+    },
+  };
+
+  return schema;
+}

package/src/memory/store.ts

@@ -0,0 +1,47 @@
+/**
+ * Memory store interface.
+ */
+
+import type {
+  MemoryRecord,
+  NewMemory,
+  MemoryRecordUpdate,
+  MemoryListOptions,
+} from "./types";
+
+/**
+ * Memory persistence store.
+ *
+ * Follows the same pattern as ThreadStore - simple CRUD operations.
+ */
+export interface MemoryStore {
+  /**
+   * Get a memory by ID.
+   */
+  get(id: string): Promise<MemoryRecord | null>;
+
+  /**
+   * List memories matching the filter.
+   */
+  list(options?: MemoryListOptions): Promise<MemoryRecord[]>;
+
+  /**
+   * Create a new memory.
+   */
+  create(memory: NewMemory): Promise<MemoryRecord>;
+
+  /**
+   * Update an existing memory.
+   */
+  update(id: string, patch: MemoryRecordUpdate): Promise<MemoryRecord>;
+
+  /**
+   * Delete a memory.
+   */
+  delete(id: string): Promise<void>;
+
+  /**
+   * Delete multiple memories.
+   */
+  mdelete(ids: string[]): Promise<void>;
+}