kernl 0.6.1 → 0.6.3

package/src/memory/handle.ts

@@ -0,0 +1,189 @@
+/**
+ * Memory index handle with lazy initialization.
+ */
+
+import {
+  planQuery,
+  type SearchIndex,
+  type IndexHandle,
+  type QueryInput,
+  type SearchHit,
+  type DocumentPatch,
+  type UpsertResult,
+  type PatchResult,
+  type DeleteResult,
+  type FieldSchema,
+  type SearchCapabilities,
+  type UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import { getAdapterCodecs, type AdapterCodecs } from "./codecs";
+import type { IndexMemoryRecord } from "./types";
+
+/**
+ * Configuration for MemoryIndexHandle.
+ */
+export interface MemoryIndexHandleConfig {
+  /**
+   * The underlying search index backend.
+   */
+  index: SearchIndex;
+
+  /**
+   * Index identifier (e.g., "kernl_memories_index").
+   */
+  indexId: string;
+
+  /**
+   * Field schema for the memory index.
+   */
+  schema: Record<string, FieldSchema>;
+
+  /**
+   * Backend-specific options passed to SearchIndex.createIndex().
+   */
+  providerOptions?: Record<string, unknown>;
+}
+
+/**
+ * Domain-aware index handle for memory records with lazy initialization.
+ *
+ * - Wraps a SearchIndex and ensures the memory index is created before any operation.
+ * - Normalizes the idiosyncrasies of search adapters (capabilities, weird rules, ... - e.g. tpuf requires vector fields named
+ * literally 'vector' - dumb shit like this..)
+ */
+export class MemoryIndexHandle implements IndexHandle<IndexMemoryRecord> {
+  readonly id: string;
+
+  private readonly index: SearchIndex;
+  private readonly schema: Record<string, FieldSchema>;
+  private readonly caps: SearchCapabilities;
+  private readonly codecs: AdapterCodecs;
+  private readonly providerOptions?: Record<string, unknown>;
+
+  private initPromise: Promise<void> | null = null;
+
+  constructor(config: MemoryIndexHandleConfig) {
+    this.index = config.index;
+    this.id = config.indexId;
+    this.schema = config.schema;
+    this.caps = this.index.capabilities();
+    this.codecs = getAdapterCodecs(this.index.id);
+    this.providerOptions = config.providerOptions;
+  }
+
+  /**
+   * Ensure memory index exists (lazy initialization).
+   *
+   * Safe to call multiple times - initialization only runs once.
+   */
+  private async ensureInit(): Promise<void> {
+    if (!this.initPromise) {
+      this.initPromise = this.createIndex().catch((err) => {
+        this.initPromise = null;
+        throw err;
+      });
+    }
+    await this.initPromise;
+  }
+
+  /**
+   * Create the memory index if it doesn't exist.
+   */
+  private async createIndex(): Promise<void> {
+    try {
+      await this.index.createIndex({
+        id: this.id,
+        schema: this.codecs.schema.encode(this.schema),
+        providerOptions: this.providerOptions,
+      });
+    } catch (err: any) {
+      // (TODO): we should probably enforce a stricter contract w/ tighter error types
+      //
+      // Ignore "already exists" errors
+      if (
+        err.message?.includes("already exists") ||
+        err.message?.includes("AlreadyExists")
+      ) {
+        return;
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * Search for memory records matching the query.
+   *
+   * Adapts the query to backend capabilities, degrading gracefully
+   * when hybrid or multi-signal queries aren't supported.
+   */
+  async query(input: QueryInput): Promise<SearchHit<IndexMemoryRecord>[]> {
+    const { input: planned } = planQuery(input, this.caps);
+    const q = this.codecs.query.encode(planned);
+    const handle = await this.handle();
+    const hits = await handle.query(q);
+
+    // decode hits back to IndexMemoryRecord format
+    return hits.map((hit) => ({
+      ...hit,
+      document: hit.document
+        ? this.codecs.doc.decode(hit.document as UnknownDocument)
+        : undefined,
+    }));
+  }
+
+  /**
+   * Insert or update memory records in the index.
+   */
+  async upsert(
+    docs: IndexMemoryRecord | IndexMemoryRecord[],
+  ): Promise<UpsertResult> {
+    const arr = Array.isArray(docs) ? docs : [docs];
+    const encoded = arr.map((doc) => this.codecs.doc.encode(doc));
+    const handle = await this.handle();
+    return handle.upsert(encoded);
+  }
+
+  /**
+   * Partially update memory records without re-indexing vectors.
+   *
+   * Note: Patches don't include vector fields so we cast directly.
+   * Metadata field type mismatch (JSONObject vs FieldValue) is handled at runtime.
+   */
+  async patch(
+    patches:
+      | DocumentPatch<IndexMemoryRecord>
+      | DocumentPatch<IndexMemoryRecord>[],
+  ): Promise<PatchResult> {
+    const handle = await this.handle();
+    return handle.patch(patches as DocumentPatch<UnknownDocument>[]);
+  }
+
+  /**
+   * Remove memory records from the index.
+   */
+  async delete(ids: string | string[]): Promise<DeleteResult> {
+    const handle = await this.handle();
+    return handle.delete(ids);
+  }
+
+  /**
+   * Add a new field to the index schema.
+   *
+   * @throws Always throws - dynamic schema modification not supported
+   */
+  async addField(_field: string, _schema: FieldSchema): Promise<void> {
+    throw new Error("addField not supported for MemoryIndexHandle");
+  }
+
+  /**
+   * Get an initialized underlying index handle.
+   *
+   * Returns a handle typed for UnknownDocument since we encode/decode
+   * through the adapter codecs for backend-specific field mapping.
+   */
+  private async handle(): Promise<IndexHandle<UnknownDocument>> {
+    await this.ensureInit();
+    return this.index.index<UnknownDocument>(this.id);
+  }
+}

package/src/memory/index.ts

@@ -0,0 +1,49 @@
+/**
+ * Memory module.
+ */
+
+export { Memory } from "./memory";
+export { MemoryByteEncoder } from "./encoder";
+export { buildMemoryIndexSchema } from "./schema";
+export { MemoryIndexHandle } from "./handle";
+export type { MemoryIndexHandleConfig } from "./handle";
+
+export type {
+  // Byte types
+  TextByte,
+  ImageByte,
+  AudioByte,
+  VideoByte,
+  MemoryByte,
+  IndexableByte,
+  MemoryByteCodec,
+  // Core types
+  MemoryScope,
+  MemoryKind,
+  NewMemory,
+  AgentMemoryCreate,
+  MemoryConfig,
+  MemoryReindexParams,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemoryFilter,
+  MemoryListOptions,
+  MemorySearchQuery,
+  IndexMemoryRecord,
+  IndexMemoryRecordPatch,
+  WorkingMemorySnapshot,
+  ShortTermMemorySnapshot,
+} from "./types";
+
+export type { MemoryStore } from "./store";
+
+export type {
+  MemoryIndexBase,
+  MemorySearchIndex,
+  MemoryGraphIndex,
+  MemoryArchiveIndex,
+  GraphTraversalQuery,
+  GraphTraversalResult,
+  ArchiveQuery,
+  ArchiveResult,
+} from "./indexes";

package/src/memory/indexes.ts

@@ -0,0 +1,108 @@
+/**
+ * Memory index interfaces.
+ *
+ * Indexes are projections of the primary store (DB) that enable
+ * specialized query patterns (vector search, graph traversal, archival).
+ */
+
+import type { SearchHit } from "@kernl-sdk/retrieval";
+
+import type {
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemorySearchQuery,
+  IndexMemoryRecord,
+} from "./types";
+
+/**
+ * Base interface for memory indexes.
+ *
+ * All indexes share common lifecycle operations (index, patch, delete)
+ * but differ in their query interface.
+ */
+export interface MemoryIndexBase<TQuery, TResult> {
+  readonly id: string /* provider id - "pgvector" | "turbopuffer", ... */;
+
+  /**
+   * Query the index.
+   */
+  query(query: TQuery): Promise<TResult>;
+
+  /**
+   * Index one or more memory records (idempotent upsert).
+   */
+  index(memories: MemoryRecord | MemoryRecord[]): Promise<void>;
+
+  /**
+   * Partially update one or more records' projections.
+   */
+  update(updates: MemoryRecordUpdate | MemoryRecordUpdate[]): Promise<void>;
+
+  /**
+   * Remove one or more records from this index (DB row remains).
+   */
+  delete(ids: string | string[]): Promise<void>;
+
+  /**
+   * Index warming (optional).
+   */
+  warm(index: string): Promise<void>;
+}
+
+/**
+ * Memory search index - vector/semantic search over memories.
+ */
+export interface MemorySearchIndex
+  extends MemoryIndexBase<MemorySearchQuery, SearchHit<IndexMemoryRecord>[]> {}
+
+/**
+ * Graph traversal query (stub).
+ */
+export interface GraphTraversalQuery {
+  // TODO: define graph query params
+  depth?: number;
+}
+
+/**
+ * Graph traversal result (stub).
+ */
+export interface GraphTraversalResult {
+  // TODO: define graph result shape
+  nodes: Array<{ id: string; record?: MemoryRecord }>;
+  edges: Array<{ from: string; to: string; relation: string }>;
+}
+
+/**
+ * Memory graph index - relationship/graph traversal over memories (stub).
+ */
+export interface MemoryGraphIndex
+  extends MemoryIndexBase<GraphTraversalQuery, GraphTraversalResult> {
+  /**
+   * Explicit traversal API (alias for query).
+   */
+  traverse(query: GraphTraversalQuery): Promise<GraphTraversalResult>;
+}
+
+/**
+ * Archive query (stub).
+ */
+export interface ArchiveQuery {
+  // TODO: define archive query params
+  before?: number;
+  collections?: string[];
+}
+
+/**
+ * Archive result (stub).
+ */
+export interface ArchiveResult {
+  // TODO: define archive result shape
+  id: string;
+  uri: string;
+}
+
+/**
+ * Memory archive index - cold storage/archival backend (stub).
+ */
+export interface MemoryArchiveIndex
+  extends MemoryIndexBase<ArchiveQuery, ArchiveResult[]> {}

package/src/memory/memory.ts

@@ -0,0 +1,143 @@
+import type { IndexHandle, SearchHit } from "@kernl-sdk/retrieval";
+import type { AsyncCodec } from "@kernl-sdk/shared/lib";
+
+import type { MemoryStore } from "./store";
+import type {
+  NewMemory,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  MemoryScope,
+  MemoryConfig,
+  MemorySearchQuery,
+  MemoryByteCodec,
+  IndexMemoryRecord,
+  WorkingMemorySnapshot,
+  ShortTermMemorySnapshot,
+  MemoryReindexParams,
+} from "./types";
+import { MEMORY_FILTER, PATCH_CODEC, recordCodec } from "./codecs";
+
+/**
+ * Memory is the primary memory abstraction for agents.
+ *
+ * Sits above storage/index layers + owns cognitive policy, eviction/TTL, consolidation.
+ *
+ *  - L1 / wmem: active working set exposed to the model
+ *  - L2 / smem: bounded recent context with a TTL
+ *  - L3 / lmem: durable, structured long-term store
+ *
+ * Delegates persistence to storage adapters and optional indexes as
+ * _projections_ of the primary memory store.
+ */
+export class Memory {
+  private readonly store: MemoryStore;
+  private readonly _search: IndexHandle<IndexMemoryRecord> | null;
+
+  private readonly encoder: MemoryByteCodec;
+  private readonly rcodec: AsyncCodec<MemoryRecord, IndexMemoryRecord>;
+
+  constructor(config: MemoryConfig) {
+    this.store = config.store;
+    this._search = config.search ?? null;
+
+    // TODO: default encoder using text-embedding-3-small
+    this.encoder = config.encoder;
+    this.rcodec = recordCodec(config.encoder);
+  }
+
+  /**
+   * Create a new memory record.
+   * Writes to primary store first, then indexes if configured.
+   */
+  async create(memory: NewMemory): Promise<MemoryRecord> {
+    const record = await this.store.create(memory);
+
+    // index into search if avail
+    if (this._search) {
+      const indexed = await this.rcodec.encode(record);
+      await this._search.upsert(indexed);
+    }
+
+    return record;
+  }
+
+  /**
+   * Update an existing memory record.
+   * Updates primary store, then re-indexes or patches search index.
+   */
+  async update(update: MemoryRecordUpdate): Promise<MemoryRecord> {
+    const record = await this.store.update(update.id, update);
+    if (!this._search) return record;
+
+    if (update.content) {
+      const indexed = await this.rcodec.encode(record); // content changed → full re-index with new embeddings
+      await this._search.upsert(indexed);
+    } else {
+      const patch = PATCH_CODEC.encode(update); // metadata only → cheap patch
+      await this._search.patch(patch);
+    }
+
+    return record;
+  }
+
+  /**
+   * Semantic/metadata search across memories.
+   *
+   * Sends rich query with both text and vector - the index handle
+   * adapts based on backend capabilities (e.g. drops text for pgvector).
+   */
+  async search(q: MemorySearchQuery): Promise<SearchHit<IndexMemoryRecord>[]> {
+    if (!this._search) {
+      throw new Error("search index not configured");
+    }
+
+    const tvec = await this.encoder.embed(q.query);
+
+    return this._search.query({
+      query: [{ text: q.query, tvec }],
+      filter: q.filter ? MEMORY_FILTER.encode(q.filter) : undefined,
+      topK: q.limit ?? 20,
+    });
+  }
+
+  /**
+   * Repair indexing for a memory without modifying the DB row.
+   */
+  async reindex(params: MemoryReindexParams): Promise<void> {
+    const record = await this.store.get(params.id);
+    if (!record) {
+      throw new Error(`memory not found: ${params.id}`);
+    }
+
+    const indexes = params.indexes ?? ["search", "graph", "archive"];
+
+    if (indexes.includes("search") && this._search) {
+      const indexed = await this.rcodec.encode(record);
+      await this._search.upsert(indexed);
+    }
+  }
+
+  /* --- (TODO): unclear what the shape of these should be.. --- */
+
+  /**
+   * Load working memory (L1) - wmem-pinned memories for the scope.
+   */
+  async loadWorkingMemory(scope: MemoryScope): Promise<WorkingMemorySnapshot> {
+    const records = await this.store.list({
+      filter: { scope, wmem: true },
+    });
+    return { scope, records };
+  }
+
+  /**
+   * Load short-term memory (L2) - active smem for the scope.
+   */
+  async loadShortTermMemory(
+    scope: MemoryScope,
+  ): Promise<ShortTermMemorySnapshot> {
+    const records = await this.store.list({
+      filter: { scope, smem: true },
+    });
+    return { scope, records };
+  }
+}

package/src/memory/schema.ts

@@ -0,0 +1,142 @@
+/**
+ * Memory index schema builder.
+ */
+
+import type { FieldSchema } from "@kernl-sdk/retrieval";
+
+/**
+ * Options for building memory index schema.
+ */
+export interface MemoryIndexSchemaOptions {
+  /**
+   * Vector dimensions for embeddings.
+   * @default 1536 (OpenAI text-embedding-3-small)
+   */
+  dimensions?: number;
+
+  /**
+   * Similarity metric for vector search.
+   * @default "cosine"
+   */
+  similarity?: "cosine" | "euclidean" | "dot_product";
+}
+
+/**
+ * Build a canonical memory index schema for vector search.
+ *
+ * Returns a schema Record that can be used with SearchIndex.createIndex().
+ *
+ * Schema includes all fields from IndexMemoryRecord:
+ * - id: primary key
+ * - namespace, entityId, agentId: scope fields (filterable, nullable)
+ * - kind, collection: memory attributes (filterable)
+ * - timestamp, createdAt, updatedAt: timestamps (filterable + sortable)
+ * - text: content text (full-text search)
+ * - tvec, ivec, avec, vvec: vector embeddings for text/image/audio/video
+ * - metadata: structured metadata (object)
+ *
+ * @example
+ * ```ts
+ * const schema = buildMemoryIndexSchema({ dimensions: 1536 });
+ * const handle = new MemoryIndexHandle({ index: vector, indexId: "kernl_memories", schema });
+ * ```
+ */
+export function buildMemoryIndexSchema(
+  options: MemoryIndexSchemaOptions = {},
+): Record<string, FieldSchema> {
+  const dimensions = options.dimensions ?? 1536;
+  const similarity = options.similarity ?? "cosine";
+
+  const schema: Record<string, FieldSchema> = {
+    id: {
+      type: "string",
+      pk: true,
+      sortable: true,
+    },
+
+    // scope fields (filterable, nullable)
+    namespace: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+    entityId: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+    agentId: {
+      type: "string",
+      filterable: true,
+      optional: true,
+    },
+
+    // memory attributes (filterable)
+    kind: {
+      type: "string",
+      filterable: true,
+    },
+    collection: {
+      type: "string",
+      filterable: true,
+    },
+
+    // timestamps (filterable + sortable) - store as bigint to safely handle
+    // millisecond Unix epoch values without overflow in SQL backends.
+    timestamp: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+    createdAt: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+    updatedAt: {
+      type: "bigint",
+      filterable: true,
+      sortable: true,
+    },
+
+    // content fields
+    text: {
+      type: "string",
+      fts: true,
+      optional: true,
+    },
+
+    // vector fields for different modalities
+    tvec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    ivec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    avec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+    vvec: {
+      type: "vector",
+      dimensions,
+      similarity,
+      optional: true,
+    },
+
+    metadata: {
+      type: "object",
+      optional: true,
+    },
+  };
+
+  return schema;
+}

package/src/memory/store.ts

@@ -0,0 +1,47 @@
+/**
+ * Memory store interface.
+ */
+
+import type {
+  MemoryRecord,
+  NewMemory,
+  MemoryRecordUpdate,
+  MemoryListOptions,
+} from "./types";
+
+/**
+ * Memory persistence store.
+ *
+ * Follows the same pattern as ThreadStore - simple CRUD operations.
+ */
+export interface MemoryStore {
+  /**
+   * Get a memory by ID.
+   */
+  get(id: string): Promise<MemoryRecord | null>;
+
+  /**
+   * List memories matching the filter.
+   */
+  list(options?: MemoryListOptions): Promise<MemoryRecord[]>;
+
+  /**
+   * Create a new memory.
+   */
+  create(memory: NewMemory): Promise<MemoryRecord>;
+
+  /**
+   * Update an existing memory.
+   */
+  update(id: string, patch: MemoryRecordUpdate): Promise<MemoryRecord>;
+
+  /**
+   * Delete a memory.
+   */
+  delete(id: string): Promise<void>;
+
+  /**
+   * Delete multiple memories.
+   */
+  mdelete(ids: string[]): Promise<void>;
+}