kernl 0.6.2 → 0.6.3

package/src/kernl/types.ts

@@ -0,0 +1,106 @@
+import { LanguageModel, EmbeddingModel } from "@kernl-sdk/protocol";
+import { SearchIndex } from "@kernl-sdk/retrieval";
+
+import { Agent } from "@/agent";
+import { KernlStorage } from "@/storage";
+
+/**
+ * Storage configuration for Kernl.
+ */
+export interface StorageOptions {
+  /**
+   * Relational database storage (threads, tasks, traces).
+   */
+  db?: KernlStorage;
+
+  /**
+   * Vector search index for semantic memory search.
+   * Supports pgvector, Turbopuffer, etc.
+   */
+  vector?: SearchIndex;
+
+  // Future storage layers (deferred):
+  // blob?: BlobStore;
+  // lake?: DataLake;
+}
+
+/**
+ * Memory system configuration.
+ */
+export interface MemoryOptions {
+  /**
+   * Embedding model for memory encoding.
+   *
+   * Can be:
+   * - A string like "openai/text-embedding-3-small" (resolved via provider registry)
+   * - An EmbeddingModel instance
+   *
+   * @default "openai/text-embedding-3-small"
+   */
+  embeddingModel?: string | EmbeddingModel<string>;
+
+  /**
+   * Logical index ID used by the search backend.
+   * - For pgvector: becomes the table name (with schema from indexProviderOptions)
+   * - For Turbopuffer: becomes the namespace
+   * @default "kernl_memories_index"
+   */
+  indexId?: string;
+
+  /**
+   * Backend-specific options passed to SearchIndex.createIndex().
+   * - For pgvector: { schema?: string } controls schema (default: "kernl")
+   * - For Turbopuffer: not used
+   */
+  indexProviderOptions?: Record<string, unknown>;
+
+  /**
+   * Vector dimensions for embeddings.
+   * Only needed if embedding model doesn't provide this automatically.
+   * @default 1536
+   */
+  dimensions?: number;
+
+  /**
+   * Similarity metric for vector search.
+   * @default "cosine"
+   */
+  similarity?: "cosine" | "euclidean" | "dot_product";
+}
+
+/**
+ * Configuration options for creating a Kernl instance.
+ */
+export interface KernlOptions {
+  /**
+   * Storage configuration for persisting threads, tasks, and traces.
+   */
+  storage?: StorageOptions;
+
+  /**
+   * Memory system configuration.
+   */
+  memory?: MemoryOptions;
+}
+
+/**
+ * Agent registry interface.
+ *
+ * Satisfied by Map<string, Agent>.
+ */
+export interface AgentRegistry {
+  get(id: string): Agent<any> | undefined;
+}
+
+/**
+ * Model registry interface.
+ *
+ * Satisfied by Map<string, LanguageModel>.
+ * Key format: "provider/modelId"
+ *
+ * TODO: Create an exhaustive model registry in the protocol package
+ * with all supported models and their metadata.
+ */
+export interface ModelRegistry {
+  get(key: string): LanguageModel | undefined;
+}

package/src/lib/error.ts

@@ -9,8 +9,8 @@ import { randomID } from "@kernl-sdk/shared/lib";
 // import { SerializedThread } from "@/lib/serde/thread";
 type SerializedThread = any;
 
-import { AgentResponseType } from "@/types/agent";
-import { TextResponse } from "@/types/thread";
+import { AgentResponseType } from "@/agent/types";
+import { TextResponse } from "@/thread/types";
 
 /**
  * Abstract base class for all `kernl` errors

package/src/lifecycle.ts

@@ -5,8 +5,8 @@ import { Context, UnknownContext } from "./context";
 import { Tool } from "./tool";
 import type { ToolCall } from "@kernl-sdk/protocol";
 
-import { AgentResponseType } from "./types/agent";
-import { TextResponse } from "./types/thread";
+import { AgentResponseType } from "@/agent/types";
+import { TextResponse } from "@/thread/types";
 
 export type EventEmitterEvents = Record<string, any[]>;
 

package/src/memory/codecs/domain.ts

@@ -0,0 +1,115 @@
+/**
+ * Domain-level memory codecs.
+ *
+ * Codecs for transforming between memory domain types and search/index formats.
+ */
+
+import type { Codec, AsyncCodec } from "@kernl-sdk/shared/lib";
+import type { Filter as SearchFilter } from "@kernl-sdk/retrieval";
+
+import type {
+  MemoryFilter,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  IndexMemoryRecord,
+  IndexMemoryRecordPatch,
+  MemoryByteCodec,
+} from "../types";
+
+/**
+ * Codec for converting MemoryFilter to retrieval Filter.
+ *
+ * - scope.namespace → namespace
+ * - scope.entityId → entityId
+ * - scope.agentId → agentId
+ * - collections → collection: { $in: [...] }
+ * - after/before → timestamp: { $gt/$lt }
+ *
+ * Content field filtering (text, metadata, kind) is not currently supported.
+ * Text relevance is handled via vector similarity in the query, not filters.
+ */
+export const MEMORY_FILTER: Codec<MemoryFilter, SearchFilter> = {
+  encode(mf: MemoryFilter): SearchFilter {
+    const sf: SearchFilter = {};
+
+    // scope
+    if (mf.scope?.namespace !== undefined) sf.namespace = mf.scope.namespace;
+    if (mf.scope?.entityId !== undefined) sf.entityId = mf.scope.entityId;
+    if (mf.scope?.agentId !== undefined) sf.agentId = mf.scope.agentId;
+
+    if (mf.collections && mf.collections.length > 0) {
+      sf.collection = { $in: mf.collections };
+    }
+
+    if (mf.after !== undefined || mf.before !== undefined) {
+      const ts: { $gt?: number; $lt?: number } = {};
+      if (mf.after !== undefined) ts.$gt = mf.after;
+      if (mf.before !== undefined) ts.$lt = mf.before;
+      sf.timestamp = ts;
+    }
+
+    return sf;
+  },
+
+  decode(_filter: SearchFilter): MemoryFilter {
+    throw new Error("MEMORY_FILTER.decode not implemented");
+  },
+};
+
+/**
+ * Create a codec for MemoryRecord -> IndexMemoryRecord.
+ */
+export function recordCodec(
+  bytecodec: MemoryByteCodec,
+): AsyncCodec<MemoryRecord, IndexMemoryRecord> {
+  return {
+    async encode(record: MemoryRecord): Promise<IndexMemoryRecord> {
+      const indexable = await bytecodec.encode(record.content);
+      return {
+        id: record.id,
+        namespace: record.scope.namespace ?? null,
+        entityId: record.scope.entityId ?? null,
+        agentId: record.scope.agentId ?? null,
+        kind: record.kind,
+        collection: record.collection,
+        timestamp: record.timestamp,
+        createdAt: record.createdAt,
+        updatedAt: record.updatedAt,
+        ...indexable,
+      };
+    },
+    async decode(): Promise<MemoryRecord> {
+      throw new Error("recordCodec.decode not implemented");
+    },
+  };
+}
+
+/**
+ * Codec for converting MemoryRecordUpdate to IndexMemoryRecordPatch.
+ *
+ * Maps patchable fields from domain update to index patch format.
+ * wmem/smem are store-only fields and are not included.
+ * content changes require full re-index, not a patch.
+ */
+export const PATCH_CODEC: Codec<MemoryRecordUpdate, IndexMemoryRecordPatch> = {
+  encode(update: MemoryRecordUpdate): IndexMemoryRecordPatch {
+    const patch: IndexMemoryRecordPatch = { id: update.id };
+
+    if (update.scope?.namespace !== undefined)
+      patch.namespace = update.scope.namespace;
+    if (update.scope?.entityId !== undefined)
+      patch.entityId = update.scope.entityId;
+    if (update.scope?.agentId !== undefined)
+      patch.agentId = update.scope.agentId;
+    if (update.collection !== undefined) patch.collection = update.collection;
+    if (update.timestamp !== undefined) patch.timestamp = update.timestamp;
+    if (update.updatedAt !== undefined) patch.updatedAt = update.updatedAt;
+    if (update.metadata !== undefined) patch.metadata = update.metadata;
+
+    return patch;
+  },
+
+  decode(_patch: IndexMemoryRecordPatch): MemoryRecordUpdate {
+    throw new Error("PATCH_CODEC.decode not implemented");
+  },
+};

package/src/memory/codecs/identity.ts

@@ -0,0 +1,28 @@
+/**
+ * Identity codecs - pass through unchanged.
+ *
+ * Used for backends that support the full IndexMemoryRecord schema natively.
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { FieldSchema, SearchQuery, UnknownDocument } from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+
+export const IDENTITY_DOC: Codec<IndexMemoryRecord, UnknownDocument> = {
+  encode: (doc) => doc as unknown as UnknownDocument,
+  decode: (row) => row as unknown as IndexMemoryRecord,
+};
+
+export const IDENTITY_SCHEMA: Codec<
+  Record<string, FieldSchema>,
+  Record<string, FieldSchema>
+> = {
+  encode: (schema) => schema,
+  decode: (schema) => schema,
+};
+
+export const IDENTITY_QUERY: Codec<SearchQuery, SearchQuery> = {
+  encode: (q) => q,
+  decode: (q) => q,
+};

package/src/memory/codecs/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Memory codecs.
+ *
+ * Re-exports all memory codecs:
+ * - Domain codecs (MEMORY_FILTER, PATCH_CODEC, recordCodec)
+ * - Backend codecs (TPUF_*, IDENTITY_*)
+ * - Backend codec registry (getBackendCodecs)
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  SearchQuery,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+import { IDENTITY_DOC, IDENTITY_SCHEMA, IDENTITY_QUERY } from "./identity";
+import { TPUF_DOC, TPUF_SCHEMA, TPUF_QUERY } from "./tpuf";
+
+// re-exports
+export { MEMORY_FILTER, PATCH_CODEC, recordCodec } from "./domain";
+export { IDENTITY_DOC, IDENTITY_SCHEMA, IDENTITY_QUERY } from "./identity";
+export { TPUF_DOC, TPUF_SCHEMA, TPUF_QUERY } from "./tpuf";
+
+/**
+ * Backend codec set.
+ */
+export interface AdapterCodecs {
+  doc: Codec<IndexMemoryRecord, UnknownDocument>;
+  schema: Codec<Record<string, FieldSchema>, Record<string, FieldSchema>>;
+  query: Codec<SearchQuery, SearchQuery>;
+}
+
+/**
+ * Registry of backend codecs.
+ */
+export const ADAPTER_CODECS: Record<string, AdapterCodecs> = {
+  turbopuffer: { doc: TPUF_DOC, schema: TPUF_SCHEMA, query: TPUF_QUERY },
+  pgvector: {
+    doc: IDENTITY_DOC,
+    schema: IDENTITY_SCHEMA,
+    query: IDENTITY_QUERY,
+  },
+};
+
+/**
+ * Default codecs (identity) for unknown backends.
+ */
+const DEFAULT_CODECS: AdapterCodecs = {
+  doc: IDENTITY_DOC,
+  schema: IDENTITY_SCHEMA,
+  query: IDENTITY_QUERY,
+};
+
+/**
+ * Get codecs for a backend, falling back to identity.
+ */
+export function getAdapterCodecs(adapterId: string): AdapterCodecs {
+  return ADAPTER_CODECS[adapterId] ?? DEFAULT_CODECS;
+}

package/src/memory/codecs/tpuf.ts

@@ -0,0 +1,115 @@
+/**
+ * Turbopuffer backend codecs.
+ *
+ * Turbopuffer constraints:
+ * - Exactly one ANN vector field named "vector" per namespace.
+ *
+ * Memory model:
+ * - IndexMemoryRecord has modality-specific vectors: tvec, ivec, avec, vvec.
+ *
+ * Mapping:
+ * - tvec (text embedding) → vector
+ * - ivec/avec/vvec are dropped (not indexed in Turbopuffer)
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  SearchQuery,
+  RankingSignal,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+
+/**
+ * Turbopuffer document codec.
+ *
+ * Maps tvec → vector, drops ivec/avec/vvec.
+ */
+export const TPUF_DOC: Codec<IndexMemoryRecord, UnknownDocument> = {
+  encode(doc: IndexMemoryRecord): UnknownDocument {
+    const { tvec, ivec, avec, vvec, metadata, ...rest } = doc;
+    const row: UnknownDocument = {
+      ...rest,
+      metadata: metadata as UnknownDocument["metadata"], // metadata is JSONObject | null, cast to FieldValue for UnknownDocument
+    };
+    if (tvec) row.vector = tvec;
+    return row;
+  },
+
+  decode(row: UnknownDocument): IndexMemoryRecord {
+    const { vector, ...rest } = row;
+    return {
+      ...(rest as unknown as IndexMemoryRecord),
+      tvec: Array.isArray(vector) ? (vector as number[]) : undefined,
+    };
+  },
+};
+
+/**
+ * Turbopuffer schema codec.
+ *
+ * Maps tvec → vector, drops ivec/avec/vvec fields.
+ */
+export const TPUF_SCHEMA: Codec<
+  Record<string, FieldSchema>,
+  Record<string, FieldSchema>
+> = {
+  encode(schema: Record<string, FieldSchema>): Record<string, FieldSchema> {
+    const result: Record<string, FieldSchema> = {};
+    for (const [name, field] of Object.entries(schema)) {
+      if (name === "tvec") {
+        result.vector = field;
+      } else if (name === "ivec" || name === "avec" || name === "vvec") {
+        continue;
+      } else {
+        result[name] = field;
+      }
+    }
+    return result;
+  },
+
+  decode(): Record<string, FieldSchema> {
+    throw new Error("TPUF_SCHEMA.decode not implemented");
+  },
+};
+
+/**
+ * Turbopuffer query codec.
+ *
+ * Maps tvec → vector in query signals, drops ivec/avec/vvec.
+ * Operates on normalized SearchQuery (after planQuery).
+ *
+ * Defaults include to true so memory searches return all document attributes.
+ */
+export const TPUF_QUERY: Codec<SearchQuery, SearchQuery> = {
+  encode(input: SearchQuery): SearchQuery {
+    // default include to true for memory search (tpuf only returns id + score without it)
+    const include = input.include ?? true;
+
+    if (!input.query) {
+      return { ...input, include };
+    }
+
+    const signals = input.query.map((signal: RankingSignal) => {
+      const { tvec, ivec, avec, vvec, ...rest } = signal as RankingSignal & {
+        tvec?: number[];
+        ivec?: number[];
+        avec?: number[];
+        vvec?: number[];
+      };
+
+      if (tvec) {
+        return { ...rest, vector: tvec };
+      }
+      return rest;
+    });
+
+    return { ...input, query: signals, include };
+  },
+
+  decode(): SearchQuery {
+    throw new Error("TPUF_QUERY.decode not implemented");
+  },
+};

package/src/memory/encoder.ts

@@ -0,0 +1,56 @@
+/**
+ * MemoryByte encoder - converts MemoryByte to IndexableByte with embeddings.
+ */
+
+import type { EmbeddingModel } from "@kernl-sdk/protocol";
+
+import type { MemoryByte, IndexableByte, MemoryByteCodec } from "./types";
+
+/**
+ * Encoder that converts MemoryByte to IndexableByte.
+ *
+ * Extracts canonical text from content and computes embeddings.
+ */
+export class MemoryByteEncoder implements MemoryByteCodec {
+  private readonly embedder: EmbeddingModel<string>;
+
+  constructor(embedder: EmbeddingModel<string>) {
+    this.embedder = embedder;
+  }
+
+  /**
+   * Encode a MemoryByte to IndexableByte.
+   * Extracts text and computes embeddings for each modality.
+   */
+  async encode(byte: MemoryByte): Promise<IndexableByte> {
+    const text = byte.text;
+    const tvec = text ? await this.embed(text) : undefined;
+
+    // TODO: embed other modalities (image, audio, video)
+    // const ivec = byte.image ? await this.embedImage(byte.image) : undefined;
+    // const avec = byte.audio ? await this.embedAudio(byte.audio) : undefined;
+    // const vvec = byte.video ? await this.embedVideo(byte.video) : undefined;
+
+    return {
+      text,
+      tvec,
+      metadata: byte.object ?? null,
+    };
+  }
+
+  /**
+   * Decode is not implemented - IndexableByte cannot be converted back to MemoryByte.
+   */
+  async decode(_indexable: IndexableByte): Promise<MemoryByte> {
+    throw new Error("MemoryByteEncoder.decode not implemented");
+  }
+
+  /**
+   * Embed a text string.
+   * Exposed for query embedding.
+   */
+  async embed(text: string): Promise<number[]> {
+    const result = await this.embedder.embed({ values: [text] });
+    return result.embeddings[0] ?? [];
+  }
+}

package/src/memory/handle.ts

@@ -0,0 +1,189 @@
+/**
+ * Memory index handle with lazy initialization.
+ */
+
+import {
+  planQuery,
+  type SearchIndex,
+  type IndexHandle,
+  type QueryInput,
+  type SearchHit,
+  type DocumentPatch,
+  type UpsertResult,
+  type PatchResult,
+  type DeleteResult,
+  type FieldSchema,
+  type SearchCapabilities,
+  type UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import { getAdapterCodecs, type AdapterCodecs } from "./codecs";
+import type { IndexMemoryRecord } from "./types";
+
+/**
+ * Configuration for MemoryIndexHandle.
+ */
+export interface MemoryIndexHandleConfig {
+  /**
+   * The underlying search index backend.
+   */
+  index: SearchIndex;
+
+  /**
+   * Index identifier (e.g., "kernl_memories_index").
+   */
+  indexId: string;
+
+  /**
+   * Field schema for the memory index.
+   */
+  schema: Record<string, FieldSchema>;
+
+  /**
+   * Backend-specific options passed to SearchIndex.createIndex().
+   */
+  providerOptions?: Record<string, unknown>;
+}
+
+/**
+ * Domain-aware index handle for memory records with lazy initialization.
+ *
+ * - Wraps a SearchIndex and ensures the memory index is created before any operation.
+ * - Normalizes the idiosyncrasies of search adapters (capabilities, weird rules, ... - e.g. tpuf requires vector fields named
+ * literally 'vector' - dumb shit like this..)
+ */
+export class MemoryIndexHandle implements IndexHandle<IndexMemoryRecord> {
+  readonly id: string;
+
+  private readonly index: SearchIndex;
+  private readonly schema: Record<string, FieldSchema>;
+  private readonly caps: SearchCapabilities;
+  private readonly codecs: AdapterCodecs;
+  private readonly providerOptions?: Record<string, unknown>;
+
+  private initPromise: Promise<void> | null = null;
+
+  constructor(config: MemoryIndexHandleConfig) {
+    this.index = config.index;
+    this.id = config.indexId;
+    this.schema = config.schema;
+    this.caps = this.index.capabilities();
+    this.codecs = getAdapterCodecs(this.index.id);
+    this.providerOptions = config.providerOptions;
+  }
+
+  /**
+   * Ensure memory index exists (lazy initialization).
+   *
+   * Safe to call multiple times - initialization only runs once.
+   */
+  private async ensureInit(): Promise<void> {
+    if (!this.initPromise) {
+      this.initPromise = this.createIndex().catch((err) => {
+        this.initPromise = null;
+        throw err;
+      });
+    }
+    await this.initPromise;
+  }
+
+  /**
+   * Create the memory index if it doesn't exist.
+   */
+  private async createIndex(): Promise<void> {
+    try {
+      await this.index.createIndex({
+        id: this.id,
+        schema: this.codecs.schema.encode(this.schema),
+        providerOptions: this.providerOptions,
+      });
+    } catch (err: any) {
+      // (TODO): we should probably enforce a stricter contract w/ tighter error types
+      //
+      // Ignore "already exists" errors
+      if (
+        err.message?.includes("already exists") ||
+        err.message?.includes("AlreadyExists")
+      ) {
+        return;
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * Search for memory records matching the query.
+   *
+   * Adapts the query to backend capabilities, degrading gracefully
+   * when hybrid or multi-signal queries aren't supported.
+   */
+  async query(input: QueryInput): Promise<SearchHit<IndexMemoryRecord>[]> {
+    const { input: planned } = planQuery(input, this.caps);
+    const q = this.codecs.query.encode(planned);
+    const handle = await this.handle();
+    const hits = await handle.query(q);
+
+    // decode hits back to IndexMemoryRecord format
+    return hits.map((hit) => ({
+      ...hit,
+      document: hit.document
+        ? this.codecs.doc.decode(hit.document as UnknownDocument)
+        : undefined,
+    }));
+  }
+
+  /**
+   * Insert or update memory records in the index.
+   */
+  async upsert(
+    docs: IndexMemoryRecord | IndexMemoryRecord[],
+  ): Promise<UpsertResult> {
+    const arr = Array.isArray(docs) ? docs : [docs];
+    const encoded = arr.map((doc) => this.codecs.doc.encode(doc));
+    const handle = await this.handle();
+    return handle.upsert(encoded);
+  }
+
+  /**
+   * Partially update memory records without re-indexing vectors.
+   *
+   * Note: Patches don't include vector fields so we cast directly.
+   * Metadata field type mismatch (JSONObject vs FieldValue) is handled at runtime.
+   */
+  async patch(
+    patches:
+      | DocumentPatch<IndexMemoryRecord>
+      | DocumentPatch<IndexMemoryRecord>[],
+  ): Promise<PatchResult> {
+    const handle = await this.handle();
+    return handle.patch(patches as DocumentPatch<UnknownDocument>[]);
+  }
+
+  /**
+   * Remove memory records from the index.
+   */
+  async delete(ids: string | string[]): Promise<DeleteResult> {
+    const handle = await this.handle();
+    return handle.delete(ids);
+  }
+
+  /**
+   * Add a new field to the index schema.
+   *
+   * @throws Always throws - dynamic schema modification not supported
+   */
+  async addField(_field: string, _schema: FieldSchema): Promise<void> {
+    throw new Error("addField not supported for MemoryIndexHandle");
+  }
+
+  /**
+   * Get an initialized underlying index handle.
+   *
+   * Returns a handle typed for UnknownDocument since we encode/decode
+   * through the adapter codecs for backend-specific field mapping.
+   */
+  private async handle(): Promise<IndexHandle<UnknownDocument>> {
+    await this.ensureInit();
+    return this.index.index<UnknownDocument>(this.id);
+  }
+}