kernl 0.6.1 → 0.6.3

package/src/{kernl.ts → kernl/kernl.ts}

@@ -1,4 +1,5 @@
 import type { LanguageModel } from "@kernl-sdk/protocol";
+import { resolveEmbeddingModel } from "@kernl-sdk/retrieval";
 
 import { Agent } from "@/agent";
 import { UnknownContext } from "@/context";
@@ -7,10 +8,17 @@ import type { Thread } from "@/thread";
 import type { ResolvedAgentResponse } from "@/guardrail";
 import { InMemoryStorage, type KernlStorage } from "@/storage";
 import { RThreads } from "@/api/resources/threads";
+import {
+  Memory,
+  MemoryByteEncoder,
+  MemoryIndexHandle,
+  buildMemoryIndexSchema,
+} from "@/memory";
 
-import type { KernlOptions } from "@/types/kernl";
-import type { ThreadExecuteResult, ThreadStreamEvent } from "@/types/thread";
-import type { AgentResponseType } from "@/types/agent";
+import type { ThreadExecuteResult, ThreadStreamEvent } from "@/thread/types";
+import type { AgentResponseType } from "@/agent/types";
+
+import type { KernlOptions } from "./types";
 
 /**
  * The kernl - manages agent processes, scheduling, and task lifecycle.
@@ -25,16 +33,43 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
   readonly storage: KernlStorage;
   athreads: Map<string, Thread<any, any>> = new Map(); /* active threads */
 
-  private initPromise: Promise<void> | null = null;
-
   // --- public API ---
   readonly threads: RThreads; /* Threads resource */
+  readonly memories: Memory; /* Memory system */
 
   constructor(options: KernlOptions = {}) {
     super();
     this.storage = options.storage?.db ?? new InMemoryStorage();
     this.storage.bind({ agents: this.agents, models: this.models });
     this.threads = new RThreads(this.storage.threads);
+
+    // initialize memory
+    const embeddingModel =
+      options.memory?.embeddingModel ?? "openai/text-embedding-3-small";
+    const embedder =
+      typeof embeddingModel === "string"
+        ? resolveEmbeddingModel<string>(embeddingModel)
+        : embeddingModel;
+    const encoder = new MemoryByteEncoder(embedder);
+
+    const vector = options.storage?.vector;
+    const indexId = options.memory?.indexId ?? "memories_sindex";
+    const dimensions = options.memory?.dimensions ?? 1536;
+    const providerOptions = options.memory?.indexProviderOptions ?? { schema: "kernl" };
+
+    this.memories = new Memory({
+      store: this.storage.memories,
+      search:
+        vector !== undefined
+          ? new MemoryIndexHandle({
+              index: vector,
+              indexId,
+              schema: buildMemoryIndexSchema({ dimensions }),
+              providerOptions,
+            })
+          : undefined,
+      encoder,
+    });
   }
 
   /**
@@ -59,7 +94,6 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
   async spawn<TContext, TResponse extends AgentResponseType>(
     thread: Thread<TContext, TResponse>,
   ): Promise<ThreadExecuteResult<ResolvedAgentResponse<TResponse>>> {
-    await this.ensureInitialized();
     this.athreads.set(thread.tid, thread);
     try {
       return await thread.execute();
@@ -76,7 +110,6 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
   async schedule<TContext, TResponse extends AgentResponseType>(
     thread: Thread<TContext, TResponse>,
   ): Promise<ThreadExecuteResult<ResolvedAgentResponse<TResponse>>> {
-    await this.ensureInitialized();
     this.athreads.set(thread.tid, thread);
     try {
       return await thread.execute();
@@ -93,7 +126,6 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
   async *spawnStream<TContext, TResponse extends AgentResponseType>(
     thread: Thread<TContext, TResponse>,
   ): AsyncIterable<ThreadStreamEvent> {
-    await this.ensureInitialized();
     this.athreads.set(thread.tid, thread);
     try {
       yield* thread.stream();
@@ -110,7 +142,6 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
   async *scheduleStream<TContext, TResponse extends AgentResponseType>(
     thread: Thread<TContext, TResponse>,
   ): AsyncIterable<ThreadStreamEvent> {
-    await this.ensureInitialized();
     this.athreads.set(thread.tid, thread);
     try {
       yield* thread.stream();
@@ -118,23 +149,4 @@ export class Kernl extends KernlHooks<UnknownContext, AgentResponseType> {
       this.athreads.delete(thread.tid);
     }
   }
-
-  /**
-   * Ensure the underlying storage backend has been initialized.
-   *
-   * This is called lazily on first use so that callers do not need to worry
-   * about calling storage.init() themselves. Safe and idempotent to call
-   * multiple times; concurrent calls share the same initialization promise.
-   */
-  private async ensureInitialized(): Promise<void> {
-    if (!this.initPromise) {
-      this.initPromise = this.storage.init().catch((error) => {
-        // allow a retry if initialization fails.
-        this.initPromise = null;
-        throw error;
-      });
-    }
-
-    await this.initPromise;
-  }
 }

package/src/kernl/types.ts

@@ -0,0 +1,106 @@
+import { LanguageModel, EmbeddingModel } from "@kernl-sdk/protocol";
+import { SearchIndex } from "@kernl-sdk/retrieval";
+
+import { Agent } from "@/agent";
+import { KernlStorage } from "@/storage";
+
+/**
+ * Storage configuration for Kernl.
+ */
+export interface StorageOptions {
+  /**
+   * Relational database storage (threads, tasks, traces).
+   */
+  db?: KernlStorage;
+
+  /**
+   * Vector search index for semantic memory search.
+   * Supports pgvector, Turbopuffer, etc.
+   */
+  vector?: SearchIndex;
+
+  // Future storage layers (deferred):
+  // blob?: BlobStore;
+  // lake?: DataLake;
+}
+
+/**
+ * Memory system configuration.
+ */
+export interface MemoryOptions {
+  /**
+   * Embedding model for memory encoding.
+   *
+   * Can be:
+   * - A string like "openai/text-embedding-3-small" (resolved via provider registry)
+   * - An EmbeddingModel instance
+   *
+   * @default "openai/text-embedding-3-small"
+   */
+  embeddingModel?: string | EmbeddingModel<string>;
+
+  /**
+   * Logical index ID used by the search backend.
+   * - For pgvector: becomes the table name (with schema from indexProviderOptions)
+   * - For Turbopuffer: becomes the namespace
+   * @default "kernl_memories_index"
+   */
+  indexId?: string;
+
+  /**
+   * Backend-specific options passed to SearchIndex.createIndex().
+   * - For pgvector: { schema?: string } controls schema (default: "kernl")
+   * - For Turbopuffer: not used
+   */
+  indexProviderOptions?: Record<string, unknown>;
+
+  /**
+   * Vector dimensions for embeddings.
+   * Only needed if embedding model doesn't provide this automatically.
+   * @default 1536
+   */
+  dimensions?: number;
+
+  /**
+   * Similarity metric for vector search.
+   * @default "cosine"
+   */
+  similarity?: "cosine" | "euclidean" | "dot_product";
+}
+
+/**
+ * Configuration options for creating a Kernl instance.
+ */
+export interface KernlOptions {
+  /**
+   * Storage configuration for persisting threads, tasks, and traces.
+   */
+  storage?: StorageOptions;
+
+  /**
+   * Memory system configuration.
+   */
+  memory?: MemoryOptions;
+}
+
+/**
+ * Agent registry interface.
+ *
+ * Satisfied by Map<string, Agent>.
+ */
+export interface AgentRegistry {
+  get(id: string): Agent<any> | undefined;
+}
+
+/**
+ * Model registry interface.
+ *
+ * Satisfied by Map<string, LanguageModel>.
+ * Key format: "provider/modelId"
+ *
+ * TODO: Create an exhaustive model registry in the protocol package
+ * with all supported models and their metadata.
+ */
+export interface ModelRegistry {
+  get(key: string): LanguageModel | undefined;
+}

package/src/lib/error.ts

@@ -9,8 +9,8 @@ import { randomID } from "@kernl-sdk/shared/lib";
 // import { SerializedThread } from "@/lib/serde/thread";
 type SerializedThread = any;
 
-import { AgentResponseType } from "@/types/agent";
-import { TextResponse } from "@/types/thread";
+import { AgentResponseType } from "@/agent/types";
+import { TextResponse } from "@/thread/types";
 
 /**
  * Abstract base class for all `kernl` errors

package/src/lifecycle.ts

@@ -5,8 +5,8 @@ import { Context, UnknownContext } from "./context";
 import { Tool } from "./tool";
 import type { ToolCall } from "@kernl-sdk/protocol";
 
-import { AgentResponseType } from "./types/agent";
-import { TextResponse } from "./types/thread";
+import { AgentResponseType } from "@/agent/types";
+import { TextResponse } from "@/thread/types";
 
 export type EventEmitterEvents = Record<string, any[]>;
 

package/src/memory/codecs/domain.ts

@@ -0,0 +1,115 @@
+/**
+ * Domain-level memory codecs.
+ *
+ * Codecs for transforming between memory domain types and search/index formats.
+ */
+
+import type { Codec, AsyncCodec } from "@kernl-sdk/shared/lib";
+import type { Filter as SearchFilter } from "@kernl-sdk/retrieval";
+
+import type {
+  MemoryFilter,
+  MemoryRecord,
+  MemoryRecordUpdate,
+  IndexMemoryRecord,
+  IndexMemoryRecordPatch,
+  MemoryByteCodec,
+} from "../types";
+
+/**
+ * Codec for converting MemoryFilter to retrieval Filter.
+ *
+ * - scope.namespace → namespace
+ * - scope.entityId → entityId
+ * - scope.agentId → agentId
+ * - collections → collection: { $in: [...] }
+ * - after/before → timestamp: { $gt/$lt }
+ *
+ * Content field filtering (text, metadata, kind) is not currently supported.
+ * Text relevance is handled via vector similarity in the query, not filters.
+ */
+export const MEMORY_FILTER: Codec<MemoryFilter, SearchFilter> = {
+  encode(mf: MemoryFilter): SearchFilter {
+    const sf: SearchFilter = {};
+
+    // scope
+    if (mf.scope?.namespace !== undefined) sf.namespace = mf.scope.namespace;
+    if (mf.scope?.entityId !== undefined) sf.entityId = mf.scope.entityId;
+    if (mf.scope?.agentId !== undefined) sf.agentId = mf.scope.agentId;
+
+    if (mf.collections && mf.collections.length > 0) {
+      sf.collection = { $in: mf.collections };
+    }
+
+    if (mf.after !== undefined || mf.before !== undefined) {
+      const ts: { $gt?: number; $lt?: number } = {};
+      if (mf.after !== undefined) ts.$gt = mf.after;
+      if (mf.before !== undefined) ts.$lt = mf.before;
+      sf.timestamp = ts;
+    }
+
+    return sf;
+  },
+
+  decode(_filter: SearchFilter): MemoryFilter {
+    throw new Error("MEMORY_FILTER.decode not implemented");
+  },
+};
+
+/**
+ * Create a codec for MemoryRecord -> IndexMemoryRecord.
+ */
+export function recordCodec(
+  bytecodec: MemoryByteCodec,
+): AsyncCodec<MemoryRecord, IndexMemoryRecord> {
+  return {
+    async encode(record: MemoryRecord): Promise<IndexMemoryRecord> {
+      const indexable = await bytecodec.encode(record.content);
+      return {
+        id: record.id,
+        namespace: record.scope.namespace ?? null,
+        entityId: record.scope.entityId ?? null,
+        agentId: record.scope.agentId ?? null,
+        kind: record.kind,
+        collection: record.collection,
+        timestamp: record.timestamp,
+        createdAt: record.createdAt,
+        updatedAt: record.updatedAt,
+        ...indexable,
+      };
+    },
+    async decode(): Promise<MemoryRecord> {
+      throw new Error("recordCodec.decode not implemented");
+    },
+  };
+}
+
+/**
+ * Codec for converting MemoryRecordUpdate to IndexMemoryRecordPatch.
+ *
+ * Maps patchable fields from domain update to index patch format.
+ * wmem/smem are store-only fields and are not included.
+ * content changes require full re-index, not a patch.
+ */
+export const PATCH_CODEC: Codec<MemoryRecordUpdate, IndexMemoryRecordPatch> = {
+  encode(update: MemoryRecordUpdate): IndexMemoryRecordPatch {
+    const patch: IndexMemoryRecordPatch = { id: update.id };
+
+    if (update.scope?.namespace !== undefined)
+      patch.namespace = update.scope.namespace;
+    if (update.scope?.entityId !== undefined)
+      patch.entityId = update.scope.entityId;
+    if (update.scope?.agentId !== undefined)
+      patch.agentId = update.scope.agentId;
+    if (update.collection !== undefined) patch.collection = update.collection;
+    if (update.timestamp !== undefined) patch.timestamp = update.timestamp;
+    if (update.updatedAt !== undefined) patch.updatedAt = update.updatedAt;
+    if (update.metadata !== undefined) patch.metadata = update.metadata;
+
+    return patch;
+  },
+
+  decode(_patch: IndexMemoryRecordPatch): MemoryRecordUpdate {
+    throw new Error("PATCH_CODEC.decode not implemented");
+  },
+};

package/src/memory/codecs/identity.ts

@@ -0,0 +1,28 @@
+/**
+ * Identity codecs - pass through unchanged.
+ *
+ * Used for backends that support the full IndexMemoryRecord schema natively.
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { FieldSchema, SearchQuery, UnknownDocument } from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+
+export const IDENTITY_DOC: Codec<IndexMemoryRecord, UnknownDocument> = {
+  encode: (doc) => doc as unknown as UnknownDocument,
+  decode: (row) => row as unknown as IndexMemoryRecord,
+};
+
+export const IDENTITY_SCHEMA: Codec<
+  Record<string, FieldSchema>,
+  Record<string, FieldSchema>
+> = {
+  encode: (schema) => schema,
+  decode: (schema) => schema,
+};
+
+export const IDENTITY_QUERY: Codec<SearchQuery, SearchQuery> = {
+  encode: (q) => q,
+  decode: (q) => q,
+};

package/src/memory/codecs/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Memory codecs.
+ *
+ * Re-exports all memory codecs:
+ * - Domain codecs (MEMORY_FILTER, PATCH_CODEC, recordCodec)
+ * - Backend codecs (TPUF_*, IDENTITY_*)
+ * - Backend codec registry (getBackendCodecs)
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  SearchQuery,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+import { IDENTITY_DOC, IDENTITY_SCHEMA, IDENTITY_QUERY } from "./identity";
+import { TPUF_DOC, TPUF_SCHEMA, TPUF_QUERY } from "./tpuf";
+
+// re-exports
+export { MEMORY_FILTER, PATCH_CODEC, recordCodec } from "./domain";
+export { IDENTITY_DOC, IDENTITY_SCHEMA, IDENTITY_QUERY } from "./identity";
+export { TPUF_DOC, TPUF_SCHEMA, TPUF_QUERY } from "./tpuf";
+
+/**
+ * Backend codec set.
+ */
+export interface AdapterCodecs {
+  doc: Codec<IndexMemoryRecord, UnknownDocument>;
+  schema: Codec<Record<string, FieldSchema>, Record<string, FieldSchema>>;
+  query: Codec<SearchQuery, SearchQuery>;
+}
+
+/**
+ * Registry of backend codecs.
+ */
+export const ADAPTER_CODECS: Record<string, AdapterCodecs> = {
+  turbopuffer: { doc: TPUF_DOC, schema: TPUF_SCHEMA, query: TPUF_QUERY },
+  pgvector: {
+    doc: IDENTITY_DOC,
+    schema: IDENTITY_SCHEMA,
+    query: IDENTITY_QUERY,
+  },
+};
+
+/**
+ * Default codecs (identity) for unknown backends.
+ */
+const DEFAULT_CODECS: AdapterCodecs = {
+  doc: IDENTITY_DOC,
+  schema: IDENTITY_SCHEMA,
+  query: IDENTITY_QUERY,
+};
+
+/**
+ * Get codecs for a backend, falling back to identity.
+ */
+export function getAdapterCodecs(adapterId: string): AdapterCodecs {
+  return ADAPTER_CODECS[adapterId] ?? DEFAULT_CODECS;
+}

package/src/memory/codecs/tpuf.ts

@@ -0,0 +1,115 @@
+/**
+ * Turbopuffer backend codecs.
+ *
+ * Turbopuffer constraints:
+ * - Exactly one ANN vector field named "vector" per namespace.
+ *
+ * Memory model:
+ * - IndexMemoryRecord has modality-specific vectors: tvec, ivec, avec, vvec.
+ *
+ * Mapping:
+ * - tvec (text embedding) → vector
+ * - ivec/avec/vvec are dropped (not indexed in Turbopuffer)
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  SearchQuery,
+  RankingSignal,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+
+import type { IndexMemoryRecord } from "../types";
+
+/**
+ * Turbopuffer document codec.
+ *
+ * Maps tvec → vector, drops ivec/avec/vvec.
+ */
+export const TPUF_DOC: Codec<IndexMemoryRecord, UnknownDocument> = {
+  encode(doc: IndexMemoryRecord): UnknownDocument {
+    const { tvec, ivec, avec, vvec, metadata, ...rest } = doc;
+    const row: UnknownDocument = {
+      ...rest,
+      metadata: metadata as UnknownDocument["metadata"], // metadata is JSONObject | null, cast to FieldValue for UnknownDocument
+    };
+    if (tvec) row.vector = tvec;
+    return row;
+  },
+
+  decode(row: UnknownDocument): IndexMemoryRecord {
+    const { vector, ...rest } = row;
+    return {
+      ...(rest as unknown as IndexMemoryRecord),
+      tvec: Array.isArray(vector) ? (vector as number[]) : undefined,
+    };
+  },
+};
+
+/**
+ * Turbopuffer schema codec.
+ *
+ * Maps tvec → vector, drops ivec/avec/vvec fields.
+ */
+export const TPUF_SCHEMA: Codec<
+  Record<string, FieldSchema>,
+  Record<string, FieldSchema>
+> = {
+  encode(schema: Record<string, FieldSchema>): Record<string, FieldSchema> {
+    const result: Record<string, FieldSchema> = {};
+    for (const [name, field] of Object.entries(schema)) {
+      if (name === "tvec") {
+        result.vector = field;
+      } else if (name === "ivec" || name === "avec" || name === "vvec") {
+        continue;
+      } else {
+        result[name] = field;
+      }
+    }
+    return result;
+  },
+
+  decode(): Record<string, FieldSchema> {
+    throw new Error("TPUF_SCHEMA.decode not implemented");
+  },
+};
+
+/**
+ * Turbopuffer query codec.
+ *
+ * Maps tvec → vector in query signals, drops ivec/avec/vvec.
+ * Operates on normalized SearchQuery (after planQuery).
+ *
+ * Defaults include to true so memory searches return all document attributes.
+ */
+export const TPUF_QUERY: Codec<SearchQuery, SearchQuery> = {
+  encode(input: SearchQuery): SearchQuery {
+    // default include to true for memory search (tpuf only returns id + score without it)
+    const include = input.include ?? true;
+
+    if (!input.query) {
+      return { ...input, include };
+    }
+
+    const signals = input.query.map((signal: RankingSignal) => {
+      const { tvec, ivec, avec, vvec, ...rest } = signal as RankingSignal & {
+        tvec?: number[];
+        ivec?: number[];
+        avec?: number[];
+        vvec?: number[];
+      };
+
+      if (tvec) {
+        return { ...rest, vector: tvec };
+      }
+      return rest;
+    });
+
+    return { ...input, query: signals, include };
+  },
+
+  decode(): SearchQuery {
+    throw new Error("TPUF_QUERY.decode not implemented");
+  },
+};

package/src/memory/encoder.ts

@@ -0,0 +1,56 @@
+/**
+ * MemoryByte encoder - converts MemoryByte to IndexableByte with embeddings.
+ */
+
+import type { EmbeddingModel } from "@kernl-sdk/protocol";
+
+import type { MemoryByte, IndexableByte, MemoryByteCodec } from "./types";
+
+/**
+ * Encoder that converts MemoryByte to IndexableByte.
+ *
+ * Extracts canonical text from content and computes embeddings.
+ */
+export class MemoryByteEncoder implements MemoryByteCodec {
+  private readonly embedder: EmbeddingModel<string>;
+
+  constructor(embedder: EmbeddingModel<string>) {
+    this.embedder = embedder;
+  }
+
+  /**
+   * Encode a MemoryByte to IndexableByte.
+   * Extracts text and computes embeddings for each modality.
+   */
+  async encode(byte: MemoryByte): Promise<IndexableByte> {
+    const text = byte.text;
+    const tvec = text ? await this.embed(text) : undefined;
+
+    // TODO: embed other modalities (image, audio, video)
+    // const ivec = byte.image ? await this.embedImage(byte.image) : undefined;
+    // const avec = byte.audio ? await this.embedAudio(byte.audio) : undefined;
+    // const vvec = byte.video ? await this.embedVideo(byte.video) : undefined;
+
+    return {
+      text,
+      tvec,
+      metadata: byte.object ?? null,
+    };
+  }
+
+  /**
+   * Decode is not implemented - IndexableByte cannot be converted back to MemoryByte.
+   */
+  async decode(_indexable: IndexableByte): Promise<MemoryByte> {
+    throw new Error("MemoryByteEncoder.decode not implemented");
+  }
+
+  /**
+   * Embed a text string.
+   * Exposed for query embedding.
+   */
+  async embed(text: string): Promise<number[]> {
+    const result = await this.embedder.embed({ values: [text] });
+    return result.embeddings[0] ?? [];
+  }
+}