kernl 0.7.4 → 0.8.1

package/src/lifecycle.ts

@@ -5,8 +5,8 @@ import { Context, UnknownContext } from "./context";
 import { Tool } from "./tool";
 import type { ToolCall } from "@kernl-sdk/protocol";
 
-import { AgentResponseType } from "@/agent/types";
-import { TextResponse } from "@/thread/types";
+import { AgentOutputType } from "@/agent/types";
+import { TextOutput } from "@/thread/types";
 
 export type EventEmitterEvents = Record<string, any[]>;
 
@@ -60,7 +60,7 @@ class TypedEventEmitter<
 
 export type AgentHookEvents<
   TContext = UnknownContext,
-  TOutput extends AgentResponseType = TextResponse,
+  TOutput extends AgentOutputType = TextOutput,
 > = {
   /**
    * @param context - The context of the run
@@ -107,7 +107,7 @@ export type AgentHookEvents<
  */
 export class AgentHooks<
   TContext = UnknownContext,
-  TOutput extends AgentResponseType = TextResponse,
+  TOutput extends AgentOutputType = TextOutput,
 > extends TypedEventEmitter<AgentHookEvents<TContext, TOutput>> {}
 
 /**
@@ -119,7 +119,7 @@ export class AgentHooks<
  */
 export type KernlHookEvents<
   TContext = UnknownContext,
-  TOutput extends AgentResponseType = TextResponse,
+  TOutput extends AgentOutputType = TextOutput,
 > = {
   /**
    * @param context - The context of the run
@@ -177,5 +177,5 @@ export type KernlHookEvents<
  */
 export class KernlHooks<
   TContext = UnknownContext,
-  TOutput extends AgentResponseType = TextResponse,
+  TOutput extends AgentOutputType = TextOutput,
 > extends TypedEventEmitter<KernlHookEvents<TContext, TOutput>> {}

package/src/memory/__tests__/encoder.test.ts

@@ -0,0 +1,154 @@
+import { describe, it, expect, vi } from "vitest";
+import type { EmbeddingModel } from "@kernl-sdk/protocol";
+
+import { MemoryByteEncoder, ObjectTextCodec } from "../encoder";
+import type { MemoryByte } from "../types";
+
+// Mock embedder that returns predictable vectors
+function createMockEmbedder(): EmbeddingModel<string> {
+  return {
+    provider: "test",
+    modelId: "test-embedder",
+    embed: vi.fn(async ({ values }: { values: string[] }) => ({
+      embeddings: values.map((v) => [v.length, 0, 0]), // simple: [length, 0, 0]
+    })),
+  } as unknown as EmbeddingModel<string>;
+}
+
+describe("ObjectTextCodec", () => {
+  it("encodes simple object to YAML", () => {
+    const obj = { name: "Tony", preference: "coffee" };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("name: Tony");
+    expect(result).toContain("preference: coffee");
+  });
+
+  it("sorts keys for determinism", () => {
+    const obj1 = { z: 1, a: 2, m: 3 };
+    const obj2 = { a: 2, m: 3, z: 1 };
+
+    expect(ObjectTextCodec.encode(obj1)).toBe(ObjectTextCodec.encode(obj2));
+  });
+
+  it("truncates long objects with ellipsis", () => {
+    const longValue = "x".repeat(4000);
+    const obj = { data: longValue };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result.length).toBeLessThanOrEqual(3005); // 3000 + "\n..."
+    expect(result.endsWith("\n...")).toBe(true);
+  });
+
+  it("handles nested objects", () => {
+    const obj = {
+      user: {
+        name: "Tony",
+        prefs: { coffee: { shots: 2 } },
+      },
+    };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("user:");
+    expect(result).toContain("name: Tony");
+    expect(result).toContain("shots: 2");
+  });
+
+  it("handles arrays", () => {
+    const obj = { items: ["a", "b", "c"] };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("items:");
+    expect(result).toContain("- a");
+  });
+});
+
+describe("MemoryByteEncoder", () => {
+  describe("encode", () => {
+    it("encodes text-only content", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = { text: "Hello world" };
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBe("Hello world");
+      expect(result.objtext).toBeUndefined();
+      expect(result.tvec).toBeDefined();
+      expect(embedder.embed).toHaveBeenCalledWith({ values: ["Hello world"] });
+    });
+
+    it("encodes object-only content with projection", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        object: { preference: "coffee", shots: 2 },
+      };
+      const result = await encoder.encode(byte);
+
+      // text falls back to objtext projection
+      expect(result.text).toContain("preference: coffee");
+      expect(result.objtext).toContain("preference: coffee");
+      expect(result.tvec).toBeDefined();
+    });
+
+    it("combines text and object for embedding", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        text: "Tony likes coffee",
+        object: { shots: 2, sugar: false },
+      };
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBe("Tony likes coffee");
+      expect(result.objtext).toContain("shots: 2");
+
+      // embedding should be called with combined text
+      const embedCall = (embedder.embed as ReturnType<typeof vi.fn>).mock
+        .calls[0][0];
+      expect(embedCall.values[0]).toContain("Tony likes coffee");
+      expect(embedCall.values[0]).toContain("shots: 2");
+    });
+
+    it("does not include metadata (lives in primary DB only)", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        text: "test",
+        object: { key: "value" },
+      };
+      const result = await encoder.encode(byte);
+
+      // metadata is not part of IndexableByte - it stays in the primary DB
+      expect("metadata" in result).toBe(false);
+    });
+
+    it("returns undefined tvec when no content", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {};
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBeUndefined();
+      expect(result.objtext).toBeUndefined();
+      expect(result.tvec).toBeUndefined();
+      expect(embedder.embed).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("embed", () => {
+    it("exposes embed method for query embedding", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const vec = await encoder.embed("search query");
+
+      expect(vec).toEqual([12, 0, 0]); // "search query".length = 12
+    });
+  });
+});

package/src/memory/codecs/domain.ts

@@ -58,6 +58,12 @@ export const MEMORY_FILTER: Codec<MemoryFilter, SearchFilter> = {
 
 /**
  * Create a codec for MemoryRecord -> IndexMemoryRecord.
+ *
+ * Combines:
+ * - Record scope/timestamps from MemoryRecord
+ * - Indexed content (text, object projection, embeddings) from byte codec
+ *
+ * Note: metadata is NOT included - it lives in the primary DB only.
  */
 export function recordCodec(
   bytecodec: MemoryByteCodec,

package/src/memory/codecs/tpuf.ts

@@ -29,11 +29,8 @@ import type { IndexMemoryRecord } from "../types";
  */
 export const TPUF_DOC: Codec<IndexMemoryRecord, UnknownDocument> = {
   encode(doc: IndexMemoryRecord): UnknownDocument {
-    const { tvec, ivec, avec, vvec, metadata, ...rest } = doc;
-    const row: UnknownDocument = {
-      ...rest,
-      metadata: metadata as UnknownDocument["metadata"], // metadata is JSONObject | null, cast to FieldValue for UnknownDocument
-    };
+    const { tvec, ivec, avec, vvec, ...rest } = doc;
+    const row: UnknownDocument = { ...rest };
     if (tvec) row.vector = tvec;
     return row;
   },

package/src/memory/encoder.ts

@@ -2,10 +2,41 @@
  * MemoryByte encoder - converts MemoryByte to IndexableByte with embeddings.
  */
 
-import type { EmbeddingModel } from "@kernl-sdk/protocol";
+import type { EmbeddingModel, JSONObject } from "@kernl-sdk/protocol";
+import { stringify as yamlStringify } from "yaml";
 
 import type { MemoryByte, IndexableByte, MemoryByteCodec } from "./types";
 
+// ---------------------
+// ObjectTextCodec
+// ---------------------
+
+const MAX_OBJECT_TEXT_LENGTH = 3000;
+
+/**
+ * Codec for converting JSONObject to a canonical text representation.
+ *
+ * Uses YAML for human-readable, deterministic output suitable for:
+ * - Full-text search indexing
+ * - Embedding input (combined with text field)
+ *
+ * TODO: Allow users to pass custom codec via MemoryOptions.
+ */
+export const ObjectTextCodec = {
+  /**
+   * Encode a JSONObject to canonical text.
+   * Uses YAML with sorted keys for determinism.
+   * Truncates at MAX_OBJECT_TEXT_LENGTH chars.
+   */
+  encode(obj: JSONObject): string {
+    const yaml = yamlStringify(obj, { sortMapEntries: true });
+    if (yaml.length <= MAX_OBJECT_TEXT_LENGTH) {
+      return yaml;
+    }
+    return yaml.slice(0, MAX_OBJECT_TEXT_LENGTH) + "\n...";
+  },
+};
+
 /**
  * Encoder that converts MemoryByte to IndexableByte.
  *
@@ -20,21 +51,35 @@ export class MemoryByteEncoder implements MemoryByteCodec {
 
   /**
    * Encode a MemoryByte to IndexableByte.
-   * Extracts text and computes embeddings for each modality.
+   *
+   * - Produces `objtext` string projection for FTS indexing
+   * - Combines text + objtext for embedding input
+   * - Returns text (fallback to objtext if no text provided)
+   *
+   * Note: metadata is NOT set here - it comes from record.metadata
+   * via the domain codec, not from MemoryByte.object.
    */
   async encode(byte: MemoryByte): Promise<IndexableByte> {
-    const text = byte.text;
-    const tvec = text ? await this.embed(text) : undefined;
+    const objtext = byte.object
+      ? ObjectTextCodec.encode(byte.object) // encode object as embeddable string
+      : undefined;
+
+    // (TODO): this behavior deserves consideration - do we always want to merge text + object?
+    //
+    // combine text + object for richer embedding
+    const combined = [byte.text, objtext].filter(Boolean).join("\n");
+    const tvec = combined ? await this.embed(combined) : undefined;
 
     // TODO: embed other modalities (image, audio, video)
+    //
     // const ivec = byte.image ? await this.embedImage(byte.image) : undefined;
     // const avec = byte.audio ? await this.embedAudio(byte.audio) : undefined;
     // const vvec = byte.video ? await this.embedVideo(byte.video) : undefined;
 
     return {
-      text,
+      text: byte.text ?? objtext, // fallback to projection if no text
+      objtext,
       tvec,
-      metadata: byte.object ?? null,
     };
   }
 

package/src/memory/index.ts

@@ -3,7 +3,7 @@
  */
 
 export { Memory } from "./memory";
-export { MemoryByteEncoder } from "./encoder";
+export { MemoryByteEncoder, ObjectTextCodec } from "./encoder";
 export { buildMemoryIndexSchema } from "./schema";
 export { MemoryIndexHandle } from "./handle";
 export type { MemoryIndexHandleConfig } from "./handle";

package/src/memory/schema.ts

@@ -105,6 +105,11 @@ export function buildMemoryIndexSchema(
       fts: true,
       optional: true,
     },
+    objtext: {
+      type: "string",
+      fts: true,
+      optional: true,
+    },
 
     // vector fields for different modalities
     tvec: {
@@ -131,11 +136,6 @@ export function buildMemoryIndexSchema(
       similarity,
       optional: true,
     },
-
-    metadata: {
-      type: "object",
-      optional: true,
-    },
   };
 
   return schema;

package/src/memory/types.ts

@@ -60,14 +60,15 @@ export interface MemoryByte {
  * Search-ready projection of a MemoryByte.
  *
  * Contains canonical text plus embeddings for each modality.
+ * Note: metadata is NOT included - it lives in the primary DB only.
  */
 export interface IndexableByte {
   text?: string; // canonical semantic text
+  objtext?: string; // string projection of object for indexing (not full JSON)
   tvec?: number[]; // text embedding
   ivec?: number[]; // image embedding
   avec?: number[]; // audio embedding
   vvec?: number[]; // video embedding
-  metadata?: JSONObject | null; // structured payload
 }
 
 /**