kernl 0.7.4 → 0.8.0

package/src/memory/__tests__/encoder.test.ts

@@ -0,0 +1,153 @@
+import { describe, it, expect, vi } from "vitest";
+import type { EmbeddingModel } from "@kernl-sdk/protocol";
+
+import { MemoryByteEncoder, ObjectTextCodec } from "../encoder";
+import type { MemoryByte } from "../types";
+
+// Mock embedder that returns predictable vectors
+function createMockEmbedder(): EmbeddingModel<string> {
+  return {
+    provider: "test",
+    modelId: "test-embedder",
+    embed: vi.fn(async ({ values }: { values: string[] }) => ({
+      embeddings: values.map((v) => [v.length, 0, 0]), // simple: [length, 0, 0]
+    })),
+  } as unknown as EmbeddingModel<string>;
+}
+
+describe("ObjectTextCodec", () => {
+  it("encodes simple object to YAML", () => {
+    const obj = { name: "Tony", preference: "coffee" };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("name: Tony");
+    expect(result).toContain("preference: coffee");
+  });
+
+  it("sorts keys for determinism", () => {
+    const obj1 = { z: 1, a: 2, m: 3 };
+    const obj2 = { a: 2, m: 3, z: 1 };
+
+    expect(ObjectTextCodec.encode(obj1)).toBe(ObjectTextCodec.encode(obj2));
+  });
+
+  it("truncates long objects with ellipsis", () => {
+    const longValue = "x".repeat(4000);
+    const obj = { data: longValue };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result.length).toBeLessThanOrEqual(3005); // 3000 + "\n..."
+    expect(result.endsWith("\n...")).toBe(true);
+  });
+
+  it("handles nested objects", () => {
+    const obj = {
+      user: {
+        name: "Tony",
+        prefs: { coffee: { shots: 2 } },
+      },
+    };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("user:");
+    expect(result).toContain("name: Tony");
+    expect(result).toContain("shots: 2");
+  });
+
+  it("handles arrays", () => {
+    const obj = { items: ["a", "b", "c"] };
+    const result = ObjectTextCodec.encode(obj);
+
+    expect(result).toContain("items:");
+    expect(result).toContain("- a");
+  });
+});
+
+describe("MemoryByteEncoder", () => {
+  describe("encode", () => {
+    it("encodes text-only content", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = { text: "Hello world" };
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBe("Hello world");
+      expect(result.objtext).toBeUndefined();
+      expect(result.tvec).toBeDefined();
+      expect(embedder.embed).toHaveBeenCalledWith({ values: ["Hello world"] });
+    });
+
+    it("encodes object-only content with projection", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        object: { preference: "coffee", shots: 2 },
+      };
+      const result = await encoder.encode(byte);
+
+      // text falls back to objtext projection
+      expect(result.text).toContain("preference: coffee");
+      expect(result.objtext).toContain("preference: coffee");
+      expect(result.tvec).toBeDefined();
+    });
+
+    it("combines text and object for embedding", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        text: "Tony likes coffee",
+        object: { shots: 2, sugar: false },
+      };
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBe("Tony likes coffee");
+      expect(result.objtext).toContain("shots: 2");
+
+      // embedding should be called with combined text
+      const embedCall = (embedder.embed as ReturnType<typeof vi.fn>).mock
+        .calls[0][0];
+      expect(embedCall.values[0]).toContain("Tony likes coffee");
+      expect(embedCall.values[0]).toContain("shots: 2");
+    });
+
+    it("does not set metadata (handled by domain codec)", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {
+        text: "test",
+        object: { key: "value" },
+      };
+      const result = await encoder.encode(byte);
+
+      expect(result.metadata).toBeUndefined();
+    });
+
+    it("returns undefined tvec when no content", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const byte: MemoryByte = {};
+      const result = await encoder.encode(byte);
+
+      expect(result.text).toBeUndefined();
+      expect(result.objtext).toBeUndefined();
+      expect(result.tvec).toBeUndefined();
+      expect(embedder.embed).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("embed", () => {
+    it("exposes embed method for query embedding", async () => {
+      const embedder = createMockEmbedder();
+      const encoder = new MemoryByteEncoder(embedder);
+
+      const vec = await encoder.embed("search query");
+
+      expect(vec).toEqual([12, 0, 0]); // "search query".length = 12
+    });
+  });
+});

package/src/memory/codecs/domain.ts

@@ -58,6 +58,11 @@ export const MEMORY_FILTER: Codec<MemoryFilter, SearchFilter> = {
 
 /**
  * Create a codec for MemoryRecord -> IndexMemoryRecord.
+ *
+ * Combines:
+ * - Record scope/timestamps from MemoryRecord
+ * - Indexed content (text, object projection, embeddings) from byte codec
+ * - User metadata from record.metadata (not from content.object)
  */
 export function recordCodec(
   bytecodec: MemoryByteCodec,
@@ -76,6 +81,7 @@ export function recordCodec(
         createdAt: record.createdAt,
         updatedAt: record.updatedAt,
         ...indexable,
+        metadata: record.metadata ?? null, // user metadata, not content.object
       };
     },
     async decode(): Promise<MemoryRecord> {

package/src/memory/encoder.ts

@@ -2,10 +2,41 @@
  * MemoryByte encoder - converts MemoryByte to IndexableByte with embeddings.
  */
 
-import type { EmbeddingModel } from "@kernl-sdk/protocol";
+import type { EmbeddingModel, JSONObject } from "@kernl-sdk/protocol";
+import { stringify as yamlStringify } from "yaml";
 
 import type { MemoryByte, IndexableByte, MemoryByteCodec } from "./types";
 
+// ---------------------
+// ObjectTextCodec
+// ---------------------
+
+const MAX_OBJECT_TEXT_LENGTH = 3000;
+
+/**
+ * Codec for converting JSONObject to a canonical text representation.
+ *
+ * Uses YAML for human-readable, deterministic output suitable for:
+ * - Full-text search indexing
+ * - Embedding input (combined with text field)
+ *
+ * TODO: Allow users to pass custom codec via MemoryOptions.
+ */
+export const ObjectTextCodec = {
+  /**
+   * Encode a JSONObject to canonical text.
+   * Uses YAML with sorted keys for determinism.
+   * Truncates at MAX_OBJECT_TEXT_LENGTH chars.
+   */
+  encode(obj: JSONObject): string {
+    const yaml = yamlStringify(obj, { sortMapEntries: true });
+    if (yaml.length <= MAX_OBJECT_TEXT_LENGTH) {
+      return yaml;
+    }
+    return yaml.slice(0, MAX_OBJECT_TEXT_LENGTH) + "\n...";
+  },
+};
+
 /**
  * Encoder that converts MemoryByte to IndexableByte.
  *
@@ -20,21 +51,35 @@ export class MemoryByteEncoder implements MemoryByteCodec {
 
   /**
    * Encode a MemoryByte to IndexableByte.
-   * Extracts text and computes embeddings for each modality.
+   *
+   * - Produces `objtext` string projection for FTS indexing
+   * - Combines text + objtext for embedding input
+   * - Returns text (fallback to objtext if no text provided)
+   *
+   * Note: metadata is NOT set here - it comes from record.metadata
+   * via the domain codec, not from MemoryByte.object.
    */
   async encode(byte: MemoryByte): Promise<IndexableByte> {
-    const text = byte.text;
-    const tvec = text ? await this.embed(text) : undefined;
+    const objtext = byte.object
+      ? ObjectTextCodec.encode(byte.object) // encode object as embeddable string
+      : undefined;
+
+    // (TODO): this behavior deserves consideration - do we always want to merge text + object?
+    //
+    // combine text + object for richer embedding
+    const combined = [byte.text, objtext].filter(Boolean).join("\n");
+    const tvec = combined ? await this.embed(combined) : undefined;
 
     // TODO: embed other modalities (image, audio, video)
+    //
     // const ivec = byte.image ? await this.embedImage(byte.image) : undefined;
     // const avec = byte.audio ? await this.embedAudio(byte.audio) : undefined;
     // const vvec = byte.video ? await this.embedVideo(byte.video) : undefined;
 
     return {
-      text,
+      text: byte.text ?? objtext, // fallback to projection if no text
+      objtext,
       tvec,
-      metadata: byte.object ?? null,
     };
   }
 

package/src/memory/index.ts

@@ -3,7 +3,7 @@
  */
 
 export { Memory } from "./memory";
-export { MemoryByteEncoder } from "./encoder";
+export { MemoryByteEncoder, ObjectTextCodec } from "./encoder";
 export { buildMemoryIndexSchema } from "./schema";
 export { MemoryIndexHandle } from "./handle";
 export type { MemoryIndexHandleConfig } from "./handle";

package/src/memory/schema.ts

@@ -105,6 +105,11 @@ export function buildMemoryIndexSchema(
       fts: true,
       optional: true,
     },
+    objtext: {
+      type: "string",
+      fts: true,
+      optional: true,
+    },
 
     // vector fields for different modalities
     tvec: {

package/src/memory/types.ts

@@ -63,6 +63,7 @@ export interface MemoryByte {
  */
 export interface IndexableByte {
   text?: string; // canonical semantic text
+  objtext?: string; // string projection of object for indexing (not full JSON)
   tvec?: number[]; // text embedding
   ivec?: number[]; // image embedding
   avec?: number[]; // audio embedding

package/src/thread/__tests__/integration.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll } from "vitest";
 import { z } from "zod";
 import { openai } from "@ai-sdk/openai";
 import { AISDKLanguageModel } from "@kernl-sdk/ai";
+import "@kernl-sdk/ai/openai"; // (TMP)
 
 import { Agent } from "@/agent";
 import { Kernl } from "@/kernl";
@@ -23,21 +24,17 @@ import type { LanguageModelItem } from "@kernl-sdk/protocol";
 
 const SKIP_INTEGRATION_TESTS = !process.env.OPENAI_API_KEY;
 
-describe.skipIf(SKIP_INTEGRATION_TESTS)(
-  "Thread streaming integration",
-  () => {
-    let kernl: Kernl;
-    let model: AISDKLanguageModel;
-
-    beforeAll(() => {
-      kernl = new Kernl();
-      model = new AISDKLanguageModel(openai("gpt-4o"));
-    });
-
-    describe("stream()", () => {
-      it(
-        "should yield both delta events and complete items",
-        async () => {
+describe.skipIf(SKIP_INTEGRATION_TESTS)("Thread streaming integration", () => {
+  let kernl: Kernl;
+  let model: AISDKLanguageModel;
+
+  beforeAll(() => {
+    kernl = new Kernl();
+    model = new AISDKLanguageModel(openai("gpt-4.1"));
+  });
+
+  describe("stream()", () => {
+    it("should yield both delta events and complete items", async () => {
       const agent = new Agent({
         id: "test-stream",
         name: "Test Stream Agent",
@@ -103,13 +100,9 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
       // Should have finish event
       const finishEvents = events.filter((e) => e.kind === "finish");
       expect(finishEvents.length).toBe(1);
-        },
-        30000,
-      );
+    }, 30000);
 
-      it(
-        "should filter deltas from history but include complete items",
-        async () => {
+    it("should filter deltas from history but include complete items", async () => {
       const agent = new Agent({
         id: "test-history",
         name: "Test History Agent",
@@ -153,8 +146,12 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
       );
       expect(streamDeltas.length).toBeGreaterThan(0);
 
-      // History should contain the input message
-      expect(history[0]).toEqual(input[0]);
+      // History should contain the input message (with ThreadEvent headers added)
+      expect(history[0]).toMatchObject({
+        kind: "message",
+        role: "user",
+        content: [{ kind: "text", text: "Count to 3" }],
+      });
 
       // History should contain complete Message items
       const historyMessages = history.filter((e) => e.kind === "message");
@@ -170,11 +167,9 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
       );
       expect(textContent.text).toBeTruthy();
       expect(textContent.text.length).toBeGreaterThan(0);
-        },
-        30000,
-      );
+    }, 30000);
 
-      it("should work with tool calls", async () => {
+    it("should work with tool calls", async () => {
       const addTool = tool({
         id: "add",
         name: "add",
@@ -257,122 +252,118 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
 
       // Verify the assistant's final response references the correct answer
       const messages = events.filter((e) => e.kind === "message");
-      const assistantMessage = messages.find((m: any) => m.role === "assistant");
+      const assistantMessage = messages.find(
+        (m: any) => m.role === "assistant",
+      );
       expect(assistantMessage).toBeDefined();
       const textContent = (assistantMessage as any).content.find(
         (c: any) => c.kind === "text",
       );
       expect(textContent).toBeDefined();
       expect(textContent.text).toContain("42");
+    }, 30000);
+
+    it("should properly encode tool results with matching callIds for multi-turn", async () => {
+      const multiplyTool = tool({
+        id: "multiply",
+        name: "multiply",
+        description: "Multiply two numbers",
+        parameters: z.object({
+          a: z.number().describe("First number"),
+          b: z.number().describe("Second number"),
+        }),
+        execute: async (ctx, { a, b }) => {
+          return a * b;
         },
-        30000,
-      );
+      });
 
-      it(
-        "should properly encode tool results with matching callIds for multi-turn",
-        async () => {
-          const multiplyTool = tool({
-            id: "multiply",
-            name: "multiply",
-            description: "Multiply two numbers",
-            parameters: z.object({
-              a: z.number().describe("First number"),
-              b: z.number().describe("Second number"),
-            }),
-            execute: async (ctx, { a, b }) => {
-              return a * b;
-            },
-          });
-
-          const toolkit = new Toolkit({
-            id: "math",
-            tools: [multiplyTool],
-          });
-
-          const agent = new Agent({
-            id: "test-multi-turn",
-            name: "Test Multi-Turn Agent",
-            instructions: "You are a helpful assistant that can do math.",
-            model,
-            toolkits: [toolkit],
-          });
-
-          const input: LanguageModelItem[] = [
-            {
-              kind: "message",
-              id: "msg-1",
-              role: "user",
-              content: [{ kind: "text", text: "What is 7 times 6?" }],
-            },
-          ];
-
-          const thread = new Thread({ agent, input });
-          const events: ThreadStreamEvent[] = [];
-
-          // Collect all events from the stream
-          for await (const event of thread.stream()) {
-            events.push(event);
-          }
-
-          // Find the tool call and result
-          const toolCalls = events.filter(
-            (e): e is Extract<ThreadStreamEvent, { kind: "tool-call" }> =>
-              e.kind === "tool-call",
-          );
-          const toolResults = events.filter(
-            (e): e is Extract<ThreadStreamEvent, { kind: "tool-result" }> =>
-              e.kind === "tool-result",
-          );
-
-          expect(toolCalls.length).toBeGreaterThan(0);
-          expect(toolResults.length).toBeGreaterThan(0);
-
-          const multiplyCall = toolCalls[0];
-          const multiplyResult = toolResults[0];
-
-          // Verify callId matches between tool call and result
-          expect(multiplyCall.callId).toBe(multiplyResult.callId);
-          expect(multiplyCall.toolId).toBe("multiply");
-          expect(multiplyResult.toolId).toBe("multiply");
-
-          // Verify the tool result has the correct structure
-          expect(multiplyResult.callId).toBeDefined();
-          expect(typeof multiplyResult.callId).toBe("string");
-          expect(multiplyResult.callId.length).toBeGreaterThan(0);
-
-          // Verify history contains both with matching callIds
-          const history = (thread as any).history as ThreadEvent[];
-          const historyToolCall = history.find(
-            (e) => e.kind === "tool-call" && e.toolId === "multiply",
-          );
-          const historyToolResult = history.find(
-            (e) => e.kind === "tool-result" && e.toolId === "multiply",
-          );
-
-          expect(historyToolCall).toBeDefined();
-          expect(historyToolResult).toBeDefined();
-          expect((historyToolCall as any).callId).toBe(
-            (historyToolResult as any).callId,
-          );
-
-          // Verify final response uses the tool result
-          const messages = events.filter((e) => e.kind === "message");
-          const assistantMessage = messages.find((m: any) => m.role === "assistant");
-          expect(assistantMessage).toBeDefined();
-          const textContent = (assistantMessage as any).content.find(
-            (c: any) => c.kind === "text",
-          );
-          expect(textContent).toBeDefined();
-          expect(textContent.text).toContain("42");
+      const toolkit = new Toolkit({
+        id: "math",
+        tools: [multiplyTool],
+      });
+
+      const agent = new Agent({
+        id: "test-multi-turn",
+        name: "Test Multi-Turn Agent",
+        instructions: "You are a helpful assistant that can do math.",
+        model,
+        toolkits: [toolkit],
+      });
+
+      const input: LanguageModelItem[] = [
+        {
+          kind: "message",
+          id: "msg-1",
+          role: "user",
+          content: [{ kind: "text", text: "What is 7 times 6?" }],
         },
-        30000,
+      ];
+
+      const thread = new Thread({ agent, input });
+      const events: ThreadStreamEvent[] = [];
+
+      // Collect all events from the stream
+      for await (const event of thread.stream()) {
+        events.push(event);
+      }
+
+      // Find the tool call and result
+      const toolCalls = events.filter(
+        (e): e is Extract<ThreadStreamEvent, { kind: "tool-call" }> =>
+          e.kind === "tool-call",
+      );
+      const toolResults = events.filter(
+        (e): e is Extract<ThreadStreamEvent, { kind: "tool-result" }> =>
+          e.kind === "tool-result",
+      );
+
+      expect(toolCalls.length).toBeGreaterThan(0);
+      expect(toolResults.length).toBeGreaterThan(0);
+
+      const multiplyCall = toolCalls[0];
+      const multiplyResult = toolResults[0];
+
+      // Verify callId matches between tool call and result
+      expect(multiplyCall.callId).toBe(multiplyResult.callId);
+      expect(multiplyCall.toolId).toBe("multiply");
+      expect(multiplyResult.toolId).toBe("multiply");
+
+      // Verify the tool result has the correct structure
+      expect(multiplyResult.callId).toBeDefined();
+      expect(typeof multiplyResult.callId).toBe("string");
+      expect(multiplyResult.callId.length).toBeGreaterThan(0);
+
+      // Verify history contains both with matching callIds
+      const history = (thread as any).history as ThreadEvent[];
+      const historyToolCall = history.find(
+        (e) => e.kind === "tool-call" && e.toolId === "multiply",
+      );
+      const historyToolResult = history.find(
+        (e) => e.kind === "tool-result" && e.toolId === "multiply",
       );
-    });
 
-    describe("execute()", () => {
-      it(
-        "should consume stream and return final response",
-        async () => {
+      expect(historyToolCall).toBeDefined();
+      expect(historyToolResult).toBeDefined();
+      expect((historyToolCall as any).callId).toBe(
+        (historyToolResult as any).callId,
+      );
+
+      // Verify final response uses the tool result
+      const messages = events.filter((e) => e.kind === "message");
+      const assistantMessage = messages.find(
+        (m: any) => m.role === "assistant",
+      );
+      expect(assistantMessage).toBeDefined();
+      const textContent = (assistantMessage as any).content.find(
+        (c: any) => c.kind === "text",
+      );
+      expect(textContent).toBeDefined();
+      expect(textContent.text).toContain("42");
+    }, 30000);
+  });
+
+  describe("execute()", () => {
+    it("should consume stream and return final response", async () => {
       const agent = new Agent({
         id: "test-blocking",
         name: "Test Blocking Agent",
@@ -399,14 +390,10 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
 
       // Should have final state
       expect(result.state).toBe("stopped");
-        },
-        30000,
-      );
+    }, 30000);
 
-      it(
-        "should validate structured output in blocking mode",
-        async () => {
-      const responseSchema = z.object({
+    it("should validate structured output in blocking mode", async () => {
+      const PersonSchema = z.object({
         name: z.string(),
         age: z.number(),
       });
@@ -417,7 +404,7 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
         instructions:
           "You are a helpful assistant. Return JSON with name and age fields.",
         model,
-        responseType: responseSchema,
+        output: PersonSchema,
       });
 
       const input: LanguageModelItem[] = [
@@ -442,9 +429,6 @@ describe.skipIf(SKIP_INTEGRATION_TESTS)(
       expect(typeof result.response).toBe("object");
       expect((result.response as any).name).toBeTruthy();
       expect(typeof (result.response as any).age).toBe("number");
-        },
-        30000,
-      );
-    });
-  },
-);
+    }, 30000);
+  });
+});