effect-agent 0.1.0-beta.85 → 0.1.0-beta.87

package/src/durable/ThreadContextHistory.ts

@@ -0,0 +1,227 @@
+import { Effect, Layer, Schema, Stream } from "effect";
+
+import type { ThreadId } from "../core/Identifiers.ts";
+import {
+  ContextHistory,
+  ContextHistoryError,
+  ContextHistoryHit,
+  ContextHistoryPage,
+  ContextHistoryRead,
+  ContextHistorySearch,
+} from "../engine/ContextHistory.ts";
+import { CanonicalRecordEnvelope, CanonicalSequence } from "./Records.ts";
+import type { ContextHistoryBoundary } from "./ThreadContextHistoryProjection.ts";
+import {
+  ContextHistoryEvidence,
+  project,
+  normalizeQuery,
+  matchText,
+  windowIdFor,
+} from "./ThreadContextHistoryProjection.ts";
+import { ThreadRead, ThreadStore, ThreadTail, ThreadTailRequest } from "./ThreadStore.ts";
+
+/** Bounds each lookup; a larger archive needs an explicitly configured or indexed adapter. */
+export const ThreadContextHistoryOptions = Schema.Struct({
+  maxRecords: Schema.optionalKey(
+    Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65_536 })),
+  ),
+  timeoutMillis: Schema.optionalKey(
+    Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 300_000 })),
+  ),
+});
+
+export type ThreadContextHistoryOptions = typeof ThreadContextHistoryOptions.Type;
+
+const unavailable = () =>
+  ContextHistoryError.make({
+    reason: "unavailable",
+    message: "Canonical context history is unavailable",
+  });
+
+const invalid = (message: string) => ContextHistoryError.make({ reason: "invalid-input", message });
+
+/**
+ * Provides bounded lexical search and paged reads over retained canonical transcript evidence.
+ * Each operation captures one tail and scans at most `maxRecords` (default 16,384), in pages of
+ * 64, with a `timeoutMillis` deadline (default 10 seconds). An oversized history fails explicitly
+ * instead of returning an incomplete search. No index, mutable archive, or background work is created.
+ * Search resolves `beforeRecordId` against eligible canonical evidence in that same scan and
+ * returns only older matches. It still verifies the entire captured tail and its boundaries.
+ *
+ * Rollover coverage boundaries assign subsequent records to the new window, including later Runs.
+ * Before the first rollover, each Run uses its initial `context:<runId>:0` identity. Pruning and
+ * summarization never remove searchable source records. Tool results remain exactly as retained,
+ * including any truncation envelope; discarded output bytes and transient recall are unavailable.
+ *
+ * The host must supply an authorized ThreadStore. This adapter does not grant access based on a
+ * Thread ID. Context Tools select the current Run's Thread, never a model-selected Thread.
+ */
+export const layer = (
+  options: ThreadContextHistoryOptions = {},
+): Layer.Layer<ContextHistory, ContextHistoryError, ThreadStore> =>
+  Layer.effect(
+    ContextHistory,
+    Effect.gen(function* () {
+      const decoded = yield* Schema.decodeEffect(ThreadContextHistoryOptions)(options).pipe(
+        Effect.mapError(() => invalid("Invalid context history limits")),
+      );
+
+      const maxRecords = decoded.maxRecords ?? 16_384;
+      const timeoutMillis = decoded.timeoutMillis ?? 10_000;
+      const store = yield* ThreadStore;
+
+      const scan = Effect.fn("ThreadContextHistory.scan")(function* (
+        threadId: ThreadId,
+        visit: (record: CanonicalRecordEnvelope) => Effect.Effect<void, ContextHistoryError>,
+      ) {
+        const tail = yield* store.inspectTail(ThreadTailRequest.make({ threadId })).pipe(
+          Effect.mapError(unavailable),
+          Effect.flatMap((value) =>
+            Schema.decodeEffect(Schema.toType(ThreadTail))(value).pipe(
+              Effect.mapError(unavailable),
+            ),
+          ),
+        );
+
+        if (tail.threadId !== threadId) return yield* unavailable();
+        if (tail.tailSequence > maxRecords)
+          return yield* ContextHistoryError.make({
+            reason: "limit",
+            message: `Context history exceeds the configured ${maxRecords} record scan limit`,
+          });
+        const boundaries: Array<ContextHistoryBoundary> = [];
+        let cursor = Schema.decodeSync(CanonicalSequence)(0);
+
+        while (cursor < tail.tailSequence) {
+          const limit = Math.min(64, tail.tailSequence - cursor);
+
+          const page = yield* store
+            .read(ThreadRead.make({ threadId, afterSequence: cursor, limit }))
+            .pipe(Stream.take(limit + 1), Stream.runCollect, Effect.mapError(unavailable));
+
+          if (page.length !== limit) return yield* unavailable();
+          for (const raw of page) {
+            const record = yield* Schema.decodeEffect(Schema.toType(CanonicalRecordEnvelope))(
+              raw,
+            ).pipe(Effect.mapError(unavailable));
+
+            if (record.threadId !== threadId || record.sequence !== cursor + 1)
+              return yield* unavailable();
+            if (record.record.payload._tag === "CompactionCreated") {
+              const { boundary } = yield* project(record);
+
+              if (boundary !== undefined) {
+                if (boundary.coversThrough < (boundaries.at(-1)?.coversThrough ?? 0))
+                  return yield* unavailable();
+                boundaries.push(boundary);
+              }
+            }
+            yield* visit(record);
+            cursor = record.sequence;
+          }
+        }
+
+        return boundaries;
+      });
+
+      const search = Effect.fn("ThreadContextHistory.search")(
+        function* (input: ContextHistorySearch) {
+          const request = yield* Schema.decodeEffect(Schema.toType(ContextHistorySearch))(
+            input,
+          ).pipe(Effect.mapError(() => invalid("Invalid context history search")));
+
+          const query = yield* normalizeQuery(request.query);
+          const matches: Array<ContextHistoryEvidence> = [];
+          let anchorFound = false;
+
+          const boundaries = yield* scan(
+            request.threadId,
+            Effect.fn(function* (record) {
+              const item = (yield* project(record)).evidence;
+
+              if (item === undefined) return;
+              if (item.recordId === request.beforeRecordId) anchorFound = true;
+              // The scan is ascending: retain older candidates until the exclusive anchor.
+              // Continue scanning to verify the captured tail and all window boundaries.
+              if (anchorFound) return;
+              const text = matchText(item.text, query);
+
+              if (text === undefined) return;
+              matches.push(ContextHistoryEvidence.make({ ...item, text }));
+              if (matches.length > request.limit) matches.shift();
+            }),
+          );
+
+          if (request.beforeRecordId !== undefined && !anchorFound)
+            return yield* ContextHistoryError.make({
+              reason: "not-found",
+              message: "Retained context record was not found in this Thread",
+            });
+
+          return matches.reverse().map((item) =>
+            ContextHistoryHit.make({
+              recordId: item.recordId,
+              windowId: windowIdFor(item, boundaries),
+              text: item.text,
+            }),
+          );
+        },
+        Effect.timeoutOrElse({
+          duration: timeoutMillis,
+          orElse: () =>
+            Effect.fail(
+              ContextHistoryError.make({
+                reason: "limit",
+                message: "Context history search exceeded its time limit",
+              }),
+            ),
+        }),
+      );
+
+      const read = Effect.fn("ThreadContextHistory.read")(
+        function* (input: ContextHistoryRead) {
+          const request = yield* Schema.decodeEffect(Schema.toType(ContextHistoryRead))(input).pipe(
+            Effect.mapError(() => invalid("Invalid context history read")),
+          );
+
+          let selected: ContextHistoryEvidence | undefined;
+
+          const boundaries = yield* scan(
+            request.threadId,
+            Effect.fn(function* (record) {
+              if (record.record.recordId === request.recordId)
+                selected = (yield* project(record)).evidence;
+            }),
+          );
+
+          if (selected === undefined)
+            return yield* ContextHistoryError.make({
+              reason: "not-found",
+              message: "Retained context record was not found in this Thread",
+            });
+          if (request.offset > selected.text.length)
+            return yield* invalid("Context history offset is beyond the retained record");
+          const end = Math.min(selected.text.length, request.offset + request.maxChars);
+
+          return ContextHistoryPage.make({
+            recordId: selected.recordId,
+            windowId: windowIdFor(selected, boundaries),
+            text: selected.text.slice(request.offset, end),
+            nextOffset: end < selected.text.length ? end : null,
+          });
+        },
+        Effect.timeoutOrElse({
+          duration: timeoutMillis,
+          orElse: () =>
+            Effect.fail(
+              ContextHistoryError.make({
+                reason: "limit",
+                message: "Context history read exceeded its time limit",
+              }),
+            ),
+        }),
+      );
+
+      return ContextHistory.of({ search, read });
+    }),
+  );

package/src/durable/ThreadContextHistoryProjection.ts

@@ -0,0 +1,216 @@
+import { Effect, Schema } from "effect";
+import { Prompt } from "effect/unstable/ai";
+
+import { RunId } from "../core/Identifiers.ts";
+import { contextWindowId } from "../engine/Compaction.ts";
+import { ContextHistoryError, ContextHistorySearch } from "../engine/ContextHistory.ts";
+import type { CanonicalRecordEnvelope } from "./Records.ts";
+import { CanonicalSequence, PersistedJson, RecordId } from "./Records.ts";
+
+/** Full retained recall text from one canonical record; never raw discarded Tool output. */
+export class ContextHistoryEvidence extends Schema.Class<ContextHistoryEvidence>(
+  "@effect-agent/thread/ContextHistoryEvidence",
+)({
+  recordId: RecordId,
+  sequence: CanonicalSequence,
+  runId: RunId,
+  text: Schema.NonEmptyString,
+}) {}
+
+/** The committing position and covered position are distinct; membership uses coversThrough. */
+export class ContextHistoryBoundary extends Schema.Class<ContextHistoryBoundary>(
+  "@effect-agent/thread/ContextHistoryBoundary",
+)({
+  sequence: CanonicalSequence,
+  coversThrough: CanonicalSequence,
+  windowId: Schema.NonEmptyString,
+}) {}
+
+/** An empty projection still consumes its canonical position in an index watermark. */
+export class ContextHistoryProjection extends Schema.Class<ContextHistoryProjection>(
+  "@effect-agent/thread/ContextHistoryProjection",
+)({
+  evidence: Schema.optionalKey(ContextHistoryEvidence),
+  boundary: Schema.optionalKey(ContextHistoryBoundary),
+}) {}
+
+const unavailable = () =>
+  ContextHistoryError.make({
+    reason: "unavailable",
+    message: "Canonical context history is unavailable",
+  });
+
+const jsonText = Effect.fn("ThreadContextHistoryProjection.jsonText")(function* (value: unknown) {
+  const json = yield* Schema.decodeUnknownEffect(PersistedJson)(value).pipe(
+    Effect.mapError(unavailable),
+  );
+
+  return JSON.stringify(json);
+});
+
+const promptText = Effect.fn("ThreadContextHistoryProjection.promptText")(function* (
+  value: PersistedJson,
+) {
+  const prompt = yield* Schema.decodeUnknownEffect(Prompt.Prompt)(value).pipe(
+    Effect.mapError(unavailable),
+  );
+
+  const messages: Array<string> = [];
+
+  for (const message of prompt.content) {
+    // Provider options, system instructions, reasoning, and attachment bytes are not recall data.
+    if (message.role === "system") continue;
+    const parts: Array<string> = [];
+
+    for (const part of message.content) {
+      switch (part.type) {
+        case "text":
+          parts.push(part.text);
+          break;
+        case "tool-call":
+          parts.push(`tool call ${part.name} (${part.id}):\n${yield* jsonText(part.params)}`);
+          break;
+        case "tool-result":
+          parts.push(
+            `tool result ${part.name} (${part.id}; ${part.isFailure ? "failure" : "success"}):\n${yield* jsonText(part.result)}`,
+          );
+          break;
+        case "file":
+          parts.push("[attachment omitted]");
+          break;
+        case "reasoning":
+        case "tool-approval-request":
+        case "tool-approval-response":
+          break;
+      }
+    }
+    if (parts.length > 0) messages.push(`${message.role}:\n${parts.join("\n")}`);
+  }
+
+  return messages.join("\n\n");
+});
+
+const retainedEvidence = Effect.fn("ThreadContextHistoryProjection.evidence")(function* (
+  envelope: CanonicalRecordEnvelope,
+): Effect.fn.Return<ContextHistoryEvidence | undefined, ContextHistoryError> {
+  const payload = envelope.record.payload;
+  let text: string;
+
+  if (payload._tag === "ModelResponseRecorded") {
+    text = yield* promptText(payload.messages);
+  } else if (payload._tag === "ModelCompleted") {
+    text =
+      payload.messages === undefined
+        ? `assistant output:\n${yield* jsonText(payload.output)}`
+        : yield* promptText(payload.messages);
+  } else if (payload._tag === "ToolCallSettled") {
+    text = `tool result ${payload.toolName} (${payload.toolCallId}; ${payload.isFailure ? "failure" : "success"}):\n${yield* jsonText(payload.result)}`;
+  } else {
+    // Raw submission input can contain fields excluded by the Agent's input projection.
+    // Step outputs, approvals, failure diagnostics, and operational records stay private.
+    return undefined;
+  }
+
+  return text.length === 0
+    ? undefined
+    : ContextHistoryEvidence.make({
+        recordId: envelope.record.recordId,
+        sequence: envelope.sequence,
+        runId: payload.runId,
+        text,
+      });
+});
+
+/**
+ * Project one decoded canonical record without reading storage or retaining mutable state.
+ * Only model transcript text and settled Tool results are evidence. Instructions, reasoning,
+ * attachment bytes, raw submissions, Steps, approvals, and diagnostics remain private. Tool
+ * truncation envelopes are rendered as retained; discarded bytes cannot be reconstructed.
+ *
+ * An index must process contiguous records, including empty projections, and commit its rows
+ * and watermark atomically. Validate nondecreasing coversThrough across rollover boundaries.
+ * Filter both evidence and committing boundary sequences to the operation's captured tail.
+ * Authorization and canonical verification of index candidates remain the host's responsibility.
+ * For search pagination, resolve and canonically verify `beforeRecordId` as eligible evidence
+ * in the same Thread and captured tail, then select matching evidence with sequence strictly
+ * below that anchor, in descending sequence order. The anchor need not match the new query.
+ * Do not restrict rollover boundaries to the anchor: later commits may assign older evidence
+ * to a window. An ID or an index row alone does not establish existence, eligibility, or access.
+ */
+export const project = Effect.fn("ThreadContextHistoryProjection.project")(function* (
+  envelope: CanonicalRecordEnvelope,
+): Effect.fn.Return<ContextHistoryProjection, ContextHistoryError> {
+  const payload = envelope.record.payload;
+
+  if (payload._tag === "CompactionCreated" && payload.kind === "rollover") {
+    if (payload.coversThrough >= envelope.sequence) return yield* unavailable();
+
+    return ContextHistoryProjection.make({
+      boundary: ContextHistoryBoundary.make({
+        sequence: envelope.sequence,
+        coversThrough: payload.coversThrough,
+        windowId: contextWindowId(payload.runId, payload.turn),
+      }),
+    });
+  }
+  const evidence = yield* retainedEvidence(envelope);
+
+  return ContextHistoryProjection.make(evidence === undefined ? {} : { evidence });
+});
+
+/** JavaScript case folding for literal substring lookup; do not substitute SQL lower or FTS. */
+export const normalizeText = (text: string): string => text.toLowerCase();
+
+/** Validate, trim, and case-fold a search query using the native history contract. */
+export const normalizeQuery = Effect.fn("ThreadContextHistoryProjection.normalizeQuery")(function* (
+  query: string,
+): Effect.fn.Return<string, ContextHistoryError> {
+  const decoded = yield* Schema.decodeEffect(ContextHistorySearch.fields.query)(query).pipe(
+    Effect.mapError(() =>
+      ContextHistoryError.make({
+        reason: "invalid-input",
+        message: "Invalid context history search",
+      }),
+    ),
+  );
+
+  const normalized = normalizeText(decoded.trim());
+
+  if (normalized.length === 0)
+    return yield* ContextHistoryError.make({
+      reason: "invalid-input",
+      message: "Context history search requires non-whitespace text",
+    });
+
+  return normalized;
+});
+
+/** Match a normalized query and retain the native 200-character lead and 2,000-character page. */
+export const matchText = (text: string, normalizedQuery: string): string | undefined => {
+  const index = normalizeText(text).indexOf(normalizedQuery);
+
+  if (index < 0) return undefined;
+  const start = Math.max(0, index - 200);
+
+  return text.slice(start, start + 2_000);
+};
+
+/**
+ * Boundaries must be in canonical order with nondecreasing coversThrough and belong to the
+ * same captured Thread tail as the evidence. An index may supply only its last boundary whose
+ * coversThrough is strictly less than evidence.sequence. Later Runs inherit that window;
+ * evidence before the first rollover keeps its own Run's initial identity.
+ */
+export const windowIdFor = (
+  evidence: ContextHistoryEvidence,
+  boundaries: ReadonlyArray<ContextHistoryBoundary>,
+): string => {
+  let windowId = contextWindowId(evidence.runId, 0);
+
+  for (const boundary of boundaries) {
+    if (evidence.sequence <= boundary.coversThrough) break;
+    windowId = boundary.windowId;
+  }
+
+  return windowId;
+};