effect-agent 0.1.0-beta.85 → 0.1.0-beta.87

package/src/capabilities/ModelContext.ts

@@ -0,0 +1,354 @@
+import { Crypto, Effect, Encoding, Schema } from "effect";
+import type { Prompt } from "effect/unstable/ai";
+
+import { ThreadId } from "../core/Identifiers.ts";
+import { utf8ByteLength } from "../core/internal/utf8.ts";
+import { ThreadMessage, Thread as ThreadSnapshot, ThreadText } from "../core/Thread.ts";
+
+const MAX_CONTEXT_MESSAGES = 1_024;
+const MAX_CONTEXT_MESSAGE_BYTES = 4 * 1024 * 1024;
+const MAX_SOURCE_SEQUENCES = 1_024;
+const MAX_RETAINED_FACTS = 256;
+const MAX_RETAINED_FACT_BYTES = 1024 * 1024;
+const MAX_COMPACTIONS = 16;
+
+const SourceSequences = Schema.Array(Schema.Natural).check(
+  Schema.isMaxLength(MAX_SOURCE_SEQUENCES),
+);
+
+/** A model-context message derived from, but never replacing, source history. */
+export class ModelContextMessage extends Schema.Class<ModelContextMessage>(
+  "@effect-agent/capabilities/ModelContextMessage",
+)({
+  role: Schema.Literals(["system", "user", "assistant", "tool"]),
+  content: ThreadText,
+  sourceSequences: SourceSequences,
+}) {}
+
+/** A fact retained explicitly by compaction, independent of the generated prose summary. */
+export class RetainedFact extends Schema.Class<RetainedFact>(
+  "@effect-agent/capabilities/RetainedFact",
+)({
+  fact: Schema.String.check(Schema.isMaxLength(4 * 1024)),
+  sourceSequences: SourceSequences,
+}) {}
+
+const ModelContextMessages = Schema.Array(ModelContextMessage)
+  .check(Schema.isMaxLength(MAX_CONTEXT_MESSAGES))
+  .pipe(
+    Schema.refine(
+      (messages): messages is ReadonlyArray<ModelContextMessage> =>
+        messages.reduce((total, message) => total + utf8ByteLength(message.content), 0) <=
+        MAX_CONTEXT_MESSAGE_BYTES,
+      { expected: `model context totaling at most ${MAX_CONTEXT_MESSAGE_BYTES} UTF-8 bytes` },
+    ),
+  );
+
+const RetainedFacts = Schema.Array(RetainedFact)
+  .check(Schema.isMaxLength(MAX_RETAINED_FACTS))
+  .pipe(
+    Schema.refine(
+      (facts): facts is ReadonlyArray<RetainedFact> =>
+        facts.reduce((total, retained) => total + utf8ByteLength(retained.fact), 0) <=
+        MAX_RETAINED_FACT_BYTES,
+      { expected: `retained facts totaling at most ${MAX_RETAINED_FACT_BYTES} UTF-8 bytes` },
+    ),
+  );
+
+const SourceDigest = Schema.String.check(Schema.isPattern(/^sha256:[a-f0-9]{64}$/));
+
+/** Versioned, replayable artifact that covers a prefix or middle range without deleting source. */
+export class CompactionArtifact extends Schema.Class<CompactionArtifact>(
+  "@effect-agent/capabilities/CompactionArtifact",
+)({
+  version: Schema.Literal(1),
+  threadId: ThreadId,
+  coversFrom: Schema.Natural,
+  coversThrough: Schema.Natural,
+  summary: ModelContextMessage,
+  retainedFacts: RetainedFacts,
+  tokenEstimate: Schema.Natural,
+  sourceDigest: SourceDigest,
+  compactorVersion: Schema.NonEmptyString,
+}) {}
+
+/** Context exposed to a model, with source history kept intact for audit and reconstruction. */
+export class PreparedModelContext extends Schema.Class<PreparedModelContext>(
+  "@effect-agent/capabilities/PreparedModelContext",
+)({
+  source: ThreadSnapshot,
+  messages: ModelContextMessages,
+  compactions: Schema.Array(CompactionArtifact).check(Schema.isMaxLength(MAX_COMPACTIONS)),
+}) {}
+
+/** Transform failure is named and bounded so model context preparation has a closed error channel. */
+export class ContextTransformError extends Schema.TaggedError<ContextTransformError>()(
+  "ContextTransformError",
+  { transformId: Schema.NonEmptyString, message: Schema.String },
+) {}
+
+/** Source encoding or hashing failed before an artifact could be trusted. */
+export class CompactionDigestError extends Schema.TaggedError<CompactionDigestError>()(
+  "CompactionDigestError",
+  { message: Schema.String },
+) {}
+
+/** Compaction artifacts that do not bind to the exact source range are rejected before use. */
+export class InvalidCompactionArtifact extends Schema.TaggedError<InvalidCompactionArtifact>()(
+  "InvalidCompactionArtifact",
+  { message: Schema.String },
+) {}
+
+/** A transform or compaction exceeded the finite model-view collection bounds. */
+export class ContextLimitExceeded extends Schema.TaggedError<ContextLimitExceeded>()(
+  "ContextLimitExceeded",
+  {
+    limit: Schema.Literals(["messages", "message-bytes", "compactions"]),
+    limitValue: Schema.Natural,
+    observedValue: Schema.Natural,
+  },
+) {}
+
+/**
+ * Ordered context transform. It receives and returns only the model-visible
+ * message view, making replacement of the authoritative source impossible.
+ * Required services remain visible in prepareModelContext; expected failures
+ * are translated by the transform into ContextTransformError.
+ */
+export interface ContextTransform<Requirements = never> {
+  readonly id: string;
+  readonly version: string;
+  readonly apply: (
+    messages: ReadonlyArray<ModelContextMessage>,
+  ) => Effect.Effect<ReadonlyArray<ModelContextMessage>, ContextTransformError, Requirements>;
+}
+
+const messageText = (message: Prompt.Message): string => {
+  if (message.role === "system") return message.content;
+
+  const content = message.content
+    .map((part) => {
+      if (part.type === "text" || part.type === "reasoning") return part.text;
+
+      return `[${part.type}]`;
+    })
+    .join("\n");
+
+  return content.slice(0, 64 * 1024);
+};
+
+const asModelMessages = (snapshot: ThreadSnapshot): ReadonlyArray<ModelContextMessage> =>
+  snapshot.messages.map((entry) =>
+    ModelContextMessage.make({
+      role: entry.message.role,
+      content: messageText(entry.message),
+      sourceSequences: [entry.sequence],
+    }),
+  );
+
+const validateModelView = (
+  messages: ReadonlyArray<ModelContextMessage>,
+  transformId: string,
+): Effect.Effect<ReadonlyArray<ModelContextMessage>, ContextTransformError> => {
+  if (messages.length > MAX_CONTEXT_MESSAGES) {
+    return Effect.fail(
+      ContextTransformError.make({
+        transformId,
+        message: `Transform produced ${messages.length} messages; maximum is ${MAX_CONTEXT_MESSAGES}`,
+      }),
+    );
+  }
+  const bytes = messages.reduce((total, message) => total + utf8ByteLength(message.content), 0);
+
+  return bytes > MAX_CONTEXT_MESSAGE_BYTES
+    ? Effect.fail(
+        ContextTransformError.make({
+          transformId,
+          message: `Transform produced ${bytes} UTF-8 bytes; maximum is ${MAX_CONTEXT_MESSAGE_BYTES}`,
+        }),
+      )
+    : Effect.succeed(messages);
+};
+
+/** Apply transforms in declaration order, always restoring the original source snapshot. */
+export const prepareModelContext = <
+  const Requirements extends ReadonlyArray<unknown> = readonly [],
+>(
+  snapshot: ThreadSnapshot,
+  transforms?: { readonly [Index in keyof Requirements]: ContextTransform<Requirements[Index]> },
+): Effect.Effect<PreparedModelContext, ContextTransformError, Requirements[number]> =>
+  (transforms ?? [])
+    .reduce<
+      Effect.Effect<ReadonlyArray<ModelContextMessage>, ContextTransformError, Requirements[number]>
+    >(
+      (messages, transform) =>
+        messages.pipe(
+          Effect.flatMap(transform.apply),
+          Effect.flatMap((transformed) => validateModelView(transformed, transform.id)),
+        ),
+      Effect.succeed(asModelMessages(snapshot)),
+    )
+    .pipe(
+      Effect.map((messages) =>
+        PreparedModelContext.make({
+          source: snapshot,
+          messages,
+          compactions: [],
+        }),
+      ),
+    );
+
+const exactSourceRange = (
+  snapshot: ThreadSnapshot,
+  coversFrom: number,
+  coversThrough: number,
+): Effect.Effect<ReadonlyArray<ThreadMessage>, InvalidCompactionArtifact> => {
+  if (coversFrom > coversThrough || coversThrough >= snapshot.nextSequence) {
+    return Effect.fail(
+      InvalidCompactionArtifact.make({
+        message: "Compaction artifact covers an invalid source range",
+      }),
+    );
+  }
+
+  const selected = snapshot.messages.filter(
+    (message) => message.sequence >= coversFrom && message.sequence <= coversThrough,
+  );
+
+  const expectedCount = coversThrough - coversFrom + 1;
+
+  if (
+    selected.length !== expectedCount ||
+    selected[0]?.sequence !== coversFrom ||
+    selected[selected.length - 1]?.sequence !== coversThrough
+  ) {
+    return Effect.fail(
+      InvalidCompactionArtifact.make({
+        message: "Compaction source range is not contiguous in authoritative history",
+      }),
+    );
+  }
+
+  return Effect.succeed(selected);
+};
+
+const utf8Bytes = (value: string): Uint8Array => {
+  const hex = Encoding.encodeHex(value);
+  const bytes = new Uint8Array(hex.length / 2);
+
+  for (let index = 0; index < bytes.length; index += 1) {
+    bytes[index] = Number.parseInt(hex.slice(index * 2, index * 2 + 2), 16);
+  }
+
+  return bytes;
+};
+
+/** Compute a cryptographic digest over the canonical encoded exact source range. */
+export const digestCompactionSource = Effect.fn("digestCompactionSource")(function* (
+  snapshot: ThreadSnapshot,
+  coversFrom: number,
+  coversThrough: number,
+) {
+  const selected = yield* exactSourceRange(snapshot, coversFrom, coversThrough);
+
+  const encoded = yield* Schema.encodeEffect(Schema.Array(ThreadMessage))(selected).pipe(
+    Effect.mapError((error) =>
+      CompactionDigestError.make({
+        message: `Could not encode compaction source: ${error.message}`,
+      }),
+    ),
+  );
+
+  const crypto = yield* Crypto.Crypto;
+
+  const digest = yield* crypto.digest("SHA-256", utf8Bytes(JSON.stringify(encoded))).pipe(
+    Effect.mapError((error) =>
+      CompactionDigestError.make({
+        message: `Could not digest compaction source: ${error.message}`,
+      }),
+    ),
+  );
+
+  return `sha256:${Encoding.encodeHex(digest)}`;
+});
+
+/**
+ * Replace only the model-view messages fully covered by the artifact range
+ * with its summary. The digest is recomputed against the exact source slice
+ * before the artifact can influence context. Messages that are not fully
+ * covered — including transform-synthesized messages without provenance and
+ * messages whose provenance only partially overlaps the range — remain
+ * visible, and the summary is inserted before the first message derived
+ * entirely from later source.
+ */
+export const applyCompaction = Effect.fn("applyCompaction")(function* (
+  context: PreparedModelContext,
+  artifact: CompactionArtifact,
+) {
+  if (artifact.threadId !== context.source.threadId) {
+    return yield* InvalidCompactionArtifact.make({
+      message: "Compaction artifact belongs to another Thread",
+    });
+  }
+
+  const actualDigest = yield* digestCompactionSource(
+    context.source,
+    artifact.coversFrom,
+    artifact.coversThrough,
+  );
+
+  if (actualDigest !== artifact.sourceDigest) {
+    return yield* InvalidCompactionArtifact.make({
+      message: "Compaction source digest does not match authoritative history",
+    });
+  }
+
+  const provenanceInRange = (sequences: ReadonlyArray<number>): boolean =>
+    sequences.length > 0 &&
+    sequences.every(
+      (sequence) => sequence >= artifact.coversFrom && sequence <= artifact.coversThrough,
+    );
+
+  if (
+    !provenanceInRange(artifact.summary.sourceSequences) ||
+    artifact.retainedFacts.some((fact) => !provenanceInRange(fact.sourceSequences))
+  ) {
+    return yield* InvalidCompactionArtifact.make({
+      message: "Compaction summary or retained-fact provenance falls outside its covered range",
+    });
+  }
+  if (context.compactions.length >= MAX_COMPACTIONS) {
+    return yield* ContextLimitExceeded.make({
+      limit: "compactions",
+      limitValue: MAX_COMPACTIONS,
+      observedValue: context.compactions.length + 1,
+    });
+  }
+
+  const fullyCovered = (message: ModelContextMessage): boolean =>
+    message.sourceSequences.length > 0 &&
+    message.sourceSequences.every(
+      (sequence) => sequence >= artifact.coversFrom && sequence <= artifact.coversThrough,
+    );
+
+  const kept = context.messages.filter((message) => !fullyCovered(message));
+
+  const summaryAt = kept.findIndex(
+    (message) =>
+      message.sourceSequences.length > 0 &&
+      message.sourceSequences.every((sequence) => sequence > artifact.coversThrough),
+  );
+
+  const messages = yield* validateModelView(
+    summaryAt === -1
+      ? [...kept, artifact.summary]
+      : [...kept.slice(0, summaryAt), artifact.summary, ...kept.slice(summaryAt)],
+    `compaction:${artifact.compactorVersion}`,
+  );
+
+  return PreparedModelContext.make({
+    source: context.source,
+    messages,
+    compactions: [...context.compactions, artifact],
+  });
+});

package/src/capabilities/Redaction.ts

@@ -0,0 +1,157 @@
+import { Context, Effect, Layer, Schema } from "effect";
+
+import { utf8ByteLength } from "../core/internal/utf8.ts";
+import { RunEvent } from "../core/RunEvent.ts";
+
+const MAX_REDACTED_PREVIEW_BYTES = 8 * 1024;
+const MAX_REDACTION_NODES = 4_096;
+const MAX_REDACTION_DEPTH = 16;
+
+/** Branded evidence that a preview passed through the configured structural Redactor. */
+export const RedactedPreview = Schema.String.pipe(
+  Schema.refine((value): value is string => utf8ByteLength(value) <= MAX_REDACTED_PREVIEW_BYTES, {
+    expected: `redacted preview of at most ${MAX_REDACTED_PREVIEW_BYTES} UTF-8 bytes`,
+  }),
+  Schema.brand("@effect-agent/capabilities/RedactedPreview"),
+);
+
+export type RedactedPreview = typeof RedactedPreview.Type;
+
+/** Structural redaction failed closed rather than returning unreviewed caller text. */
+export class RedactionError extends Schema.TaggedError<RedactionError>()("RedactionError", {
+  reason: Schema.Literals(["input-too-deep", "input-too-large", "encoding-failed"]),
+  message: Schema.String,
+}) {}
+
+/** Redaction owns the boundary from decoded Tool values to bounded audit/model previews. */
+export class Redactor extends Context.Service<
+  Redactor,
+  {
+    readonly redact: (decodedValue: unknown) => Effect.Effect<RedactedPreview, RedactionError>;
+  }
+>()("@effect-agent/capabilities/Redactor") {}
+
+interface RedactionCounter {
+  nodes: number;
+}
+
+const redactValue = (value: unknown, depth: number, counter: RedactionCounter): Schema.Json => {
+  if (depth > MAX_REDACTION_DEPTH) {
+    throw RedactionError.make({
+      reason: "input-too-deep",
+      message: `Decoded value exceeds redaction depth ${MAX_REDACTION_DEPTH}`,
+    });
+  }
+  counter.nodes += 1;
+  if (counter.nodes > MAX_REDACTION_NODES) {
+    throw RedactionError.make({
+      reason: "input-too-large",
+      message: `Decoded value exceeds redaction node count ${MAX_REDACTION_NODES}`,
+    });
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => redactValue(item, depth + 1, counter));
+  }
+  if (value !== null && typeof value === "object") {
+    const redacted: Record<string, Schema.Json> = {};
+
+    for (const [key, item] of Object.entries(value)) {
+      redacted[key] = redactValue(item, depth + 1, counter);
+    }
+
+    return redacted;
+  }
+  // The default policy is deliberately type-aware and fail-closed: it preserves
+  // object/array shape for reviewer context but never copies a decoded scalar,
+  // regardless of its field name. Applications that need visible values provide
+  // a Tool-specific Redactor after validating those values against that Tool's
+  // input Schema.
+  if (value === null) return "[REDACTED:null]";
+  if (typeof value === "string") return "[REDACTED:string]";
+  if (typeof value === "number") return "[REDACTED:number]";
+  if (typeof value === "boolean") return "[REDACTED:boolean]";
+
+  return "[REDACTED:unsupported]";
+};
+
+const truncateUtf8 = (value: string, maxBytes: number): string => {
+  if (utf8ByteLength(value) <= maxBytes) {
+    return value;
+  }
+  const suffix = "…";
+  const suffixBytes = utf8ByteLength(suffix);
+  let output = "";
+  let bytes = 0;
+
+  for (const character of value) {
+    const characterBytes = utf8ByteLength(character);
+
+    if (bytes + characterBytes + suffixBytes > maxBytes) {
+      break;
+    }
+    output += character;
+    bytes += characterBytes;
+  }
+
+  return `${output}${suffix}`;
+};
+
+const encodeRunEvent = Schema.encodeEffect(RunEvent);
+
+/**
+ * One composed step from live Run Events to structurally redacted transcript lines
+ * (P7 WP7 friction fix; SEC-008/testing.md §12 "structurally redacted" live transcripts):
+ * Schema-encode each event through the canonical `RunEvent` union, then pass the encoded
+ * value through the configured `Redactor` — so the safe path is the short path and no live
+ * suite hand-assembles (or accidentally skips) the encode→redact pair. Engine-constructed
+ * events always encode, so an encode failure is a defect, never a silent omission.
+ */
+export const redactedTranscript = (
+  events: Iterable<RunEvent>,
+): Effect.Effect<ReadonlyArray<RedactedPreview>, RedactionError, Redactor> =>
+  Effect.gen(function* () {
+    const redactor = yield* Redactor;
+    const lines: Array<RedactedPreview> = [];
+
+    for (const event of events) {
+      const encoded = yield* encodeRunEvent(event).pipe(Effect.orDie);
+
+      lines.push(yield* redactor.redact(encoded));
+    }
+
+    return lines;
+  });
+
+/**
+ * Default structural implementation. It exposes only collection shape and
+ * scalar types, so an ordinary field such as `note` cannot bypass redaction.
+ * Tool-specific Layers may expose explicitly reviewed fields instead.
+ */
+export const StructuralRedactorLive = Layer.succeed(Redactor)({
+  redact: (decodedValue) =>
+    Effect.try({
+      try: () => {
+        const structurallyRedacted = redactValue(decodedValue, 0, { nodes: 0 });
+
+        return truncateUtf8(JSON.stringify(structurallyRedacted), MAX_REDACTED_PREVIEW_BYTES);
+      },
+      catch: (cause) =>
+        Schema.is(RedactionError)(cause)
+          ? cause
+          : RedactionError.make({
+              reason: "encoding-failed",
+              message: "Could not encode the structurally redacted preview",
+            }),
+    }).pipe(
+      Effect.flatMap((preview) =>
+        Schema.decodeEffect(RedactedPreview)(preview).pipe(
+          Effect.mapError(() =>
+            RedactionError.make({
+              reason: "encoding-failed",
+              message: "Redacted preview did not satisfy its bounded schema",
+            }),
+          ),
+        ),
+      ),
+    ),
+});