effect 4.0.0-beta.32 → 4.0.0-beta.34

package/src/unstable/ai/LanguageModel.ts

@@ -51,7 +51,7 @@
 import type * as Cause from "../../Cause.ts"
 import * as Effect from "../../Effect.ts"
 import * as FiberSet from "../../FiberSet.ts"
-import { constFalse } from "../../Function.ts"
+import { constFalse, identity, pipe } from "../../Function.ts"
 import type * as JsonSchema from "../../JsonSchema.ts"
 import * as Option from "../../Option.ts"
 import * as Predicate from "../../Predicate.ts"
@@ -69,6 +69,7 @@ import { defaultIdGenerator, IdGenerator } from "./IdGenerator.ts"
 import * as InternalCodecTransformer from "./internal/codec-transformer.ts"
 import * as Prompt from "./Prompt.ts"
 import * as Response from "./Response.ts"
+import * as ResponseIdTracker from "./ResponseIdTracker.ts"
 import type { SpanTransformer } from "./Telemetry.ts"
 import { CurrentSpanTransformer } from "./Telemetry.ts"
 import type * as Tool from "./Tool.ts"
@@ -648,6 +649,16 @@ export interface ProviderOptions {
    * The span to use to trace interactions with the large language model.
    */
   readonly span: Span
+
+  /**
+   * The previous response identifier for incremental provider calls.
+   */
+  readonly previousResponseId: string | undefined
+
+  /**
+   * The prompt reduced to messages not yet seen by the provider.
+   */
+  readonly incrementalPrompt: Prompt.Prompt | undefined
 }
 
 /**
@@ -727,7 +738,9 @@ export const make: (params: {
             tools: [],
             toolChoice: "none",
             responseFormat: { type: "text" },
-            span
+            span,
+            previousResponseId: undefined,
+            incrementalPrompt: undefined
           }
           const content = yield* generateContent(options, providerOptions)
 
@@ -790,7 +803,9 @@ export const make: (params: {
               objectName,
               schema: options.schema
             },
-            span
+            span,
+            previousResponseId: undefined,
+            incrementalPrompt: undefined
           }
 
           const content = yield* generateContent(options, providerOptions)
@@ -857,7 +872,9 @@ export const make: (params: {
         tools: [],
         toolChoice: "none",
         responseFormat: { type: "text" },
-        span
+        span,
+        previousResponseId: undefined,
+        incrementalPrompt: undefined
       }
 
       // Resolve the content stream for the request
@@ -917,8 +934,23 @@ export const make: (params: {
     options: Options & GenerateTextOptions<Tools>,
     providerOptions: Mutable<ProviderOptions>
   ) {
+    const tracker = Option.getOrUndefined(yield* Effect.serviceOption(ResponseIdTracker.ResponseIdTracker))
     const toolChoice = options.toolChoice ?? "auto"
 
+    const withNonIncrementalFallback = <R>(
+      effect: Effect.Effect<Array<Response.PartEncoded>, AiError.AiError, R>
+    ): Effect.Effect<Array<Response.PartEncoded>, AiError.AiError, R | IdGenerator> =>
+      providerOptions.incrementalPrompt ?
+        effect.pipe(
+          Effect.catchReason("AiError", "InvalidRequestError", (_) =>
+            params.generateText({
+              ...providerOptions,
+              incrementalPrompt: undefined,
+              previousResponseId: undefined
+            }))
+        ) :
+        effect
+
     // Check for pending approvals that need resolution
     const { approved, denied } = collectToolApprovals(
       providerOptions.prompt.content,
@@ -940,11 +972,24 @@ export const make: (params: {
           })
         })
       }
+      if (tracker) {
+        const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+        if (Option.isSome(prepared)) {
+          providerOptions.previousResponseId = prepared.value.previousResponseId
+          providerOptions.incrementalPrompt = prepared.value.prompt
+        }
+      }
       const ResponseSchema = Schema.mutable(
         Schema.Array(Response.Part(Toolkit.empty))
       )
-      const rawContent = yield* params.generateText(providerOptions)
+      const rawContent = yield* withNonIncrementalFallback(params.generateText(providerOptions))
       const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent)
+      if (tracker) {
+        const responseMetadata = content.find((part) => part.type === "response-metadata")
+        if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
+          tracker.markParts(providerOptions.prompt.content, responseMetadata.id)
+        }
+      }
       return content as Array<Response.Part<Tools>>
     }
 
@@ -965,11 +1010,24 @@ export const make: (params: {
           })
         })
       }
+      if (tracker) {
+        const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+        if (Option.isSome(prepared)) {
+          providerOptions.previousResponseId = prepared.value.previousResponseId
+          providerOptions.incrementalPrompt = prepared.value.prompt
+        }
+      }
       const ResponseSchema = Schema.mutable(
         Schema.Array(Response.Part(Toolkit.empty))
       )
-      const rawContent = yield* params.generateText(providerOptions)
+      const rawContent = yield* withNonIncrementalFallback(params.generateText(providerOptions))
       const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent)
+      if (tracker) {
+        const responseMetadata = content.find((part) => part.type === "response-metadata")
+        if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
+          tracker.markParts(providerOptions.prompt.content, responseMetadata.id)
+        }
+      }
       return content as Array<Response.Part<Tools>>
     }
 
@@ -1025,6 +1083,14 @@ export const make: (params: {
     providerOptions.tools = tools
     providerOptions.toolChoice = toolChoice
 
+    if (tracker) {
+      const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+      if (Option.isSome(prepared)) {
+        providerOptions.previousResponseId = prepared.value.previousResponseId
+        providerOptions.incrementalPrompt = prepared.value.prompt
+      }
+    }
+
     // Construct the response schema with the tools from the toolkit
     const ResponseSchema = Schema.mutable(
       Schema.Array(Response.Part(toolkit))
@@ -1033,12 +1099,18 @@ export const make: (params: {
     // If tool call resolution is disabled, return the response without
     // resolving the tool calls that were generated
     if (options.disableToolCallResolution === true) {
-      const rawContent = yield* params.generateText(providerOptions)
+      const rawContent = yield* withNonIncrementalFallback(params.generateText(providerOptions))
       const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent)
+      if (tracker) {
+        const responseMetadata = content.find((part) => part.type === "response-metadata")
+        if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
+          tracker.markParts(providerOptions.prompt.content, responseMetadata.id)
+        }
+      }
       return content as Array<Response.Part<Tools>>
     }
 
-    const rawContent = yield* params.generateText(providerOptions)
+    const rawContent = yield* withNonIncrementalFallback(params.generateText(providerOptions))
 
     // Resolve the generated tool calls
     const toolResults = yield* resolveToolCalls(
@@ -1057,6 +1129,13 @@ export const make: (params: {
 
     const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent)
 
+    if (tracker) {
+      const responseMetadata = content.find((part) => part.type === "response-metadata")
+      if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
+        tracker.markParts(providerOptions.prompt.content, responseMetadata.id)
+      }
+    }
+
     // Return the content merged with the tool call results
     return [...content, ...toolResults] as Array<Response.Part<Tools>>
   })
@@ -1086,8 +1165,23 @@ export const make: (params: {
     options: Options & GenerateTextOptions<Tools>,
     providerOptions: Mutable<ProviderOptions>
   ) {
+    const tracker = Option.getOrUndefined(yield* Effect.serviceOption(ResponseIdTracker.ResponseIdTracker))
     const toolChoice = options.toolChoice ?? "auto"
 
+    const withNonIncrementalFallback = <R>(
+      stream: Stream.Stream<Response.StreamPartEncoded, AiError.AiError, R>
+    ): Stream.Stream<Response.StreamPartEncoded, AiError.AiError, R | IdGenerator> =>
+      providerOptions.incrementalPrompt ?
+        stream.pipe(
+          Stream.catchReason("AiError", "InvalidRequestError", (_) =>
+            params.streamText({
+              ...providerOptions,
+              incrementalPrompt: undefined,
+              previousResponseId: undefined
+            }))
+        ) :
+        stream
+
     // Check for pending approvals that need resolution
     const { approved: pendingApproved, denied: pendingDenied } = collectToolApprovals(providerOptions.prompt.content, {
       excludeResolved: true
@@ -1108,17 +1202,37 @@ export const make: (params: {
           })
         })
       }
+      if (tracker) {
+        const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+        if (Option.isSome(prepared)) {
+          providerOptions.previousResponseId = prepared.value.previousResponseId
+          providerOptions.incrementalPrompt = prepared.value.prompt
+        }
+      }
       const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty))
       const decodeParts = Schema.decodeEffect(schema)
-      return params
-        .streamText(providerOptions)
-        .pipe(
-          Stream.mapArrayEffect((parts) => decodeParts(parts))
-        ) as Stream.Stream<
-          Response.StreamPart<Tools>,
-          AiError.AiError | Schema.SchemaError,
-          IdGenerator
-        >
+      return pipe(
+        params.streamText(providerOptions),
+        withNonIncrementalFallback,
+        Stream.mapArrayEffect((parts) =>
+          decodeParts(parts).pipe(
+            tracker ?
+              Effect.tap((decodedParts) => {
+                for (const part of decodedParts) {
+                  if (part.type === "response-metadata" && Predicate.isNotUndefined(part.id)) {
+                    tracker.markParts(providerOptions.prompt.content, part.id)
+                  }
+                }
+                return Effect.void
+              }) :
+              identity
+          )
+        )
+      ) as Stream.Stream<
+        Response.StreamPart<Tools>,
+        AiError.AiError | Schema.SchemaError,
+        IdGenerator
+      >
     }
 
     // If there is a toolkit resolve and apply it to the provider options
@@ -1138,17 +1252,37 @@ export const make: (params: {
           })
         })
       }
+      if (tracker) {
+        const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+        if (Option.isSome(prepared)) {
+          providerOptions.previousResponseId = prepared.value.previousResponseId
+          providerOptions.incrementalPrompt = prepared.value.prompt
+        }
+      }
       const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty))
       const decodeParts = Schema.decodeEffect(schema)
-      return params
-        .streamText(providerOptions)
-        .pipe(
-          Stream.mapArrayEffect((parts) => decodeParts(parts))
-        ) as Stream.Stream<
-          Response.StreamPart<Tools>,
-          AiError.AiError | Schema.SchemaError,
-          IdGenerator
-        >
+      return pipe(
+        params.streamText(providerOptions),
+        withNonIncrementalFallback,
+        Stream.mapArrayEffect((parts) =>
+          decodeParts(parts).pipe(
+            tracker ?
+              Effect.tap((decodedParts) => {
+                for (const part of decodedParts) {
+                  if (part.type === "response-metadata" && part.id) {
+                    tracker.markParts(providerOptions.prompt.content, part.id)
+                  }
+                }
+                return Effect.void
+              }) :
+              identity
+          )
+        )
+      ) as Stream.Stream<
+        Response.StreamPart<Tools>,
+        AiError.AiError | Schema.SchemaError,
+        IdGenerator
+      >
     }
 
     // Pre-resolve pending tool approvals before calling the LLM
@@ -1222,20 +1356,40 @@ export const make: (params: {
     providerOptions.tools = tools
     providerOptions.toolChoice = toolChoice
 
+    if (tracker) {
+      const prepared = tracker.prepareUnsafe(providerOptions.prompt)
+      if (Option.isSome(prepared)) {
+        providerOptions.previousResponseId = prepared.value.previousResponseId
+        providerOptions.incrementalPrompt = prepared.value.prompt
+      }
+    }
+
     // If tool call resolution is disabled, return the response without
     // resolving the tool calls that were generated
     if (options.disableToolCallResolution === true) {
       const schema = Schema.NonEmptyArray(Response.StreamPart(toolkit))
       const decodeParts = Schema.decodeEffect(schema)
-      return params
-        .streamText(providerOptions)
-        .pipe(
-          Stream.mapArrayEffect((parts) => decodeParts(parts))
-        ) as Stream.Stream<
-          Response.StreamPart<Tools>,
-          AiError.AiError | Schema.SchemaError,
-          IdGenerator
-        >
+      return params.streamText(providerOptions).pipe(
+        withNonIncrementalFallback,
+        Stream.mapArrayEffect((parts) =>
+          decodeParts(parts).pipe(
+            tracker ?
+              Effect.tap((decodedParts) => {
+                for (const part of decodedParts) {
+                  if (part.type === "response-metadata" && Predicate.isNotUndefined(part.id)) {
+                    tracker.markParts(providerOptions.prompt.content, part.id)
+                  }
+                }
+                return Effect.void
+              }) :
+              identity
+          )
+        )
+      ) as Stream.Stream<
+        Response.StreamPart<Tools>,
+        AiError.AiError | Schema.SchemaError,
+        IdGenerator
+      >
     }
 
     const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart(toolkit))
@@ -1260,48 +1414,53 @@ export const make: (params: {
     const toolCallFibers = yield* FiberSet.make<void, AiError.AiError>()
 
     // Helper function to handle tool calls with approval logic
-    const handleToolCall = (part: Response.ToolCallPartEncoded) =>
-      Effect.gen(function*() {
-        const tool = toolkit.tools[part.name]
-        if (!tool) {
-          return
-        }
+    const handleToolCall = Effect.fnUntraced(function*(part: Response.ToolCallPartEncoded) {
+      const tool = toolkit.tools[part.name]
+      if (!tool) return
 
-        const needsApproval = yield* isApprovalNeeded(
-          tool,
-          part,
-          providerOptions.prompt.content
-        )
+      const needsApproval = yield* isApprovalNeeded(
+        tool,
+        part,
+        providerOptions.prompt.content
+      )
 
-        if (needsApproval) {
-          const idGen = yield* IdGenerator
-          const approvalId = yield* idGen.generateId()
-          const approvalPart = Response.makePart("tool-approval-request", {
-            approvalId,
-            toolCallId: part.id
-          }) as Response.StreamPart<Tools>
-          yield* Queue.offer(queue, approvalPart)
-          return
-        }
+      if (needsApproval) {
+        const idGen = yield* IdGenerator
+        const approvalId = yield* idGen.generateId()
+        const approvalPart = Response.makePart("tool-approval-request", {
+          approvalId,
+          toolCallId: part.id
+        }) as Response.StreamPart<Tools>
+        yield* Queue.offer(queue, approvalPart)
+        return
+      }
 
-        yield* toolkit.handle(part.name, part.params as any).pipe(
-          Stream.unwrap,
-          Stream.runForEach((result) => {
-            const toolResultPart = Response.makePart("tool-result", {
-              id: part.id,
-              name: part.name,
-              providerExecuted: false,
-              ...result
-            }) as Response.StreamPart<Tools>
-            return Queue.offer(queue, toolResultPart)
-          })
-        )
-      })
+      yield* toolkit.handle(part.name, part.params as any).pipe(
+        Stream.unwrap,
+        Stream.runForEach((result) => {
+          const toolResultPart = Response.makePart("tool-result", {
+            id: part.id,
+            name: part.name,
+            providerExecuted: false,
+            ...result
+          }) as Response.StreamPart<Tools>
+          return Queue.offer(queue, toolResultPart)
+        })
+      )
+    })
 
     yield* params.streamText(providerOptions).pipe(
+      withNonIncrementalFallback,
       Stream.runForEachArray(
         Effect.fnUntraced(function*(chunk) {
           const parts = yield* decodeParts(chunk)
+          if (tracker) {
+            for (const part of parts) {
+              if (part.type === "response-metadata" && part.id) {
+                tracker.markParts(providerOptions.prompt.content, part.id)
+              }
+            }
+          }
           // Add decoded response parts to the output queue
           yield* Queue.offerAll(queue, parts)
           // Fork tool call handlers - use the raw chunk for encoded params

package/src/unstable/ai/ResponseIdTracker.ts

@@ -0,0 +1,97 @@
+/**
+ * @since 4.0.0
+ */
+import * as Effect from "../../Effect.ts"
+import * as Option from "../../Option.ts"
+import * as ServiceMap from "../../ServiceMap.ts"
+import * as Prompt from "./Prompt.ts"
+
+/**
+ * @since 4.0.0
+ * @category models
+ */
+export interface PrepareResult {
+  readonly previousResponseId: string
+  readonly prompt: Prompt.Prompt
+}
+
+/**
+ * @since 4.0.0
+ * @category models
+ */
+export interface Service {
+  clearUnsafe(): void
+  markParts(parts: ReadonlyArray<object>, responseId: string): void
+  prepareUnsafe(prompt: Prompt.Prompt): Option.Option<PrepareResult>
+}
+
+/**
+ * @since 4.0.0
+ * @category Services
+ */
+export class ResponseIdTracker
+  extends ServiceMap.Service<ResponseIdTracker, Service>()("effect/ai/ResponseIdTracker")
+{}
+
+/**
+ * @since 4.0.0
+ * @category constructors
+ */
+export const make: Effect.Effect<Service> = Effect.sync(() => {
+  const sentParts = new Map<object, string>()
+
+  const none = () => {
+    sentParts.clear()
+    return Option.none<PrepareResult>()
+  }
+
+  return {
+    clearUnsafe() {
+      sentParts.clear()
+    },
+    markParts(parts, responseId) {
+      for (let i = 0; i < parts.length; i++) {
+        sentParts.set(parts[i], responseId)
+      }
+    },
+    prepareUnsafe(prompt) {
+      const messages = prompt.content
+
+      let anyTracked = false
+      for (let i = 0; i < messages.length; i++) {
+        if (sentParts.has(messages[i])) {
+          anyTracked = true
+          break
+        }
+      }
+      if (!anyTracked) return none()
+
+      let lastAssistantIndex = -1
+      for (let i = messages.length - 1; i >= 0; i--) {
+        if (messages[i].role === "assistant") {
+          lastAssistantIndex = i
+          break
+        }
+      }
+      if (lastAssistantIndex === -1) return none()
+
+      let responseId: string | undefined
+      for (let i = 0; i < lastAssistantIndex; i++) {
+        const id = sentParts.get(messages[i])
+        if (id === undefined) return none()
+        responseId = id
+      }
+      if (responseId === undefined) return none()
+
+      const partsAfterLastAssistant = messages.slice(lastAssistantIndex + 1)
+      if (partsAfterLastAssistant.length === 0) {
+        return none()
+      }
+
+      return Option.some({
+        previousResponseId: responseId,
+        prompt: Prompt.fromMessages(partsAfterLastAssistant)
+      })
+    }
+  }
+})

package/src/unstable/ai/Tool.ts

@@ -1149,7 +1149,7 @@ const dynamicProto = <
  */
 export const make = <
   const Name extends string,
-  Parameters extends Schema.Top = typeof Schema.Void,
+  Parameters extends Schema.Top = typeof EmptyParams,
   Success extends Schema.Top = typeof Schema.Void,
   Failure extends Schema.Top = typeof Schema.Never,
   Mode extends FailureMode | undefined = undefined,
@@ -1210,7 +1210,7 @@ export const make = <
   return userDefinedProto({
     name,
     description: options?.description,
-    parametersSchema: options?.parameters ?? Schema.Void,
+    parametersSchema: options?.parameters ?? EmptyParams,
     successSchema,
     failureSchema,
     failureMode: options?.failureMode ?? "error",

package/src/unstable/ai/index.ts

@@ -153,6 +153,24 @@ export * as AnthropicStructuredOutput from "./AnthropicStructuredOutput.ts"
  */
 export * as Chat from "./Chat.ts"
 
+/**
+ * The `EmbeddingModel` module provides provider-agnostic text embedding capabilities.
+ *
+ * @example
+ * ```ts
+ * import { Effect } from "effect"
+ * import { EmbeddingModel } from "effect/unstable/ai"
+ *
+ * const program = Effect.gen(function*() {
+ *   const model = yield* EmbeddingModel.EmbeddingModel
+ *   return yield* model.embed("hello world")
+ * })
+ * ```
+ *
+ * @since 4.0.0
+ */
+export * as EmbeddingModel from "./EmbeddingModel.ts"
+
 /**
  * The `IdGenerator` module provides a pluggable system for generating unique identifiers
  * for tool calls and other items in the Effect AI SDKs.
@@ -388,6 +406,11 @@ export * as Prompt from "./Prompt.ts"
  */
 export * as Response from "./Response.ts"
 
+/**
+ * @since 4.0.0
+ */
+export * as ResponseIdTracker from "./ResponseIdTracker.ts"
+
 /**
  * The `Telemetry` module provides OpenTelemetry integration for operations
  * performed against a large language model provider by defining telemetry