@llm4ts/flow 2.29.0 → 2.30.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llm4ts/flow",
-  "version": "2.29.0",
+  "version": "2.30.1",
   "description": "Effect-native LLM workflow, persistence, review, and repository automation",
   "license": "MIT",
   "type": "module",
@@ -74,6 +74,7 @@
     "./ReviewCache": "./dist/ReviewCache.js",
     "./Reviewer": "./dist/Reviewer.js",
     "./SpecChecks": "./dist/SpecChecks.js",
+    "./Spans": "./dist/Spans.js",
     "./Roster": "./dist/Roster.js",
     "./RosterSeats": "./dist/RosterSeats.js",
     "./Stories": "./dist/Stories.js",
@@ -112,7 +113,7 @@
     "typescript"
   ],
   "dependencies": {
-    "@llm4ts/core": "2.29.0"
+    "@llm4ts/core": "2.30.1"
   },
   "peerDependencies": {
     "effect": "4.0.0"

package/src/PlanExecution.ts

@@ -1,25 +1,36 @@
 import * as Effect from "effect/Effect"
 import { StageCompleted, StageFailed, StageStarted, type FlowEventsShape } from "./FlowEvents.ts"
+import { withKindSpan, type SpanOptions } from "./Spans.ts"
 import { describeFlowError, type FlowError } from "./FlowError.ts"
 import type { Plan, Task } from "./Plan.ts"
 import type { PlanStoreShape } from "./Persistence.ts"
 
 const errorMessage = describeFlowError
 
+/**
+ * A named step of a flow: published as StageStarted/Completed/Failed for the
+ * terminal, the trace and the profile, and a span of `span.kind` (CHAIN by
+ * default) for OpenTelemetry (ADR 0026).
+ */
 export const stage = <A, E, R>(
   events: FlowEventsShape,
   name: string,
-  effect: Effect.Effect<A, E, R>
+  effect: Effect.Effect<A, E, R>,
+  span: SpanOptions = { kind: "CHAIN" }
 ): Effect.Effect<A, E, R> =>
-  events.publish(StageStarted.make({ stage: name })).pipe(
-    Effect.andThen(effect),
-    Effect.tap(() => events.publish(StageCompleted.make({ stage: name }))),
-    Effect.tapError((error) =>
-      events.publish(
-        StageFailed.make({
-          stage: name,
-          message: errorMessage(error)
-        })
+  withKindSpan(
+    name,
+    span,
+    events.publish(StageStarted.make({ stage: name })).pipe(
+      Effect.andThen(effect),
+      Effect.tap(() => events.publish(StageCompleted.make({ stage: name }))),
+      Effect.tapError((error) =>
+        events.publish(
+          StageFailed.make({
+            stage: name,
+            message: errorMessage(error)
+          })
+        )
       )
     )
   )

package/src/Review.ts

@@ -21,6 +21,7 @@ import {
   Timed,
   type FlowEventsShape
 } from "./FlowEvents.ts"
+import { attr, withKindSpan } from "./Spans.ts"
 import { Reviewer } from "./Reviewer.ts"
 import { publishUsage } from "./Usage.ts"
 import type { JudgmentShape } from "@llm4ts/core/judgment/Judgment"
@@ -275,20 +276,25 @@ export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function
     return ReviewResult.make({ issues: [], summary: "" })
   }
   const started = yield* Clock.currentTimeMillis
-  const result = yield* guarded(
-    Capabilities.Exec(executable),
-    `lint: ${command.join(" ")}`,
-    events,
-    process.run(command, workDir, {}).pipe(
-      Effect.mapError((cause) =>
-        ProcessError.make({
-          message: command.join(" "),
-          detail: cause.message
-        })
+  // A TOOL span (ADR 0026) and a Timed event: the command as configured and
+  // its exit code, never its output.
+  const result = yield* withKindSpan(
+    `gate ${command.join(" ")}`,
+    { kind: "TOOL", attributes: { [attr.gateCommand]: command.join(" ") } },
+    guarded(
+      Capabilities.Exec(executable),
+      `lint: ${command.join(" ")}`,
+      events,
+      process.run(command, workDir, {}).pipe(
+        Effect.mapError((cause) =>
+          ProcessError.make({
+            message: command.join(" "),
+            detail: cause.message
+          })
+        )
       )
-    )
+    ).pipe(Effect.tap((ran) => Effect.annotateCurrentSpan(attr.gateExit, ran.exitCode)))
   )
-  // The command as configured and its exit code: never its output.
   yield* events.publish(
     Timed.make({
       kind: "gate",

package/src/Spans.ts

@@ -0,0 +1,150 @@
+// The vocabulary every llm4ts span speaks (ADR 0026): OpenTelemetry GenAI
+// attributes first, OpenInference's span kind and token keys beside them so
+// Phoenix of any version renders the LLM views, and llm4ts's own keys for
+// what no convention names (role, story, epic, estimated usage). Pure: the
+// exporter is the runner's, the seams only call `withKindSpan`.
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import type * as Tracer from "effect/Tracer"
+import type { TokenUsage } from "@llm4ts/core/Models"
+import { isEstimatedModel } from "./EstimatedUsage.ts"
+
+export type SpanKind = "CHAIN" | "AGENT" | "LLM" | "TOOL" | "EVALUATOR"
+
+export const kindAttribute = "openinference.span.kind"
+
+export const attr = {
+  role: "llm4ts.role",
+  executor: "llm4ts.executor",
+  story: "llm4ts.story",
+  epic: "llm4ts.epic",
+  flow: "llm4ts.flow",
+  run: "llm4ts.run",
+  session: "session.id",
+  toolCategory: "llm4ts.tool.category",
+  estimated: "llm4ts.usage.estimated",
+  costUsd: "llm4ts.cost.usd",
+  costSource: "llm4ts.cost.source",
+  gateCommand: "llm4ts.gate.command",
+  gateExit: "llm4ts.gate.exit_code",
+  model: "gen_ai.request.model",
+  provider: "gen_ai.provider.name",
+  operation: "gen_ai.operation.name",
+  inputTokens: "gen_ai.usage.input_tokens",
+  outputTokens: "gen_ai.usage.output_tokens",
+  oiModel: "llm.model_name",
+  oiPrompt: "llm.token_count.prompt",
+  oiCompletion: "llm.token_count.completion",
+  oiTotal: "llm.token_count.total",
+  input: "input.value",
+  output: "output.value",
+  inputMessages: "gen_ai.input.messages",
+  outputMessages: "gen_ai.output.messages"
+} as const
+
+/** How much content leaves the machine: nothing, prompts and tools, or the system prompt too. */
+export type OtelContent = "off" | "on" | "full"
+
+/** LLM4TS_OTEL_CONTENT: `on` or `full`; anything else is off. */
+export const otelContent = (
+  environment: Readonly<Record<string, string | undefined>>
+): OtelContent => {
+  const value = environment.LLM4TS_OTEL_CONTENT?.trim().toLowerCase()
+  return value === "on" ? "on" : value === "full" ? "full" : "off"
+}
+
+export interface SpanOptions {
+  readonly kind: SpanKind
+  /** Start a new trace here (a story), linked to the span it was started from. */
+  readonly root?: boolean
+  readonly attributes?: Readonly<Record<string, unknown>>
+}
+
+/**
+ * The nearest span `withKindSpan` opened. Effect names a span for every
+ * `Effect.fn`, so the fiber's current span is usually an internal one; kind
+ * spans chain to each other instead, and the runner leaves the internal ones
+ * unsampled, so an exported trace shows the flow, not the call stack.
+ */
+const CurrentKindSpan = Context.Reference<Option.Option<Tracer.Span>>(
+  "@llm4ts/flow/Spans/CurrentKindSpan",
+  { defaultValue: () => Option.none() }
+)
+
+/** The kind span the current code runs under, when there is one. */
+export const currentKindSpan: Effect.Effect<Option.Option<Tracer.Span>> = CurrentKindSpan
+
+/**
+ * A span with the OpenInference kind set, parented to the nearest kind span,
+ * always sampled, ended with the effect's exit. With no kind span above it
+ * (the run; a story asked to root) it starts a trace of its own, linked to
+ * the kind span it was started from when there is one. The wrapped effect
+ * keeps the caller's scope: only the span's lifetime is managed here.
+ */
+export const withKindSpan = <A, E, R>(
+  name: string,
+  options: SpanOptions,
+  effect: Effect.Effect<A, E, R>
+): Effect.Effect<A, E, R> =>
+  Effect.flatMap(CurrentKindSpan, (parent) =>
+    Effect.useSpan(
+      name,
+      {
+        kind: options.kind === "LLM" ? "client" : "internal",
+        sampled: true,
+        ...(options.root === true || Option.isNone(parent)
+          ? {
+              root: true,
+              ...(Option.isSome(parent) ? { links: [{ span: parent.value, attributes: {} }] } : {})
+            }
+          : { parent: parent.value }),
+        attributes: { [kindAttribute]: options.kind, ...(options.attributes ?? {}) }
+      },
+      (span) =>
+        Effect.withParentSpan(
+          Effect.provideService(effect, CurrentKindSpan, Option.some(span)),
+          span
+        )
+    )
+  )
+
+/** The model name a backend would price: the estimate label stripped. */
+export const modelNameOf = (model: string | undefined): string | undefined =>
+  model === undefined
+    ? undefined
+    : isEstimatedModel(model)
+      ? model.replace(/^estimated:/u, "")
+      : model
+
+/**
+ * Token and cost attributes for one model call; `{}` when nothing is known.
+ * A cost travels only when the backend reported it on measured usage: the
+ * backends price estimates themselves from the model name.
+ */
+export const usageAttributes = (
+  model: string | undefined,
+  usage: TokenUsage | undefined,
+  estimated: boolean
+): Record<string, unknown> => {
+  const name = modelNameOf(model)
+  if (name === undefined && usage === undefined) {
+    return {}
+  }
+  return {
+    ...(name === undefined ? {} : { [attr.model]: name, [attr.oiModel]: name }),
+    ...(usage === undefined
+      ? {}
+      : {
+          [attr.inputTokens]: usage.prompt,
+          [attr.outputTokens]: usage.completion,
+          [attr.oiPrompt]: usage.prompt,
+          [attr.oiCompletion]: usage.completion,
+          [attr.oiTotal]: usage.total,
+          ...(usage.costUsd === undefined || estimated
+            ? {}
+            : { [attr.costUsd]: usage.costUsd, [attr.costSource]: "reported" })
+        }),
+    [attr.estimated]: estimated
+  }
+}

package/src/Stories.ts

@@ -74,6 +74,7 @@ import {
   type StoryRelation
 } from "./StoryPlan.ts"
 import { isOutageMessage } from "./TransientRetry.ts"
+import { attr, withKindSpan } from "./Spans.ts"
 
 const join = (root: string, path: string): string =>
   `${root.replace(/[\\/]+$/, "")}/${path.replace(/^[\\/]+/, "")}`
@@ -1243,40 +1244,68 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
         // rerun that finds the same diff gets the same answer without a
         // model call, and the last verdict is on disk to read. Only the
         // first round reuses it; a later round follows a revision and asks.
-        const judged =
-          empty && subject.trim().length === 0
-            ? // Nothing to judge is decided here, not asked of a model.
-              {
-                value: ReviewResult.make({
-                  issues: [
-                    ReviewIssue.make({
-                      severity: "Critical",
-                      title: `nothing exists yet under ${story.owned.join(", ")}`,
-                      description: "The story's owned paths are empty on the epic branch."
+        // The round is an EVALUATOR span (ADR 0026): the verdict's scores and
+        // findings land on it, so a trace shows why a story was sent back.
+        const judged = yield* withKindSpan(
+          `story ${story.id}: judge ${round}`,
+          { kind: "EVALUATOR" },
+          Effect.gen(function* () {
+            const outcome =
+              empty && subject.trim().length === 0
+                ? // Nothing to judge is decided here, not asked of a model.
+                  {
+                    value: ReviewResult.make({
+                      issues: [
+                        ReviewIssue.make({
+                          severity: "Critical",
+                          title: `nothing exists yet under ${story.owned.join(", ")}`,
+                          description: "The story's owned paths are empty on the epic branch."
+                        })
+                      ],
+                      summary: `judge:${story.id}`
+                    }),
+                    reused: false
+                  }
+                : yield* cachedValue(
+                    files,
+                    judgePath(story),
+                    Verdict,
+                    fingerprintOf([
+                      empty ? "code" : "diff",
+                      subject,
+                      story.id,
+                      story.title,
+                      story.description,
+                      story.provides.join("\n")
+                    ]),
+                    withTimedRole(
+                      "judge",
+                      judge(story, subject, watchedSeats, empty ? "code" : "diff")
+                    ),
+                    { reuse: round === 1 }
+                  )
+            const scored = outcome.value instanceof StoryVerdict ? outcome.value.dimensions : []
+            yield* Effect.annotateCurrentSpan({
+              "llm4ts.judge.cleared": outcome.value.isClean,
+              "llm4ts.judge.reused": outcome.reused,
+              ...Object.fromEntries(scored.map((d) => [`llm4ts.judge.${d.id}`, d.score]))
+            })
+            const now = yield* Clock.currentTimeNanos
+            yield* Effect.option(Effect.currentSpan).pipe(
+              Effect.map((span) => {
+                if (span._tag === "Some") {
+                  for (const issue of outcome.value.issues) {
+                    span.value.event("judge finding", now, {
+                      title: issue.title,
+                      severity: issue.severity
                     })
-                  ],
-                  summary: `judge:${story.id}`
-                }),
-                reused: false
-              }
-            : yield* cachedValue(
-                files,
-                judgePath(story),
-                Verdict,
-                fingerprintOf([
-                  empty ? "code" : "diff",
-                  subject,
-                  story.id,
-                  story.title,
-                  story.description,
-                  story.provides.join("\n")
-                ]),
-                withTimedRole(
-                  "judge",
-                  judge(story, subject, watchedSeats, empty ? "code" : "diff")
-                ),
-                { reuse: round === 1 }
-              )
+                  }
+                }
+              })
+            )
+            return outcome
+          })
+        )
         const verdict = judged.value
         if (judged.reused) {
           yield* laneOf(story).publish(
@@ -1390,8 +1419,33 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   })
 
   /** A story's run as a completion — failures become outcomes, never fiber deaths. */
+  // One trace per story (ADR 0026): the story span roots it, linked to the
+  // run's span, and every span under it — tasks, model calls, tools, gates —
+  // carries the story, the epic and the run's session.
+  const storySpanAttributes = (story: Story): Readonly<Record<string, unknown>> => ({
+    [attr.story]: story.id,
+    [attr.epic]: plan.epicId,
+    ...(context.trace === undefined
+      ? {}
+      : { [attr.session]: context.trace.runId, [attr.run]: context.trace.runId })
+  })
+  const annotated = <A, E, R>(
+    story: Story,
+    effect: Effect.Effect<A, E, R>
+  ): Effect.Effect<A, E, R> =>
+    Object.entries(storySpanAttributes(story)).reduce(
+      (acc, [key, value]) => Effect.annotateSpans(key, value)(acc),
+      effect
+    )
   const attempt = (story: Story): Effect.Effect<Completion> =>
-    stage(laneOf(story), `story ${story.id}`, runStory(story)).pipe(
+    annotated(
+      story,
+      stage(laneOf(story), `story ${story.id}`, runStory(story), {
+        kind: "AGENT",
+        root: true,
+        attributes: storySpanAttributes(story)
+      })
+    ).pipe(
       Effect.map((outcome): Completion => ({ story, outcome })),
       Effect.catch((error) =>
         Effect.gen(function* () {