@llm4ts/flow 2.28.1 → 2.30.0

package/src/Spans.ts

@@ -0,0 +1,150 @@
+// The vocabulary every llm4ts span speaks (ADR 0026): OpenTelemetry GenAI
+// attributes first, OpenInference's span kind and token keys beside them so
+// Phoenix of any version renders the LLM views, and llm4ts's own keys for
+// what no convention names (role, story, epic, estimated usage). Pure: the
+// exporter is the runner's, the seams only call `withKindSpan`.
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import type * as Tracer from "effect/Tracer"
+import type { TokenUsage } from "@llm4ts/core/Models"
+import { isEstimatedModel } from "./EstimatedUsage.ts"
+
+export type SpanKind = "CHAIN" | "AGENT" | "LLM" | "TOOL" | "EVALUATOR"
+
+export const kindAttribute = "openinference.span.kind"
+
+export const attr = {
+  role: "llm4ts.role",
+  executor: "llm4ts.executor",
+  story: "llm4ts.story",
+  epic: "llm4ts.epic",
+  flow: "llm4ts.flow",
+  run: "llm4ts.run",
+  session: "session.id",
+  toolCategory: "llm4ts.tool.category",
+  estimated: "llm4ts.usage.estimated",
+  costUsd: "llm4ts.cost.usd",
+  costSource: "llm4ts.cost.source",
+  gateCommand: "llm4ts.gate.command",
+  gateExit: "llm4ts.gate.exit_code",
+  model: "gen_ai.request.model",
+  provider: "gen_ai.provider.name",
+  operation: "gen_ai.operation.name",
+  inputTokens: "gen_ai.usage.input_tokens",
+  outputTokens: "gen_ai.usage.output_tokens",
+  oiModel: "llm.model_name",
+  oiPrompt: "llm.token_count.prompt",
+  oiCompletion: "llm.token_count.completion",
+  oiTotal: "llm.token_count.total",
+  input: "input.value",
+  output: "output.value",
+  inputMessages: "gen_ai.input.messages",
+  outputMessages: "gen_ai.output.messages"
+} as const
+
+/** How much content leaves the machine: nothing, prompts and tools, or the system prompt too. */
+export type OtelContent = "off" | "on" | "full"
+
+/** LLM4TS_OTEL_CONTENT: `on` or `full`; anything else is off. */
+export const otelContent = (
+  environment: Readonly<Record<string, string | undefined>>
+): OtelContent => {
+  const value = environment.LLM4TS_OTEL_CONTENT?.trim().toLowerCase()
+  return value === "on" ? "on" : value === "full" ? "full" : "off"
+}
+
+export interface SpanOptions {
+  readonly kind: SpanKind
+  /** Start a new trace here (a story), linked to the span it was started from. */
+  readonly root?: boolean
+  readonly attributes?: Readonly<Record<string, unknown>>
+}
+
+/**
+ * The nearest span `withKindSpan` opened. Effect names a span for every
+ * `Effect.fn`, so the fiber's current span is usually an internal one; kind
+ * spans chain to each other instead, and the runner leaves the internal ones
+ * unsampled, so an exported trace shows the flow, not the call stack.
+ */
+const CurrentKindSpan = Context.Reference<Option.Option<Tracer.Span>>(
+  "@llm4ts/flow/Spans/CurrentKindSpan",
+  { defaultValue: () => Option.none() }
+)
+
+/** The kind span the current code runs under, when there is one. */
+export const currentKindSpan: Effect.Effect<Option.Option<Tracer.Span>> = CurrentKindSpan
+
+/**
+ * A span with the OpenInference kind set, parented to the nearest kind span,
+ * always sampled, ended with the effect's exit. With no kind span above it
+ * (the run; a story asked to root) it starts a trace of its own, linked to
+ * the kind span it was started from when there is one. The wrapped effect
+ * keeps the caller's scope: only the span's lifetime is managed here.
+ */
+export const withKindSpan = <A, E, R>(
+  name: string,
+  options: SpanOptions,
+  effect: Effect.Effect<A, E, R>
+): Effect.Effect<A, E, R> =>
+  Effect.flatMap(CurrentKindSpan, (parent) =>
+    Effect.useSpan(
+      name,
+      {
+        kind: options.kind === "LLM" ? "client" : "internal",
+        sampled: true,
+        ...(options.root === true || Option.isNone(parent)
+          ? {
+              root: true,
+              ...(Option.isSome(parent) ? { links: [{ span: parent.value, attributes: {} }] } : {})
+            }
+          : { parent: parent.value }),
+        attributes: { [kindAttribute]: options.kind, ...(options.attributes ?? {}) }
+      },
+      (span) =>
+        Effect.withParentSpan(
+          Effect.provideService(effect, CurrentKindSpan, Option.some(span)),
+          span
+        )
+    )
+  )
+
+/** The model name a backend would price: the estimate label stripped. */
+export const modelNameOf = (model: string | undefined): string | undefined =>
+  model === undefined
+    ? undefined
+    : isEstimatedModel(model)
+      ? model.replace(/^estimated:/u, "")
+      : model
+
+/**
+ * Token and cost attributes for one model call; `{}` when nothing is known.
+ * A cost travels only when the backend reported it on measured usage: the
+ * backends price estimates themselves from the model name.
+ */
+export const usageAttributes = (
+  model: string | undefined,
+  usage: TokenUsage | undefined,
+  estimated: boolean
+): Record<string, unknown> => {
+  const name = modelNameOf(model)
+  if (name === undefined && usage === undefined) {
+    return {}
+  }
+  return {
+    ...(name === undefined ? {} : { [attr.model]: name, [attr.oiModel]: name }),
+    ...(usage === undefined
+      ? {}
+      : {
+          [attr.inputTokens]: usage.prompt,
+          [attr.outputTokens]: usage.completion,
+          [attr.oiPrompt]: usage.prompt,
+          [attr.oiCompletion]: usage.completion,
+          [attr.oiTotal]: usage.total,
+          ...(usage.costUsd === undefined || estimated
+            ? {}
+            : { [attr.costUsd]: usage.costUsd, [attr.costSource]: "reported" })
+        }),
+    [attr.estimated]: estimated
+  }
+}

package/src/Stories.ts

@@ -74,6 +74,7 @@ import {
   type StoryRelation
 } from "./StoryPlan.ts"
 import { isOutageMessage } from "./TransientRetry.ts"
+import { attr, withKindSpan } from "./Spans.ts"
 
 const join = (root: string, path: string): string =>
   `${root.replace(/[\\/]+$/, "")}/${path.replace(/^[\\/]+/, "")}`
@@ -314,10 +315,11 @@ const defaultContextChars = 40_000
 const fileChars = 8_000
 
 /**
- * The code a story starts from, for its coder's system prompt: the shared
- * read-only files it builds on, then the files it owns, each capped, until
- * `budget` runs out; the rest by path only. Reading them up front saves the
- * `ls`/`find`/`cat` round trips the coder would otherwise spend on them.
+ * The code a story starts from, for its coder's system prompt: the planner's
+ * read-first anchors, then the shared read-only files it builds on, then the
+ * files it owns, each capped, until `budget` runs out; the rest by path only.
+ * Reading them up front saves the `ls`/`find`/`cat` round trips the coder
+ * would otherwise spend on them.
  */
 export const startingCodeOf = (
   story: Story,
@@ -330,42 +332,66 @@ export const startingCodeOf = (
     if (budget <= 0) {
       return undefined
     }
+    const anchors = yield* git.listFiles(story.readFirst)
     const shared = yield* git.listFiles(story.sharedReadOnly)
     const owned = yield* git.listFiles(story.owned)
-    if (shared.length + owned.length === 0) {
+    if (anchors.length + shared.length + owned.length === 0) {
       return undefined
     }
     let left = budget
     const leftOut: Array<string> = []
-    const section = (title: string, paths: ReadonlyArray<string>) =>
+    const shown = new Set<string>()
+    // `limit`: how far this section may draw the budget down. The anchors
+    // stop at half, so a broad anchor (a whole feature folder) cannot starve
+    // the story's own files; the later sections may use everything left.
+    const section = (title: string, paths: ReadonlyArray<string>, limit = 0) =>
       Effect.gen(function* () {
         const parts: Array<string> = []
         for (const path of paths) {
+          if (shown.has(path)) {
+            continue
+          }
           const text = yield* Effect.orElseSucceed(
             files.read(join(worktree, path)),
             () => undefined
           )
-          if (text === undefined || left <= 0) {
+          if (text === undefined || left <= limit) {
             leftOut.push(path)
             continue
           }
-          const shown = cap(text, Math.min(fileChars, left)).text
-          left -= shown.length
-          parts.push(`### ${path}\n\`\`\`\n${shown}\n\`\`\``)
+          shown.add(path)
+          const piece = cap(text, Math.min(fileChars, left - limit)).text
+          left -= piece.length
+          parts.push(`### ${path}\n\`\`\`\n${piece}\n\`\`\``)
         }
         return parts.length === 0 ? [] : [title, ...parts]
       })
     return [
       "## The code you start from",
       "Read this before exploring: it is the current content of the files below.",
+      ...(yield* section(
+        "Read first — what the planner says to imitate or build on:",
+        anchors,
+        Math.floor(budget / 2)
+      )),
       ...(yield* section("Shared, read-only — use these as they are:", shared)),
       ...(yield* section("Yours — the story's owned files so far:", owned)),
       ...(leftOut.length === 0 ? [] : [`Not shown (over the budget): ${leftOut.join(", ")}`])
     ].join("\n\n")
   })
 
+const numbered = (items: ReadonlyArray<string>): ReadonlyArray<string> =>
+  items.map((item, index) => `${index + 1}. ${item}`)
+
 export const storyPrompt = (story: Story): string =>
-  [`Story: ${story.title}`, "", story.description.trim()].join("\n")
+  [
+    `Story: ${story.title}`,
+    "",
+    story.description.trim(),
+    ...(story.acceptance.length === 0
+      ? []
+      : ["", "Done when (each must hold, observably):", ...numbered(story.acceptance)])
+  ].join("\n")
 
 export const storyTaskPlanInstructions = (story: Story): string =>
   [
@@ -373,6 +399,14 @@ export const storyTaskPlanInstructions = (story: Story): string =>
     "into an ordered list of small, independently verifiable tasks, each described by its",
     "observable outcome. Every task must stay inside the story's owned paths:",
     bullets(story.owned),
+    ...(story.acceptance.length === 0
+      ? []
+      : [
+          "The story is done when every one of these holds:",
+          ...numbered(story.acceptance),
+          "End every task's description with `Satisfies: <n>` naming the criteria it serves, and",
+          "make sure every criterion is served by at least one task."
+        ]),
     "Never plan a task that creates or changes anything outside them — registering or wiring",
     "the story in elsewhere (the app's composition point, a shared kit file) is another story's job.",
     `Use exactly this epicId: "${story.id}".`,
@@ -681,6 +715,8 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   /** What reviewers and the judge found, round by round, for people to read. */
   const findingsPath = (story: Story): string =>
     join(options.stateDir, `stories/${story.id}.findings.md`)
+  /** What the coder's tasks learned, carried into the next task of the same story. */
+  const notesPath = (story: Story): string => join(options.stateDir, `stories/${story.id}.notes.md`)
   const findingLines = (result: ReviewResult): ReadonlyArray<string> =>
     result.issues.map((issue) => {
       const where =
@@ -824,7 +860,9 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       }
       // The task checkpoint belongs to the old branch: left in place, the
       // fresh branch would inherit "every task complete" and skip the coder.
+      // The notes its tasks carried describe the old definition: gone too.
       yield* files.remove(planPath(story))
+      yield* files.remove(notesPath(story))
       stored = undefined
     }
     if (stored !== undefined && stored.status === "merged") {
@@ -1127,6 +1165,10 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       ),
       system: taskSystem,
       chatPerTask: true,
+      carry: {
+        read: files.read(notesPath(story)),
+        write: (notes) => files.writeAtomic(notesPath(story), notes)
+      },
       checkoutBranch: false,
       lint: gates,
       // A story's final state is judged and gated downstream (judge round,
@@ -1202,40 +1244,68 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
         // rerun that finds the same diff gets the same answer without a
         // model call, and the last verdict is on disk to read. Only the
         // first round reuses it; a later round follows a revision and asks.
-        const judged =
-          empty && subject.trim().length === 0
-            ? // Nothing to judge is decided here, not asked of a model.
-              {
-                value: ReviewResult.make({
-                  issues: [
-                    ReviewIssue.make({
-                      severity: "Critical",
-                      title: `nothing exists yet under ${story.owned.join(", ")}`,
-                      description: "The story's owned paths are empty on the epic branch."
+        // The round is an EVALUATOR span (ADR 0026): the verdict's scores and
+        // findings land on it, so a trace shows why a story was sent back.
+        const judged = yield* withKindSpan(
+          `story ${story.id}: judge ${round}`,
+          { kind: "EVALUATOR" },
+          Effect.gen(function* () {
+            const outcome =
+              empty && subject.trim().length === 0
+                ? // Nothing to judge is decided here, not asked of a model.
+                  {
+                    value: ReviewResult.make({
+                      issues: [
+                        ReviewIssue.make({
+                          severity: "Critical",
+                          title: `nothing exists yet under ${story.owned.join(", ")}`,
+                          description: "The story's owned paths are empty on the epic branch."
+                        })
+                      ],
+                      summary: `judge:${story.id}`
+                    }),
+                    reused: false
+                  }
+                : yield* cachedValue(
+                    files,
+                    judgePath(story),
+                    Verdict,
+                    fingerprintOf([
+                      empty ? "code" : "diff",
+                      subject,
+                      story.id,
+                      story.title,
+                      story.description,
+                      story.provides.join("\n")
+                    ]),
+                    withTimedRole(
+                      "judge",
+                      judge(story, subject, watchedSeats, empty ? "code" : "diff")
+                    ),
+                    { reuse: round === 1 }
+                  )
+            const scored = outcome.value instanceof StoryVerdict ? outcome.value.dimensions : []
+            yield* Effect.annotateCurrentSpan({
+              "llm4ts.judge.cleared": outcome.value.isClean,
+              "llm4ts.judge.reused": outcome.reused,
+              ...Object.fromEntries(scored.map((d) => [`llm4ts.judge.${d.id}`, d.score]))
+            })
+            const now = yield* Clock.currentTimeNanos
+            yield* Effect.option(Effect.currentSpan).pipe(
+              Effect.map((span) => {
+                if (span._tag === "Some") {
+                  for (const issue of outcome.value.issues) {
+                    span.value.event("judge finding", now, {
+                      title: issue.title,
+                      severity: issue.severity
                     })
-                  ],
-                  summary: `judge:${story.id}`
-                }),
-                reused: false
-              }
-            : yield* cachedValue(
-                files,
-                judgePath(story),
-                Verdict,
-                fingerprintOf([
-                  empty ? "code" : "diff",
-                  subject,
-                  story.id,
-                  story.title,
-                  story.description,
-                  story.provides.join("\n")
-                ]),
-                withTimedRole(
-                  "judge",
-                  judge(story, subject, watchedSeats, empty ? "code" : "diff")
-                ),
-                { reuse: round === 1 }
-              )
+                  }
+                }
+              })
+            )
+            return outcome
+          })
+        )
         const verdict = judged.value
         if (judged.reused) {
           yield* laneOf(story).publish(
@@ -1349,8 +1419,33 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   })
 
   /** A story's run as a completion — failures become outcomes, never fiber deaths. */
+  // One trace per story (ADR 0026): the story span roots it, linked to the
+  // run's span, and every span under it — tasks, model calls, tools, gates —
+  // carries the story, the epic and the run's session.
+  const storySpanAttributes = (story: Story): Readonly<Record<string, unknown>> => ({
+    [attr.story]: story.id,
+    [attr.epic]: plan.epicId,
+    ...(context.trace === undefined
+      ? {}
+      : { [attr.session]: context.trace.runId, [attr.run]: context.trace.runId })
+  })
+  const annotated = <A, E, R>(
+    story: Story,
+    effect: Effect.Effect<A, E, R>
+  ): Effect.Effect<A, E, R> =>
+    Object.entries(storySpanAttributes(story)).reduce(
+      (acc, [key, value]) => Effect.annotateSpans(key, value)(acc),
+      effect
+    )
   const attempt = (story: Story): Effect.Effect<Completion> =>
-    stage(laneOf(story), `story ${story.id}`, runStory(story)).pipe(
+    annotated(
+      story,
+      stage(laneOf(story), `story ${story.id}`, runStory(story), {
+        kind: "AGENT",
+        root: true,
+        attributes: storySpanAttributes(story)
+      })
+    ).pipe(
       Effect.map((outcome): Completion => ({ story, outcome })),
       Effect.catch((error) =>
         Effect.gen(function* () {

package/src/StoryPlan.ts

@@ -30,6 +30,20 @@ export class Story extends Schema.Class<Story>("Story")({
   provides: Schema.Array(Schema.String).pipe(
     Schema.withConstructorDefault(Effect.succeed([])),
     Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /**
+   * Paths the planner says to read before touching anything: the exemplar
+   * feature to imitate, the contract this story extends, the kit component to
+   * reuse. Pruned to paths that exist; their contents open the starting code.
+   */
+  readFirst: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /** Observable outcomes that make the story done; the coder plans against them, the judge scores against them. */
+  acceptance: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
   )
 }) {}
 
@@ -120,6 +134,51 @@ export const storyPlanViolations = (plan: StoryPlan): ReadonlyArray<string> => {
   return violations
 }
 
+/**
+ * An anchor the repository can answer: relative, inside the checkout, and
+ * narrower than the whole tree. `../x`, `/abs/x`, `.`, `` and `a/../b` are
+ * not — `git ls-files` is fatal on the first two, and the last ones would
+ * name every file.
+ */
+export const isSafeAnchor = (anchor: string): boolean => {
+  const path = normalizePath(anchor)
+  return (
+    path.length > 0 &&
+    !path.startsWith("/") &&
+    !/^[A-Za-z]:/u.test(path) &&
+    path !== "." &&
+    !path.split("/").includes("..")
+  )
+}
+
+/**
+ * The plan with every `readFirst` anchor that no tracked file lies under
+ * removed — unsafe anchors (`isSafeAnchor`) first, without a lookup — and
+ * the list of what was dropped. A planner names paths from a layout digest
+ * and sometimes guesses; a guess must not reach a coder as something to read.
+ */
+export const pruneReadFirst = (
+  plan: StoryPlan,
+  known: ReadonlySet<string>
+): {
+  readonly plan: StoryPlan
+  readonly dropped: ReadonlyArray<{ readonly story: string; readonly path: string }>
+} => {
+  const files = [...known].map(normalizePath)
+  const dropped: Array<{ readonly story: string; readonly path: string }> = []
+  const stories = plan.stories.map((story) => {
+    const kept = story.readFirst.filter((anchor) => {
+      const exists = isSafeAnchor(anchor) && files.some((file) => pathWithin(file, anchor))
+      if (!exists) {
+        dropped.push({ story: story.id, path: anchor })
+      }
+      return exists
+    })
+    return kept.length === story.readFirst.length ? story : new Story({ ...story, readFirst: kept })
+  })
+  return { plan: dropped.length === 0 ? plan : new StoryPlan({ ...plan, stories }), dropped }
+}
+
 const cycles = (plan: StoryPlan): ReadonlyArray<ReadonlyArray<string>> => {
   const found: Array<ReadonlyArray<string>> = []
   const state = new Map<string, "visiting" | "done">()
@@ -289,7 +348,11 @@ export const pathsNamedIn = (plan: StoryPlan, text: string): ReadonlyArray<strin
   return [...new Set(found)]
 }
 
-/** Stable over the story entry's content — a changed entry means a fresh branch. */
+/**
+ * Stable over the story entry's content — a changed entry means a fresh
+ * branch. The 2.29 fields enter only when set, so a plan written earlier
+ * keeps every hash and an upgrade restarts no story.
+ */
 export const storyHash = (story: Story): string =>
   stableHash(
     JSON.stringify({
@@ -299,7 +362,11 @@ export const storyHash = (story: Story): string =>
       dependsOn: [...story.dependsOn],
       owned: [...story.owned].map(normalizePath),
       sharedReadOnly: [...story.sharedReadOnly].map(normalizePath),
-      provides: [...story.provides]
+      provides: [...story.provides],
+      ...(story.readFirst.length === 0
+        ? {}
+        : { readFirst: [...story.readFirst].map(normalizePath) }),
+      ...(story.acceptance.length === 0 ? {} : { acceptance: [...story.acceptance] })
     })
   )
 
@@ -367,9 +434,18 @@ export const renderStoryPlan = Effect.fn("@llm4ts/flow/StoryPlan.render")(functi
       "",
       story.description.trim(),
       "",
+      ...(story.acceptance.length === 0
+        ? []
+        : [
+            "Done when:",
+            "",
+            ...story.acceptance.map((criterion, index) => `${index + 1}. ${criterion}`),
+            ""
+          ]),
       `- depends on: ${list(story.dependsOn)}`,
       `- owned: ${list(story.owned)}`,
       `- shared read-only: ${list(story.sharedReadOnly)}`,
+      `- read first: ${list(story.readFirst)}`,
       `- provides: ${list(story.provides)}`,
       ""
     )