@llm4ts/flow 2.28.0 → 2.29.0

package/src/RefineRound.ts

@@ -38,9 +38,21 @@ export const refineProposalJsonSchema: JsonSchema = {
           dependsOn: strings,
           owned: strings,
           sharedReadOnly: strings,
+          readFirst: strings,
+          acceptance: strings,
           provides: strings
         },
-        required: ["id", "title", "description", "dependsOn", "owned", "sharedReadOnly", "provides"]
+        required: [
+          "id",
+          "title",
+          "description",
+          "dependsOn",
+          "owned",
+          "sharedReadOnly",
+          "readFirst",
+          "acceptance",
+          "provides"
+        ]
       }
     },
     notPlanned: {

package/src/Retro.ts

@@ -213,6 +213,16 @@ const countBy = (items: ReadonlyArray<string>): string =>
     .map(([key, count]) => `${key} ×${count}`)
     .join(", ")
 
+/**
+ * A transcript `--land` compacted: every call's input is empty and no reply
+ * is left (ADR 0025). Its shape is still worth reading; its words are gone.
+ */
+const isCompacted = (entries: ReadonlyArray<TranscriptEntry>): boolean =>
+  entries.length > 0 &&
+  entries.every((entry) =>
+    entry._tag === "Call" ? entry.input.length === 0 : entry._tag !== "Reply"
+  )
+
 const head = (text: string, limit = transcriptHead): string => {
   const flat = text.replace(/\s+/g, " ").trim()
   return flat.length <= limit ? flat : `${flat.slice(0, limit)}…`
@@ -373,11 +383,17 @@ const storySection = (
         `- failed tool results: ${failures.length}`
       ]
       const firstFailure = failures[0]
-      if (firstFailure !== undefined && firstFailure._tag === "ToolResult") {
+      if (
+        firstFailure !== undefined &&
+        firstFailure._tag === "ToolResult" &&
+        firstFailure.output.length > 0
+      ) {
         transcriptLines.push(`- first failed tool result: ${head(firstFailure.output)}`)
       }
       if (lastCall !== undefined && lastCall._tag === "Call") {
-        transcriptLines.push(`- last call: ${lastCall.role} — ${head(lastCall.input)}`)
+        transcriptLines.push(
+          `- last call: ${lastCall.role} — ${isCompacted(entries) ? "(compacted)" : head(lastCall.input)}`
+        )
       }
       if (lastReply !== undefined && lastReply._tag === "Reply") {
         transcriptLines.push(`- last reply: ${head(lastReply.text)}`)
@@ -460,7 +476,14 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
     "",
     `- run outcome: ${trace.outcome ?? "unknown (the trace has no end line: the process died)"}`,
     `- stories: ${inputs.plan.stories.length} planned, ${candidates.length} to look at (${candidates.map((c) => c.story.id).join(", ") || "none"})`,
-    `- transcripts: ${inputs.transcripts === undefined ? "none (the run was started without --transcript)" : "present"}`,
+    `- transcripts: ${
+      inputs.transcripts === undefined
+        ? "none (the run was started without --transcript)"
+        : [...inputs.transcripts.values()].some((entries) => entries.length > 0) &&
+            [...inputs.transcripts.values()].every(isCompacted)
+          ? "present, compacted (calls, tools and timings; content removed on land)"
+          : "present"
+    }`,
     ...(retries.length === 0 ? [] : [`- retries across the run: ${countBy(retries)}`]),
     ...(runLines.length === 0
       ? []

package/src/Roster.ts

@@ -391,6 +391,8 @@ export interface Lease {
   readonly role: Role
   /** Frees the slot; idempotent, and also run when the lease's scope closes. */
   readonly release: Effect.Effect<void>
+  /** Took no slot: the context's own coder executor serves the call (see `LeaseOptions.borrow`). */
+  readonly borrowed: boolean
 }
 
 export interface LeaseOptions {
@@ -400,6 +402,18 @@ export interface LeaseOptions {
   readonly prefer?: string
   /** Who is asking, for the events (a story's worktree). */
   readonly label?: string
+  /**
+   * The executor holding this context's coder, used without a slot (its
+   * coder is idle while the context makes this call) when independence
+   * cannot be had: no executor outside `avoid` can ever take the role, or
+   * every one in the round that could has all its slots held by coders. A
+   * coder's slot frees only when its story ends, and that story may itself
+   * be waiting on a call like this one — waiting for it would saturate the
+   * roster. An executor that is out of the round is still borrowed in the
+   * first case (the call fails on it, as it would on anyone), never in the
+   * second (a story in the round will free a slot).
+   */
+  readonly borrow?: ExecutorSpec
 }
 
 export interface ExecutorStatus {
@@ -423,6 +437,12 @@ export interface RosterShape {
   ) => Effect.Effect<Lease | undefined, never, Scope.Scope>
   /** Whether an executor outside `avoid` has the role and is not out for the run. */
   readonly canEverServe: (role: Role, avoid?: ReadonlyArray<string>) => Effect.Effect<boolean>
+  /**
+   * Whether every executor in the round that takes `role`, outside `avoid`,
+   * has all its slots held by coders — and there is at least one. A lease
+   * for `role` would then wait for a story to end, not for a call.
+   */
+  readonly coderHeld: (role: Role, avoid?: ReadonlyArray<string>) => Effect.Effect<boolean>
   /** Free slots for `role` now, across executors in the round. */
   readonly available: (role: Role) => Effect.Effect<number>
   /** Configured slots for `role`. */
@@ -567,7 +587,29 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
             )
       )
       yield* Effect.addFinalizer(() => free)
-      return { executor: spec, role, release: free }
+      return { executor: spec, role, release: free, borrowed: false }
+    })
+
+  /** A lease over the context's own coder executor: no slot taken, none freed. */
+  const borrowedLease = (
+    spec: ExecutorSpec,
+    role: Role,
+    label: string | undefined,
+    because: "nobody" | "held"
+  ): Effect.Effect<Lease, never, Scope.Scope> =>
+    Effect.gen(function* () {
+      const labelled = label === undefined ? {} : { label }
+      yield* events.publish(
+        ExecutorLeased.make({ executor: spec.id, role, ...labelled, borrowed: true, because })
+      )
+      const done = yield* Ref.make(false)
+      const free = Effect.flatMap(Ref.getAndSet(done, true), (was) =>
+        was
+          ? Effect.void
+          : events.publish(ExecutorReleased.make({ executor: spec.id, role, ...labelled }))
+      )
+      yield* Effect.addFinalizer(() => free)
+      return { executor: spec, role, release: free, borrowed: true }
     })
 
   /** One attempt: expire, pick, take the slot — all under the lock. */
@@ -616,6 +658,19 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
       )
     )
 
+  const coderHeld = (role: Role, avoid: ReadonlyArray<string> = []): Effect.Effect<boolean> =>
+    lock.withPermit(
+      Effect.sync(() => {
+        const inRound = eligible(role, avoid)
+        return (
+          inRound.length > 0 &&
+          inRound.every(
+            (spec) => (usage.get(spec.id) ?? { busy: 0, busyCoders: 0 }).busyCoders >= slotsOf(spec)
+          )
+        )
+      })
+    )
+
   /** Probes every `health` exclusion whose next probe is due; a live one comes back. */
   const probeDue: Effect.Effect<void> = Effect.gen(function* () {
     const probe = options.probe
@@ -710,6 +765,17 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
   ): Effect.Effect<Lease, RosterExhausted, Scope.Scope> =>
     Effect.gen(function* () {
       const avoid = leaseOptions.avoid ?? []
+      const borrow =
+        leaseOptions.borrow !== undefined && hasRole(leaseOptions.borrow, role)
+          ? leaseOptions.borrow
+          : undefined
+      /** Borrowing when the round is held by coders needs an executor in the round. */
+      const borrowInRound = (): Effect.Effect<ExecutorSpec | undefined> =>
+        lock.withPermit(
+          Effect.sync(() =>
+            borrow !== undefined && !exclusions.has(borrow.id) ? borrow : undefined
+          )
+        )
       let announced = false
       const asked = yield* Clock.currentTimeMillis
       // A lease that had to wait says how long, for the story it is for.
@@ -732,6 +798,10 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
           return yield* leaseOf(spec, role, leaseOptions.label)
         }
         if (!(yield* canEverServe(role, avoid))) {
+          if (borrow !== undefined) {
+            yield* waited
+            return yield* borrowedLease(borrow, role, leaseOptions.label, "nobody")
+          }
           const reasons = yield* lock.withPermit(
             Effect.sync(() =>
               executors
@@ -747,6 +817,13 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
             reasons: reasons.length === 0 ? ["no executor in the roster takes this role"] : reasons
           })
         }
+        if (yield* coderHeld(role, avoid)) {
+          const own = yield* borrowInRound()
+          if (own !== undefined) {
+            yield* waited
+            return yield* borrowedLease(own, role, leaseOptions.label, "held")
+          }
+        }
         if (!announced) {
           announced = true
           yield* say(
@@ -876,6 +953,7 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
     lease,
     tryLease,
     canEverServe,
+    coderHeld,
     available,
     slots,
     report,

package/src/RosterSeats.ts

@@ -12,10 +12,9 @@ import { ConfigError, type LlmError } from "@llm4ts/core/Errors"
 import type { LlmServiceShape } from "@llm4ts/core/LlmService"
 import { Message, type ConnectorCapabilities, type LlmChunk } from "@llm4ts/core/Models"
 import { describeFlowError, type FlowError } from "./FlowError.ts"
-import { ExecutorHandedOver, ExecutorLeased, type FlowEventsShape } from "./FlowEvents.ts"
+import { ExecutorHandedOver, type FlowEventsShape } from "./FlowEvents.ts"
 import {
   describeExclusion,
-  hasRole,
   type ExecutorSpec,
   type Lease,
   type Role,
@@ -102,7 +101,11 @@ export interface RosterSeatOptions {
   readonly events: FlowEventsShape
   /** Executors this seat must not use: the context's coder, for independence. */
   readonly avoid?: Effect.Effect<ReadonlyArray<string>>
-  /** Used without a slot when nobody outside `avoid` can ever serve the role. */
+  /**
+   * The context's coder executor, used without a slot when independence
+   * cannot be had: nobody outside `avoid` can ever serve the role, or every
+   * executor that could is held by a coder (`LeaseOptions.borrow`).
+   */
   readonly borrow?: Effect.Effect<ExecutorSpec | undefined>
   /** Who is asking, for the events. */
   readonly label?: string
@@ -127,22 +130,15 @@ export const rosterSeat = (
   ): Effect.Effect<ExecutorSpec, LlmError, Scope.Scope> =>
     Effect.gen(function* () {
       const avoid = [...(options.avoid === undefined ? [] : yield* options.avoid), ...alsoAvoid]
-      if (avoid.length > 0 && !(yield* roster.canEverServe(role, avoid))) {
-        const borrowed = options.borrow === undefined ? undefined : yield* options.borrow
-        if (borrowed !== undefined && hasRole(borrowed, role)) {
-          yield* options.events.publish(
-            ExecutorLeased.make({
-              executor: borrowed.id,
-              role,
-              ...(options.label === undefined ? {} : { label: options.label }),
-              borrowed: true
-            })
-          )
-          return borrowed
-        }
-      }
+      // The roster decides who serves: an independent executor, or, when
+      // none can ever serve or every one is held by a coder, the context's own.
+      const borrow = options.borrow === undefined ? undefined : yield* options.borrow
       const lease = yield* roster
-        .lease(role, { avoid, ...(options.label === undefined ? {} : { label: options.label }) })
+        .lease(role, {
+          avoid,
+          ...(options.label === undefined ? {} : { label: options.label }),
+          ...(borrow === undefined ? {} : { borrow })
+        })
         .pipe(Effect.mapError(asLlmError))
       return lease.executor
     })

package/src/Stories.ts

@@ -314,10 +314,11 @@ const defaultContextChars = 40_000
 const fileChars = 8_000
 
 /**
- * The code a story starts from, for its coder's system prompt: the shared
- * read-only files it builds on, then the files it owns, each capped, until
- * `budget` runs out; the rest by path only. Reading them up front saves the
- * `ls`/`find`/`cat` round trips the coder would otherwise spend on them.
+ * The code a story starts from, for its coder's system prompt: the planner's
+ * read-first anchors, then the shared read-only files it builds on, then the
+ * files it owns, each capped, until `budget` runs out; the rest by path only.
+ * Reading them up front saves the `ls`/`find`/`cat` round trips the coder
+ * would otherwise spend on them.
  */
 export const startingCodeOf = (
   story: Story,
@@ -330,42 +331,66 @@ export const startingCodeOf = (
     if (budget <= 0) {
       return undefined
     }
+    const anchors = yield* git.listFiles(story.readFirst)
     const shared = yield* git.listFiles(story.sharedReadOnly)
     const owned = yield* git.listFiles(story.owned)
-    if (shared.length + owned.length === 0) {
+    if (anchors.length + shared.length + owned.length === 0) {
       return undefined
     }
     let left = budget
     const leftOut: Array<string> = []
-    const section = (title: string, paths: ReadonlyArray<string>) =>
+    const shown = new Set<string>()
+    // `limit`: how far this section may draw the budget down. The anchors
+    // stop at half, so a broad anchor (a whole feature folder) cannot starve
+    // the story's own files; the later sections may use everything left.
+    const section = (title: string, paths: ReadonlyArray<string>, limit = 0) =>
       Effect.gen(function* () {
         const parts: Array<string> = []
         for (const path of paths) {
+          if (shown.has(path)) {
+            continue
+          }
           const text = yield* Effect.orElseSucceed(
             files.read(join(worktree, path)),
             () => undefined
           )
-          if (text === undefined || left <= 0) {
+          if (text === undefined || left <= limit) {
             leftOut.push(path)
             continue
           }
-          const shown = cap(text, Math.min(fileChars, left)).text
-          left -= shown.length
-          parts.push(`### ${path}\n\`\`\`\n${shown}\n\`\`\``)
+          shown.add(path)
+          const piece = cap(text, Math.min(fileChars, left - limit)).text
+          left -= piece.length
+          parts.push(`### ${path}\n\`\`\`\n${piece}\n\`\`\``)
         }
         return parts.length === 0 ? [] : [title, ...parts]
       })
     return [
       "## The code you start from",
       "Read this before exploring: it is the current content of the files below.",
+      ...(yield* section(
+        "Read first — what the planner says to imitate or build on:",
+        anchors,
+        Math.floor(budget / 2)
+      )),
       ...(yield* section("Shared, read-only — use these as they are:", shared)),
       ...(yield* section("Yours — the story's owned files so far:", owned)),
       ...(leftOut.length === 0 ? [] : [`Not shown (over the budget): ${leftOut.join(", ")}`])
     ].join("\n\n")
   })
 
+const numbered = (items: ReadonlyArray<string>): ReadonlyArray<string> =>
+  items.map((item, index) => `${index + 1}. ${item}`)
+
 export const storyPrompt = (story: Story): string =>
-  [`Story: ${story.title}`, "", story.description.trim()].join("\n")
+  [
+    `Story: ${story.title}`,
+    "",
+    story.description.trim(),
+    ...(story.acceptance.length === 0
+      ? []
+      : ["", "Done when (each must hold, observably):", ...numbered(story.acceptance)])
+  ].join("\n")
 
 export const storyTaskPlanInstructions = (story: Story): string =>
   [
@@ -373,6 +398,14 @@ export const storyTaskPlanInstructions = (story: Story): string =>
     "into an ordered list of small, independently verifiable tasks, each described by its",
     "observable outcome. Every task must stay inside the story's owned paths:",
     bullets(story.owned),
+    ...(story.acceptance.length === 0
+      ? []
+      : [
+          "The story is done when every one of these holds:",
+          ...numbered(story.acceptance),
+          "End every task's description with `Satisfies: <n>` naming the criteria it serves, and",
+          "make sure every criterion is served by at least one task."
+        ]),
     "Never plan a task that creates or changes anything outside them — registering or wiring",
     "the story in elsewhere (the app's composition point, a shared kit file) is another story's job.",
     `Use exactly this epicId: "${story.id}".`,
@@ -681,6 +714,8 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   /** What reviewers and the judge found, round by round, for people to read. */
   const findingsPath = (story: Story): string =>
     join(options.stateDir, `stories/${story.id}.findings.md`)
+  /** What the coder's tasks learned, carried into the next task of the same story. */
+  const notesPath = (story: Story): string => join(options.stateDir, `stories/${story.id}.notes.md`)
   const findingLines = (result: ReviewResult): ReadonlyArray<string> =>
     result.issues.map((issue) => {
       const where =
@@ -824,7 +859,9 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       }
       // The task checkpoint belongs to the old branch: left in place, the
       // fresh branch would inherit "every task complete" and skip the coder.
+      // The notes its tasks carried describe the old definition: gone too.
       yield* files.remove(planPath(story))
+      yield* files.remove(notesPath(story))
       stored = undefined
     }
     if (stored !== undefined && stored.status === "merged") {
@@ -1127,6 +1164,10 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       ),
       system: taskSystem,
       chatPerTask: true,
+      carry: {
+        read: files.read(notesPath(story)),
+        write: (notes) => files.writeAtomic(notesPath(story), notes)
+      },
       checkoutBranch: false,
       lint: gates,
       // A story's final state is judged and gated downstream (judge round,

package/src/StoryPlan.ts

@@ -30,6 +30,20 @@ export class Story extends Schema.Class<Story>("Story")({
   provides: Schema.Array(Schema.String).pipe(
     Schema.withConstructorDefault(Effect.succeed([])),
     Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /**
+   * Paths the planner says to read before touching anything: the exemplar
+   * feature to imitate, the contract this story extends, the kit component to
+   * reuse. Pruned to paths that exist; their contents open the starting code.
+   */
+  readFirst: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /** Observable outcomes that make the story done; the coder plans against them, the judge scores against them. */
+  acceptance: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
   )
 }) {}
 
@@ -120,6 +134,51 @@ export const storyPlanViolations = (plan: StoryPlan): ReadonlyArray<string> => {
   return violations
 }
 
+/**
+ * An anchor the repository can answer: relative, inside the checkout, and
+ * narrower than the whole tree. `../x`, `/abs/x`, `.`, `` and `a/../b` are
+ * not — `git ls-files` is fatal on the first two, and the last ones would
+ * name every file.
+ */
+export const isSafeAnchor = (anchor: string): boolean => {
+  const path = normalizePath(anchor)
+  return (
+    path.length > 0 &&
+    !path.startsWith("/") &&
+    !/^[A-Za-z]:/u.test(path) &&
+    path !== "." &&
+    !path.split("/").includes("..")
+  )
+}
+
+/**
+ * The plan with every `readFirst` anchor that no tracked file lies under
+ * removed — unsafe anchors (`isSafeAnchor`) first, without a lookup — and
+ * the list of what was dropped. A planner names paths from a layout digest
+ * and sometimes guesses; a guess must not reach a coder as something to read.
+ */
+export const pruneReadFirst = (
+  plan: StoryPlan,
+  known: ReadonlySet<string>
+): {
+  readonly plan: StoryPlan
+  readonly dropped: ReadonlyArray<{ readonly story: string; readonly path: string }>
+} => {
+  const files = [...known].map(normalizePath)
+  const dropped: Array<{ readonly story: string; readonly path: string }> = []
+  const stories = plan.stories.map((story) => {
+    const kept = story.readFirst.filter((anchor) => {
+      const exists = isSafeAnchor(anchor) && files.some((file) => pathWithin(file, anchor))
+      if (!exists) {
+        dropped.push({ story: story.id, path: anchor })
+      }
+      return exists
+    })
+    return kept.length === story.readFirst.length ? story : new Story({ ...story, readFirst: kept })
+  })
+  return { plan: dropped.length === 0 ? plan : new StoryPlan({ ...plan, stories }), dropped }
+}
+
 const cycles = (plan: StoryPlan): ReadonlyArray<ReadonlyArray<string>> => {
   const found: Array<ReadonlyArray<string>> = []
   const state = new Map<string, "visiting" | "done">()
@@ -289,7 +348,11 @@ export const pathsNamedIn = (plan: StoryPlan, text: string): ReadonlyArray<strin
   return [...new Set(found)]
 }
 
-/** Stable over the story entry's content — a changed entry means a fresh branch. */
+/**
+ * Stable over the story entry's content — a changed entry means a fresh
+ * branch. The 2.29 fields enter only when set, so a plan written earlier
+ * keeps every hash and an upgrade restarts no story.
+ */
 export const storyHash = (story: Story): string =>
   stableHash(
     JSON.stringify({
@@ -299,7 +362,11 @@ export const storyHash = (story: Story): string =>
       dependsOn: [...story.dependsOn],
       owned: [...story.owned].map(normalizePath),
       sharedReadOnly: [...story.sharedReadOnly].map(normalizePath),
-      provides: [...story.provides]
+      provides: [...story.provides],
+      ...(story.readFirst.length === 0
+        ? {}
+        : { readFirst: [...story.readFirst].map(normalizePath) }),
+      ...(story.acceptance.length === 0 ? {} : { acceptance: [...story.acceptance] })
     })
   )
 
@@ -367,9 +434,18 @@ export const renderStoryPlan = Effect.fn("@llm4ts/flow/StoryPlan.render")(functi
       "",
       story.description.trim(),
       "",
+      ...(story.acceptance.length === 0
+        ? []
+        : [
+            "Done when:",
+            "",
+            ...story.acceptance.map((criterion, index) => `${index + 1}. ${criterion}`),
+            ""
+          ]),
       `- depends on: ${list(story.dependsOn)}`,
       `- owned: ${list(story.owned)}`,
       `- shared read-only: ${list(story.sharedReadOnly)}`,
+      `- read first: ${list(story.readFirst)}`,
       `- provides: ${list(story.provides)}`,
       ""
     )

package/src/Transcript.ts

@@ -10,8 +10,9 @@ import { redactText } from "@llm4ts/core/observability/Redaction"
 import { roleOr } from "./Timing.ts"
 
 /**
- * What the model was told and what it answered, per seat call — an opt-in
- * record (`llm4ts run --transcript`) that `llm4ts watch` tails. It lives in
+ * What the model was told and what it answered, per seat call — a record
+ * (`llm4ts run --transcript`; on by default in epic-stories, where
+ * `LLM4TS_TRANSCRIPT=off` turns it off) that `llm4ts watch` tails. It lives in
  * its own files, never in the trace, so the trace and `llm4ts profile` stay
  * content-free. Secrets are redacted and every field is capped.
  */