@llm4ts/flow 2.28.1 → 2.29.0

package/src/Orientation.ts

@@ -0,0 +1,192 @@
+// The repository as a planner and a coder should see it before they look:
+// a deterministic digest of the tracked files — folders with their counts,
+// small folders in full, the package scripts, where the tests live. No model
+// call, computed once per epic, so every story stops rediscovering the
+// layout with `ls`, `find` and `grep` round trips.
+import { cap } from "./Context.ts"
+
+export interface OrientationInput {
+  /** Tracked files, repo-relative, as `git ls-files` prints them. */
+  readonly files: ReadonlyArray<string>
+  /** The application's `package.json` text, when there is one. */
+  readonly packageJson?: string
+  /** Where the application lives: `.` for the repository root, else a subfolder. */
+  readonly appDir?: string
+  /** Characters the digest may take; `<= 0` leaves it out. */
+  readonly budget: number
+}
+
+export const defaultOrientationChars = 8_000
+
+/** Folders never worth a coder's attention, wherever they sit: dependencies and state. */
+const noiseAnywhere = new Set(["node_modules", ".git", ".llm4ts"])
+/** Build output is noise at the top level only: `src/features/build/` is code. */
+const noiseAtRoot = new Set(["dist", "build", "out", "coverage"])
+const noiseFiles = new Set(["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lockb"])
+
+/** Folders deeper than this are summarised by their count. */
+const maxDepth = 3
+/** A folder with this many files or fewer is listed in full. */
+const smallFolder = 12
+const testFile = /\.(test|spec)\.[cm]?[jt]sx?$/u
+
+interface Folder {
+  readonly path: string
+  files: number
+  readonly children: Map<string, Folder>
+  readonly own: Array<string>
+}
+
+const folderOf = (path: string): Folder => ({ path, files: 0, children: new Map(), own: [] })
+
+const isNoise = (file: string): boolean => {
+  const parts = file.split("/")
+  return (
+    parts.some((part) => noiseAnywhere.has(part)) ||
+    (parts.length > 1 && noiseAtRoot.has(parts[0] ?? "")) ||
+    noiseFiles.has(parts.at(-1) ?? "")
+  )
+}
+
+const treeOf = (files: ReadonlyArray<string>): Folder => {
+  const root = folderOf("")
+  for (const file of files) {
+    const parts = file.split("/").filter((part) => part.length > 0)
+    let folder = root
+    folder.files += 1
+    for (const part of parts.slice(0, -1)) {
+      let child = folder.children.get(part)
+      if (child === undefined) {
+        child = folderOf(folder.path.length === 0 ? part : `${folder.path}/${part}`)
+        folder.children.set(part, child)
+      }
+      child.files += 1
+      folder = child
+    }
+    folder.own.push(file)
+  }
+  return root
+}
+
+const sortedChildren = (folder: Folder): ReadonlyArray<Folder> =>
+  [...folder.children.values()].sort((left, right) => left.path.localeCompare(right.path))
+
+const folderLine = (folder: Folder, depth: number): string =>
+  `${"  ".repeat(depth)}${folder.path}/ (${folder.files} ${folder.files === 1 ? "file" : "files"})`
+
+/** A small folder in full: its own files, then every subfolder the same way. */
+const expanded = (folder: Folder, depth: number): ReadonlyArray<string> => [
+  ...[...folder.own].sort().map((file) => `${"  ".repeat(depth)}${file}`),
+  ...sortedChildren(folder).flatMap((child) => [
+    folderLine(child, depth),
+    ...expanded(child, depth + 1)
+  ])
+]
+
+/**
+ * The layout: every folder with its count; a folder of `smallFolder` files or
+ * fewer listed in full, a larger one descended into up to `maxDepth`.
+ */
+const layoutLines = (folder: Folder, depth: number): ReadonlyArray<string> =>
+  sortedChildren(folder).flatMap((child) => {
+    if (child.files <= smallFolder) {
+      return [folderLine(child, depth), ...expanded(child, depth + 1)]
+    }
+    const own =
+      child.own.length <= smallFolder
+        ? [...child.own].sort().map((file) => `${"  ".repeat(depth + 1)}${file}`)
+        : []
+    return [
+      folderLine(child, depth),
+      ...own,
+      ...(depth + 1 < maxDepth ? layoutLines(child, depth + 1) : [])
+    ]
+  })
+
+const scriptsLine = (packageJson: string | undefined): string | undefined => {
+  if (packageJson === undefined) {
+    return undefined
+  }
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(packageJson)
+  } catch {
+    return undefined
+  }
+  if (typeof parsed !== "object" || parsed === null || !("scripts" in parsed)) {
+    return undefined
+  }
+  const scripts = parsed.scripts
+  if (typeof scripts !== "object" || scripts === null) {
+    return undefined
+  }
+  const entries = Object.entries(scripts).flatMap(([name, command]) =>
+    typeof command === "string" ? [`${name}: ${cap(command, 80).text}`] : []
+  )
+  return entries.length === 0 ? undefined : `Scripts (package.json): ${entries.join("; ")}`
+}
+
+const testsLine = (files: ReadonlyArray<string>): string | undefined => {
+  const counts = new Map<string, number>()
+  for (const file of files) {
+    if (testFile.test(file) || file.includes("/__tests__/")) {
+      const folder = file.split("/").slice(0, -1).join("/") || "."
+      counts.set(folder, (counts.get(folder) ?? 0) + 1)
+    }
+  }
+  if (counts.size === 0) {
+    return undefined
+  }
+  const shown = [...counts.entries()]
+    .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
+    .slice(0, 8)
+    .map(([folder, count]) => `${folder} (${count})`)
+  const more = counts.size - shown.length
+  return `Tests live beside the code in: ${shown.join(", ")}${more > 0 ? ` and ${more} more folder(s)` : ""}`
+}
+
+/**
+ * The digest, or `undefined` when there is nothing to say (no tracked files)
+ * or no budget. Never longer than `budget`: the head (the layout) and the
+ * tail (scripts, tests) survive a cut, the middle of a long listing does not.
+ */
+export const orientationOf = (input: OrientationInput): string | undefined => {
+  if (input.budget <= 0) {
+    return undefined
+  }
+  const files = input.files.filter((file) => !isNoise(file))
+  if (files.length === 0) {
+    return undefined
+  }
+  const root = treeOf(files)
+  const appDir = input.appDir ?? "."
+  const text = [
+    "## Repository orientation",
+    "Tracked files by folder (counts; small folders listed in full) — go straight to the right",
+    "place instead of listing directories or searching for where things live.",
+    ...(appDir === "."
+      ? []
+      : [`The application lives in ${appDir}/ — its package.json, sources and tests.`]),
+    "",
+    ...[...root.own].sort(),
+    ...layoutLines(root, 0),
+    ...[scriptsLine(input.packageJson), testsLine(files)].flatMap((line) =>
+      line === undefined ? [] : ["", line]
+    )
+  ].join("\n")
+  return cap(text, input.budget).text
+}
+
+/**
+ * LLM4TS_ORIENTATION_CHARS: how much of the repository digest the planner and
+ * every coder see (0 leaves it out); the default otherwise.
+ */
+export const orientationChars = (
+  environment: Readonly<Record<string, string | undefined>>
+): { readonly orientationChars?: number } => {
+  const raw = environment.LLM4TS_ORIENTATION_CHARS?.trim()
+  const value = Number(raw ?? "")
+  return raw === undefined || raw.length === 0 || !Number.isInteger(value) || value < 0
+    ? {}
+    : { orientationChars: value }
+}

package/src/RefineRound.ts

@@ -38,9 +38,21 @@ export const refineProposalJsonSchema: JsonSchema = {
           dependsOn: strings,
           owned: strings,
           sharedReadOnly: strings,
+          readFirst: strings,
+          acceptance: strings,
           provides: strings
         },
-        required: ["id", "title", "description", "dependsOn", "owned", "sharedReadOnly", "provides"]
+        required: [
+          "id",
+          "title",
+          "description",
+          "dependsOn",
+          "owned",
+          "sharedReadOnly",
+          "readFirst",
+          "acceptance",
+          "provides"
+        ]
       }
     },
     notPlanned: {

package/src/Retro.ts

@@ -213,6 +213,16 @@ const countBy = (items: ReadonlyArray<string>): string =>
     .map(([key, count]) => `${key} ×${count}`)
     .join(", ")
 
+/**
+ * A transcript `--land` compacted: every call's input is empty and no reply
+ * is left (ADR 0025). Its shape is still worth reading; its words are gone.
+ */
+const isCompacted = (entries: ReadonlyArray<TranscriptEntry>): boolean =>
+  entries.length > 0 &&
+  entries.every((entry) =>
+    entry._tag === "Call" ? entry.input.length === 0 : entry._tag !== "Reply"
+  )
+
 const head = (text: string, limit = transcriptHead): string => {
   const flat = text.replace(/\s+/g, " ").trim()
   return flat.length <= limit ? flat : `${flat.slice(0, limit)}…`
@@ -373,11 +383,17 @@ const storySection = (
         `- failed tool results: ${failures.length}`
       ]
       const firstFailure = failures[0]
-      if (firstFailure !== undefined && firstFailure._tag === "ToolResult") {
+      if (
+        firstFailure !== undefined &&
+        firstFailure._tag === "ToolResult" &&
+        firstFailure.output.length > 0
+      ) {
         transcriptLines.push(`- first failed tool result: ${head(firstFailure.output)}`)
       }
       if (lastCall !== undefined && lastCall._tag === "Call") {
-        transcriptLines.push(`- last call: ${lastCall.role} — ${head(lastCall.input)}`)
+        transcriptLines.push(
+          `- last call: ${lastCall.role} — ${isCompacted(entries) ? "(compacted)" : head(lastCall.input)}`
+        )
       }
       if (lastReply !== undefined && lastReply._tag === "Reply") {
         transcriptLines.push(`- last reply: ${head(lastReply.text)}`)
@@ -460,7 +476,14 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
     "",
     `- run outcome: ${trace.outcome ?? "unknown (the trace has no end line: the process died)"}`,
     `- stories: ${inputs.plan.stories.length} planned, ${candidates.length} to look at (${candidates.map((c) => c.story.id).join(", ") || "none"})`,
-    `- transcripts: ${inputs.transcripts === undefined ? "none (the run was started without --transcript)" : "present"}`,
+    `- transcripts: ${
+      inputs.transcripts === undefined
+        ? "none (the run was started without --transcript)"
+        : [...inputs.transcripts.values()].some((entries) => entries.length > 0) &&
+            [...inputs.transcripts.values()].every(isCompacted)
+          ? "present, compacted (calls, tools and timings; content removed on land)"
+          : "present"
+    }`,
     ...(retries.length === 0 ? [] : [`- retries across the run: ${countBy(retries)}`]),
     ...(runLines.length === 0
       ? []

package/src/Stories.ts

@@ -314,10 +314,11 @@ const defaultContextChars = 40_000
 const fileChars = 8_000
 
 /**
- * The code a story starts from, for its coder's system prompt: the shared
- * read-only files it builds on, then the files it owns, each capped, until
- * `budget` runs out; the rest by path only. Reading them up front saves the
- * `ls`/`find`/`cat` round trips the coder would otherwise spend on them.
+ * The code a story starts from, for its coder's system prompt: the planner's
+ * read-first anchors, then the shared read-only files it builds on, then the
+ * files it owns, each capped, until `budget` runs out; the rest by path only.
+ * Reading them up front saves the `ls`/`find`/`cat` round trips the coder
+ * would otherwise spend on them.
  */
 export const startingCodeOf = (
   story: Story,
@@ -330,42 +331,66 @@ export const startingCodeOf = (
     if (budget <= 0) {
       return undefined
     }
+    const anchors = yield* git.listFiles(story.readFirst)
     const shared = yield* git.listFiles(story.sharedReadOnly)
     const owned = yield* git.listFiles(story.owned)
-    if (shared.length + owned.length === 0) {
+    if (anchors.length + shared.length + owned.length === 0) {
       return undefined
     }
     let left = budget
     const leftOut: Array<string> = []
-    const section = (title: string, paths: ReadonlyArray<string>) =>
+    const shown = new Set<string>()
+    // `limit`: how far this section may draw the budget down. The anchors
+    // stop at half, so a broad anchor (a whole feature folder) cannot starve
+    // the story's own files; the later sections may use everything left.
+    const section = (title: string, paths: ReadonlyArray<string>, limit = 0) =>
       Effect.gen(function* () {
         const parts: Array<string> = []
         for (const path of paths) {
+          if (shown.has(path)) {
+            continue
+          }
           const text = yield* Effect.orElseSucceed(
             files.read(join(worktree, path)),
             () => undefined
           )
-          if (text === undefined || left <= 0) {
+          if (text === undefined || left <= limit) {
             leftOut.push(path)
             continue
           }
-          const shown = cap(text, Math.min(fileChars, left)).text
-          left -= shown.length
-          parts.push(`### ${path}\n\`\`\`\n${shown}\n\`\`\``)
+          shown.add(path)
+          const piece = cap(text, Math.min(fileChars, left - limit)).text
+          left -= piece.length
+          parts.push(`### ${path}\n\`\`\`\n${piece}\n\`\`\``)
         }
         return parts.length === 0 ? [] : [title, ...parts]
       })
     return [
       "## The code you start from",
       "Read this before exploring: it is the current content of the files below.",
+      ...(yield* section(
+        "Read first — what the planner says to imitate or build on:",
+        anchors,
+        Math.floor(budget / 2)
+      )),
       ...(yield* section("Shared, read-only — use these as they are:", shared)),
       ...(yield* section("Yours — the story's owned files so far:", owned)),
       ...(leftOut.length === 0 ? [] : [`Not shown (over the budget): ${leftOut.join(", ")}`])
     ].join("\n\n")
   })
 
+const numbered = (items: ReadonlyArray<string>): ReadonlyArray<string> =>
+  items.map((item, index) => `${index + 1}. ${item}`)
+
 export const storyPrompt = (story: Story): string =>
-  [`Story: ${story.title}`, "", story.description.trim()].join("\n")
+  [
+    `Story: ${story.title}`,
+    "",
+    story.description.trim(),
+    ...(story.acceptance.length === 0
+      ? []
+      : ["", "Done when (each must hold, observably):", ...numbered(story.acceptance)])
+  ].join("\n")
 
 export const storyTaskPlanInstructions = (story: Story): string =>
   [
@@ -373,6 +398,14 @@ export const storyTaskPlanInstructions = (story: Story): string =>
     "into an ordered list of small, independently verifiable tasks, each described by its",
     "observable outcome. Every task must stay inside the story's owned paths:",
     bullets(story.owned),
+    ...(story.acceptance.length === 0
+      ? []
+      : [
+          "The story is done when every one of these holds:",
+          ...numbered(story.acceptance),
+          "End every task's description with `Satisfies: <n>` naming the criteria it serves, and",
+          "make sure every criterion is served by at least one task."
+        ]),
     "Never plan a task that creates or changes anything outside them — registering or wiring",
     "the story in elsewhere (the app's composition point, a shared kit file) is another story's job.",
     `Use exactly this epicId: "${story.id}".`,
@@ -681,6 +714,8 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   /** What reviewers and the judge found, round by round, for people to read. */
   const findingsPath = (story: Story): string =>
     join(options.stateDir, `stories/${story.id}.findings.md`)
+  /** What the coder's tasks learned, carried into the next task of the same story. */
+  const notesPath = (story: Story): string => join(options.stateDir, `stories/${story.id}.notes.md`)
   const findingLines = (result: ReviewResult): ReadonlyArray<string> =>
     result.issues.map((issue) => {
       const where =
@@ -824,7 +859,9 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       }
       // The task checkpoint belongs to the old branch: left in place, the
       // fresh branch would inherit "every task complete" and skip the coder.
+      // The notes its tasks carried describe the old definition: gone too.
       yield* files.remove(planPath(story))
+      yield* files.remove(notesPath(story))
       stored = undefined
     }
     if (stored !== undefined && stored.status === "merged") {
@@ -1127,6 +1164,10 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       ),
       system: taskSystem,
       chatPerTask: true,
+      carry: {
+        read: files.read(notesPath(story)),
+        write: (notes) => files.writeAtomic(notesPath(story), notes)
+      },
       checkoutBranch: false,
       lint: gates,
       // A story's final state is judged and gated downstream (judge round,

package/src/StoryPlan.ts

@@ -30,6 +30,20 @@ export class Story extends Schema.Class<Story>("Story")({
   provides: Schema.Array(Schema.String).pipe(
     Schema.withConstructorDefault(Effect.succeed([])),
     Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /**
+   * Paths the planner says to read before touching anything: the exemplar
+   * feature to imitate, the contract this story extends, the kit component to
+   * reuse. Pruned to paths that exist; their contents open the starting code.
+   */
+  readFirst: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
+  ),
+  /** Observable outcomes that make the story done; the coder plans against them, the judge scores against them. */
+  acceptance: Schema.Array(Schema.String).pipe(
+    Schema.withConstructorDefault(Effect.succeed([])),
+    Schema.withDecodingDefaultKey(Effect.succeed([]))
   )
 }) {}
 
@@ -120,6 +134,51 @@ export const storyPlanViolations = (plan: StoryPlan): ReadonlyArray<string> => {
   return violations
 }
 
+/**
+ * An anchor the repository can answer: relative, inside the checkout, and
+ * narrower than the whole tree. `../x`, `/abs/x`, `.`, `` and `a/../b` are
+ * not — `git ls-files` is fatal on the first two, and the last ones would
+ * name every file.
+ */
+export const isSafeAnchor = (anchor: string): boolean => {
+  const path = normalizePath(anchor)
+  return (
+    path.length > 0 &&
+    !path.startsWith("/") &&
+    !/^[A-Za-z]:/u.test(path) &&
+    path !== "." &&
+    !path.split("/").includes("..")
+  )
+}
+
+/**
+ * The plan with every `readFirst` anchor that no tracked file lies under
+ * removed — unsafe anchors (`isSafeAnchor`) first, without a lookup — and
+ * the list of what was dropped. A planner names paths from a layout digest
+ * and sometimes guesses; a guess must not reach a coder as something to read.
+ */
+export const pruneReadFirst = (
+  plan: StoryPlan,
+  known: ReadonlySet<string>
+): {
+  readonly plan: StoryPlan
+  readonly dropped: ReadonlyArray<{ readonly story: string; readonly path: string }>
+} => {
+  const files = [...known].map(normalizePath)
+  const dropped: Array<{ readonly story: string; readonly path: string }> = []
+  const stories = plan.stories.map((story) => {
+    const kept = story.readFirst.filter((anchor) => {
+      const exists = isSafeAnchor(anchor) && files.some((file) => pathWithin(file, anchor))
+      if (!exists) {
+        dropped.push({ story: story.id, path: anchor })
+      }
+      return exists
+    })
+    return kept.length === story.readFirst.length ? story : new Story({ ...story, readFirst: kept })
+  })
+  return { plan: dropped.length === 0 ? plan : new StoryPlan({ ...plan, stories }), dropped }
+}
+
 const cycles = (plan: StoryPlan): ReadonlyArray<ReadonlyArray<string>> => {
   const found: Array<ReadonlyArray<string>> = []
   const state = new Map<string, "visiting" | "done">()
@@ -289,7 +348,11 @@ export const pathsNamedIn = (plan: StoryPlan, text: string): ReadonlyArray<strin
   return [...new Set(found)]
 }
 
-/** Stable over the story entry's content — a changed entry means a fresh branch. */
+/**
+ * Stable over the story entry's content — a changed entry means a fresh
+ * branch. The 2.29 fields enter only when set, so a plan written earlier
+ * keeps every hash and an upgrade restarts no story.
+ */
 export const storyHash = (story: Story): string =>
   stableHash(
     JSON.stringify({
@@ -299,7 +362,11 @@ export const storyHash = (story: Story): string =>
       dependsOn: [...story.dependsOn],
       owned: [...story.owned].map(normalizePath),
       sharedReadOnly: [...story.sharedReadOnly].map(normalizePath),
-      provides: [...story.provides]
+      provides: [...story.provides],
+      ...(story.readFirst.length === 0
+        ? {}
+        : { readFirst: [...story.readFirst].map(normalizePath) }),
+      ...(story.acceptance.length === 0 ? {} : { acceptance: [...story.acceptance] })
     })
   )
 
@@ -367,9 +434,18 @@ export const renderStoryPlan = Effect.fn("@llm4ts/flow/StoryPlan.render")(functi
       "",
       story.description.trim(),
       "",
+      ...(story.acceptance.length === 0
+        ? []
+        : [
+            "Done when:",
+            "",
+            ...story.acceptance.map((criterion, index) => `${index + 1}. ${criterion}`),
+            ""
+          ]),
       `- depends on: ${list(story.dependsOn)}`,
       `- owned: ${list(story.owned)}`,
       `- shared read-only: ${list(story.sharedReadOnly)}`,
+      `- read first: ${list(story.readFirst)}`,
       `- provides: ${list(story.provides)}`,
       ""
     )

package/src/Transcript.ts

@@ -10,8 +10,9 @@ import { redactText } from "@llm4ts/core/observability/Redaction"
 import { roleOr } from "./Timing.ts"
 
 /**
- * What the model was told and what it answered, per seat call — an opt-in
- * record (`llm4ts run --transcript`) that `llm4ts watch` tails. It lives in
+ * What the model was told and what it answered, per seat call — a record
+ * (`llm4ts run --transcript`; on by default in epic-stories, where
+ * `LLM4TS_TRANSCRIPT=off` turns it off) that `llm4ts watch` tails. It lives in
  * its own files, never in the trace, so the trace and `llm4ts profile` stay
  * content-free. Secrets are redacted and every field is capped.
  */