@llm4ts/flow 2.28.1 → 2.30.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llm4ts/flow",
-  "version": "2.28.1",
+  "version": "2.30.0",
   "description": "Effect-native LLM workflow, persistence, review, and repository automation",
   "license": "MIT",
   "type": "module",
@@ -24,6 +24,7 @@
     "./Blackboard": "./dist/Blackboard.js",
     "./BoardSync": "./dist/BoardSync.js",
     "./CapabilityGuard": "./dist/CapabilityGuard.js",
+    "./CarriedNotes": "./dist/CarriedNotes.js",
     "./Chat": "./dist/Chat.js",
     "./Classified": "./dist/Classified.js",
     "./Context": "./dist/Context.js",
@@ -57,6 +58,7 @@
     "./JudgmentLog": "./dist/JudgmentLog.js",
     "./Perimeter": "./dist/Perimeter.js",
     "./NodePreflight": "./dist/NodePreflight.js",
+    "./Orientation": "./dist/Orientation.js",
     "./Persistence": "./dist/Persistence.js",
     "./Plan": "./dist/Plan.js",
     "./PlanExecution": "./dist/PlanExecution.js",
@@ -72,6 +74,7 @@
     "./ReviewCache": "./dist/ReviewCache.js",
     "./Reviewer": "./dist/Reviewer.js",
     "./SpecChecks": "./dist/SpecChecks.js",
+    "./Spans": "./dist/Spans.js",
     "./Roster": "./dist/Roster.js",
     "./RosterSeats": "./dist/RosterSeats.js",
     "./Stories": "./dist/Stories.js",
@@ -110,7 +113,7 @@
     "typescript"
   ],
   "dependencies": {
-    "@llm4ts/core": "2.28.1"
+    "@llm4ts/core": "2.30.0"
   },
   "peerDependencies": {
     "effect": "4.0.0"

package/src/CarriedNotes.ts

@@ -0,0 +1,103 @@
+// What a story's earlier tasks learned, carried into the next task's prompt.
+// With one chat per task (ADR 0003) every task starts cold; without this a
+// six-task story explores the repository six times. The coder ends each task
+// with a short Findings section; code saves it beside the story and prepends
+// the accumulated notes to the next task. Pure helpers here, the seam in Flow.
+import type * as Effect from "effect/Effect"
+import type { FlowError } from "./FlowError.ts"
+
+export const notesHeading = "## Findings"
+
+/** Appended to every task prompt that carries notes. */
+export const findingsRequest = [
+  "",
+  `When the task is done, end your reply with a section headed \`${notesHeading}\` of at most`,
+  "ten lines: files you read or changed that matter for the rest of this story, conventions",
+  "you learned (where tests live, how a feature is registered, which kit component to use),",
+  "and commands that worked. Nothing follows it — unless you must stop with BLOCKED_ON:,",
+  "which stays the last line."
+].join("\n")
+
+const sectionChars = 1_500
+const headingPattern = /^#{1,3}[ \t]*findings[ \t]*:?[ \t]*$/gimu
+const blockedLine = /^BLOCKED_ON:.*$/u
+
+/**
+ * The text after the LAST Findings heading, without a trailing BLOCKED_ON
+ * line (that stays the reply's last line for `blockedOnIn`), capped;
+ * undefined when there is no heading or nothing under it.
+ */
+export const findingsIn = (reply: string): string | undefined => {
+  let last: RegExpExecArray | undefined
+  headingPattern.lastIndex = 0
+  for (let match = headingPattern.exec(reply); match !== null; match = headingPattern.exec(reply)) {
+    last = match
+  }
+  if (last === undefined) {
+    return undefined
+  }
+  const lines = reply
+    .slice(last.index + last[0].length)
+    .split(/\r?\n/u)
+    .map((line) => line.trimEnd())
+  while (lines.length > 0 && (lines.at(-1) ?? "").trim().length === 0) {
+    lines.pop()
+  }
+  if (
+    lines.length > 0 &&
+    blockedLine.test((lines.at(-1) ?? "").trim().replace(/^[`*_\s]+|[`*_\s]+$/gu, ""))
+  ) {
+    lines.pop()
+  }
+  const text = lines.join("\n").trim()
+  return text.length === 0 ? undefined : text.slice(0, sectionChars)
+}
+
+export const notesLimit = 6_000
+
+/**
+ * `notes` plus a titled section for `found`; when the result is over `limit`
+ * the oldest sections go first, the newest always stays.
+ */
+export const appendNote = (
+  notes: string | undefined,
+  title: string,
+  found: string,
+  limit = notesLimit
+): string => {
+  // A finding's own headings are flattened: `### ` is this file's section
+  // marker, and eviction must never split a section in two.
+  const flat = found
+    .trim()
+    .split(/\r?\n/u)
+    .map((line) => line.replace(/^#{1,6}[ \t]*/u, ""))
+    .join("\n")
+  const sections = [
+    ...(notes === undefined || notes.trim().length === 0 ? [] : notes.split(/\n\n(?=### )/u)),
+    `### ${title}\n${flat}`
+  ]
+  while (sections.length > 1 && sections.join("\n\n").length > limit) {
+    sections.shift()
+  }
+  return sections.join("\n\n")
+}
+
+/** The prompt with the notes in front, or the prompt itself when there are none. */
+export const withNotes = (prompt: string, notes: string | undefined): string =>
+  notes === undefined || notes.trim().length === 0
+    ? prompt
+    : [
+        "What earlier tasks of this story learned — trust it before exploring:",
+        "",
+        notes.trim(),
+        "",
+        "---",
+        "",
+        prompt
+      ].join("\n")
+
+/** Where a story keeps its carried notes: read before a task, written after it. */
+export interface CarriedNotes {
+  readonly read: Effect.Effect<string | undefined, FlowError>
+  readonly write: (notes: string) => Effect.Effect<void, FlowError>
+}

package/src/Flow.ts

@@ -2,6 +2,13 @@ import * as Effect from "effect/Effect"
 import type { LlmServiceShape } from "@llm4ts/core/LlmService"
 import { collect } from "@llm4ts/core/Streaming"
 import { withToolActivity } from "./Activity.ts"
+import {
+  appendNote,
+  findingsIn,
+  findingsRequest,
+  withNotes,
+  type CarriedNotes
+} from "./CarriedNotes.ts"
 import { makeChat, type Chat } from "./Chat.ts"
 import type { FlowContextShape } from "./FlowContext.ts"
 import { FlowAborted, FlowLlmError, type FlowError } from "./FlowError.ts"
@@ -97,6 +104,13 @@ export interface ImplementPlanOptions {
    * or failure. The legacy "judgment" string remains an alias for { mode: "act" }.
    */
   readonly satisfiedProbe?: "literal" | "judgment" | { readonly mode?: JudgmentMode }
+  /**
+   * Notes carried from one task to the next (meant for `chatPerTask`, where
+   * every task starts cold): the accumulated notes open each task's prompt,
+   * the task is asked to end with a Findings section, and what it reports is
+   * appended. Omit to carry nothing.
+   */
+  readonly carry?: CarriedNotes
 }
 
 /** The judgment form of the empty-diff probe: one Truth question over the coder's reply. */
@@ -193,7 +207,22 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
         // the top of this function: a task prompt only needs that task's own
         // details. `planSoFar`, threaded through by implementTaskLoop, is the
         // single source of truth for completion progress instead.
-        yield* coder.ask(plan.taskPrompt(task))
+        const notes = options.carry === undefined ? undefined : yield* options.carry.read
+        /** What the task's reply learned goes to the carried notes, whichever turn did the work. */
+        const keepFindings = (reply: string): Effect.Effect<void, FlowError> =>
+          Effect.gen(function* () {
+            const found = options.carry === undefined ? undefined : findingsIn(reply)
+            if (options.carry !== undefined && found !== undefined) {
+              yield* options.carry.write(appendNote(notes, task.title, found))
+            }
+          })
+        yield* keepFindings(
+          yield* coder.ask(
+            options.carry === undefined
+              ? plan.taskPrompt(task)
+              : `${withNotes(plan.taskPrompt(task), notes)}\n${findingsRequest}`
+          )
+        )
         const produced = yield* context.git.diffAll
         if (produced.trim().length === 0) {
           // An empty diff is ambiguous: the task may be genuinely satisfied
@@ -208,6 +237,7 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
               "Otherwise, implement the task now."
             ].join("\n")
           )
+          yield* keepFindings(confirmation)
           const afterConfirmation = yield* context.git.diffAll
           if (afterConfirmation.trim().length === 0) {
             const judged =

package/src/Orientation.ts

@@ -0,0 +1,192 @@
+// The repository as a planner and a coder should see it before they look:
+// a deterministic digest of the tracked files — folders with their counts,
+// small folders in full, the package scripts, where the tests live. No model
+// call, computed once per epic, so every story stops rediscovering the
+// layout with `ls`, `find` and `grep` round trips.
+import { cap } from "./Context.ts"
+
+export interface OrientationInput {
+  /** Tracked files, repo-relative, as `git ls-files` prints them. */
+  readonly files: ReadonlyArray<string>
+  /** The application's `package.json` text, when there is one. */
+  readonly packageJson?: string
+  /** Where the application lives: `.` for the repository root, else a subfolder. */
+  readonly appDir?: string
+  /** Characters the digest may take; `<= 0` leaves it out. */
+  readonly budget: number
+}
+
+export const defaultOrientationChars = 8_000
+
+/** Folders never worth a coder's attention, wherever they sit: dependencies and state. */
+const noiseAnywhere = new Set(["node_modules", ".git", ".llm4ts"])
+/** Build output is noise at the top level only: `src/features/build/` is code. */
+const noiseAtRoot = new Set(["dist", "build", "out", "coverage"])
+const noiseFiles = new Set(["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lockb"])
+
+/** Folders deeper than this are summarised by their count. */
+const maxDepth = 3
+/** A folder with this many files or fewer is listed in full. */
+const smallFolder = 12
+const testFile = /\.(test|spec)\.[cm]?[jt]sx?$/u
+
+interface Folder {
+  readonly path: string
+  files: number
+  readonly children: Map<string, Folder>
+  readonly own: Array<string>
+}
+
+const folderOf = (path: string): Folder => ({ path, files: 0, children: new Map(), own: [] })
+
+const isNoise = (file: string): boolean => {
+  const parts = file.split("/")
+  return (
+    parts.some((part) => noiseAnywhere.has(part)) ||
+    (parts.length > 1 && noiseAtRoot.has(parts[0] ?? "")) ||
+    noiseFiles.has(parts.at(-1) ?? "")
+  )
+}
+
+const treeOf = (files: ReadonlyArray<string>): Folder => {
+  const root = folderOf("")
+  for (const file of files) {
+    const parts = file.split("/").filter((part) => part.length > 0)
+    let folder = root
+    folder.files += 1
+    for (const part of parts.slice(0, -1)) {
+      let child = folder.children.get(part)
+      if (child === undefined) {
+        child = folderOf(folder.path.length === 0 ? part : `${folder.path}/${part}`)
+        folder.children.set(part, child)
+      }
+      child.files += 1
+      folder = child
+    }
+    folder.own.push(file)
+  }
+  return root
+}
+
+const sortedChildren = (folder: Folder): ReadonlyArray<Folder> =>
+  [...folder.children.values()].sort((left, right) => left.path.localeCompare(right.path))
+
+const folderLine = (folder: Folder, depth: number): string =>
+  `${"  ".repeat(depth)}${folder.path}/ (${folder.files} ${folder.files === 1 ? "file" : "files"})`
+
+/** A small folder in full: its own files, then every subfolder the same way. */
+const expanded = (folder: Folder, depth: number): ReadonlyArray<string> => [
+  ...[...folder.own].sort().map((file) => `${"  ".repeat(depth)}${file}`),
+  ...sortedChildren(folder).flatMap((child) => [
+    folderLine(child, depth),
+    ...expanded(child, depth + 1)
+  ])
+]
+
+/**
+ * The layout: every folder with its count; a folder of `smallFolder` files or
+ * fewer listed in full, a larger one descended into up to `maxDepth`.
+ */
+const layoutLines = (folder: Folder, depth: number): ReadonlyArray<string> =>
+  sortedChildren(folder).flatMap((child) => {
+    if (child.files <= smallFolder) {
+      return [folderLine(child, depth), ...expanded(child, depth + 1)]
+    }
+    const own =
+      child.own.length <= smallFolder
+        ? [...child.own].sort().map((file) => `${"  ".repeat(depth + 1)}${file}`)
+        : []
+    return [
+      folderLine(child, depth),
+      ...own,
+      ...(depth + 1 < maxDepth ? layoutLines(child, depth + 1) : [])
+    ]
+  })
+
+const scriptsLine = (packageJson: string | undefined): string | undefined => {
+  if (packageJson === undefined) {
+    return undefined
+  }
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(packageJson)
+  } catch {
+    return undefined
+  }
+  if (typeof parsed !== "object" || parsed === null || !("scripts" in parsed)) {
+    return undefined
+  }
+  const scripts = parsed.scripts
+  if (typeof scripts !== "object" || scripts === null) {
+    return undefined
+  }
+  const entries = Object.entries(scripts).flatMap(([name, command]) =>
+    typeof command === "string" ? [`${name}: ${cap(command, 80).text}`] : []
+  )
+  return entries.length === 0 ? undefined : `Scripts (package.json): ${entries.join("; ")}`
+}
+
+const testsLine = (files: ReadonlyArray<string>): string | undefined => {
+  const counts = new Map<string, number>()
+  for (const file of files) {
+    if (testFile.test(file) || file.includes("/__tests__/")) {
+      const folder = file.split("/").slice(0, -1).join("/") || "."
+      counts.set(folder, (counts.get(folder) ?? 0) + 1)
+    }
+  }
+  if (counts.size === 0) {
+    return undefined
+  }
+  const shown = [...counts.entries()]
+    .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
+    .slice(0, 8)
+    .map(([folder, count]) => `${folder} (${count})`)
+  const more = counts.size - shown.length
+  return `Tests live beside the code in: ${shown.join(", ")}${more > 0 ? ` and ${more} more folder(s)` : ""}`
+}
+
+/**
+ * The digest, or `undefined` when there is nothing to say (no tracked files)
+ * or no budget. Never longer than `budget`: the head (the layout) and the
+ * tail (scripts, tests) survive a cut, the middle of a long listing does not.
+ */
+export const orientationOf = (input: OrientationInput): string | undefined => {
+  if (input.budget <= 0) {
+    return undefined
+  }
+  const files = input.files.filter((file) => !isNoise(file))
+  if (files.length === 0) {
+    return undefined
+  }
+  const root = treeOf(files)
+  const appDir = input.appDir ?? "."
+  const text = [
+    "## Repository orientation",
+    "Tracked files by folder (counts; small folders listed in full) — go straight to the right",
+    "place instead of listing directories or searching for where things live.",
+    ...(appDir === "."
+      ? []
+      : [`The application lives in ${appDir}/ — its package.json, sources and tests.`]),
+    "",
+    ...[...root.own].sort(),
+    ...layoutLines(root, 0),
+    ...[scriptsLine(input.packageJson), testsLine(files)].flatMap((line) =>
+      line === undefined ? [] : ["", line]
+    )
+  ].join("\n")
+  return cap(text, input.budget).text
+}
+
+/**
+ * LLM4TS_ORIENTATION_CHARS: how much of the repository digest the planner and
+ * every coder see (0 leaves it out); the default otherwise.
+ */
+export const orientationChars = (
+  environment: Readonly<Record<string, string | undefined>>
+): { readonly orientationChars?: number } => {
+  const raw = environment.LLM4TS_ORIENTATION_CHARS?.trim()
+  const value = Number(raw ?? "")
+  return raw === undefined || raw.length === 0 || !Number.isInteger(value) || value < 0
+    ? {}
+    : { orientationChars: value }
+}

package/src/PlanExecution.ts

@@ -1,25 +1,36 @@
 import * as Effect from "effect/Effect"
 import { StageCompleted, StageFailed, StageStarted, type FlowEventsShape } from "./FlowEvents.ts"
+import { withKindSpan, type SpanOptions } from "./Spans.ts"
 import { describeFlowError, type FlowError } from "./FlowError.ts"
 import type { Plan, Task } from "./Plan.ts"
 import type { PlanStoreShape } from "./Persistence.ts"
 
 const errorMessage = describeFlowError
 
+/**
+ * A named step of a flow: published as StageStarted/Completed/Failed for the
+ * terminal, the trace and the profile, and a span of `span.kind` (CHAIN by
+ * default) for OpenTelemetry (ADR 0026).
+ */
 export const stage = <A, E, R>(
   events: FlowEventsShape,
   name: string,
-  effect: Effect.Effect<A, E, R>
+  effect: Effect.Effect<A, E, R>,
+  span: SpanOptions = { kind: "CHAIN" }
 ): Effect.Effect<A, E, R> =>
-  events.publish(StageStarted.make({ stage: name })).pipe(
-    Effect.andThen(effect),
-    Effect.tap(() => events.publish(StageCompleted.make({ stage: name }))),
-    Effect.tapError((error) =>
-      events.publish(
-        StageFailed.make({
-          stage: name,
-          message: errorMessage(error)
-        })
+  withKindSpan(
+    name,
+    span,
+    events.publish(StageStarted.make({ stage: name })).pipe(
+      Effect.andThen(effect),
+      Effect.tap(() => events.publish(StageCompleted.make({ stage: name }))),
+      Effect.tapError((error) =>
+        events.publish(
+          StageFailed.make({
+            stage: name,
+            message: errorMessage(error)
+          })
+        )
       )
     )
   )

package/src/RefineRound.ts

@@ -38,9 +38,21 @@ export const refineProposalJsonSchema: JsonSchema = {
           dependsOn: strings,
           owned: strings,
           sharedReadOnly: strings,
+          readFirst: strings,
+          acceptance: strings,
           provides: strings
         },
-        required: ["id", "title", "description", "dependsOn", "owned", "sharedReadOnly", "provides"]
+        required: [
+          "id",
+          "title",
+          "description",
+          "dependsOn",
+          "owned",
+          "sharedReadOnly",
+          "readFirst",
+          "acceptance",
+          "provides"
+        ]
       }
     },
     notPlanned: {

package/src/Retro.ts

@@ -213,6 +213,16 @@ const countBy = (items: ReadonlyArray<string>): string =>
     .map(([key, count]) => `${key} ×${count}`)
     .join(", ")
 
+/**
+ * A transcript `--land` compacted: every call's input is empty and no reply
+ * is left (ADR 0025). Its shape is still worth reading; its words are gone.
+ */
+const isCompacted = (entries: ReadonlyArray<TranscriptEntry>): boolean =>
+  entries.length > 0 &&
+  entries.every((entry) =>
+    entry._tag === "Call" ? entry.input.length === 0 : entry._tag !== "Reply"
+  )
+
 const head = (text: string, limit = transcriptHead): string => {
   const flat = text.replace(/\s+/g, " ").trim()
   return flat.length <= limit ? flat : `${flat.slice(0, limit)}…`
@@ -373,11 +383,17 @@ const storySection = (
         `- failed tool results: ${failures.length}`
       ]
       const firstFailure = failures[0]
-      if (firstFailure !== undefined && firstFailure._tag === "ToolResult") {
+      if (
+        firstFailure !== undefined &&
+        firstFailure._tag === "ToolResult" &&
+        firstFailure.output.length > 0
+      ) {
         transcriptLines.push(`- first failed tool result: ${head(firstFailure.output)}`)
       }
       if (lastCall !== undefined && lastCall._tag === "Call") {
-        transcriptLines.push(`- last call: ${lastCall.role} — ${head(lastCall.input)}`)
+        transcriptLines.push(
+          `- last call: ${lastCall.role} — ${isCompacted(entries) ? "(compacted)" : head(lastCall.input)}`
+        )
       }
       if (lastReply !== undefined && lastReply._tag === "Reply") {
         transcriptLines.push(`- last reply: ${head(lastReply.text)}`)
@@ -460,7 +476,14 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
     "",
     `- run outcome: ${trace.outcome ?? "unknown (the trace has no end line: the process died)"}`,
     `- stories: ${inputs.plan.stories.length} planned, ${candidates.length} to look at (${candidates.map((c) => c.story.id).join(", ") || "none"})`,
-    `- transcripts: ${inputs.transcripts === undefined ? "none (the run was started without --transcript)" : "present"}`,
+    `- transcripts: ${
+      inputs.transcripts === undefined
+        ? "none (the run was started without --transcript)"
+        : [...inputs.transcripts.values()].some((entries) => entries.length > 0) &&
+            [...inputs.transcripts.values()].every(isCompacted)
+          ? "present, compacted (calls, tools and timings; content removed on land)"
+          : "present"
+    }`,
     ...(retries.length === 0 ? [] : [`- retries across the run: ${countBy(retries)}`]),
     ...(runLines.length === 0
       ? []

package/src/Review.ts

@@ -21,6 +21,7 @@ import {
   Timed,
   type FlowEventsShape
 } from "./FlowEvents.ts"
+import { attr, withKindSpan } from "./Spans.ts"
 import { Reviewer } from "./Reviewer.ts"
 import { publishUsage } from "./Usage.ts"
 import type { JudgmentShape } from "@llm4ts/core/judgment/Judgment"
@@ -275,20 +276,25 @@ export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function
     return ReviewResult.make({ issues: [], summary: "" })
   }
   const started = yield* Clock.currentTimeMillis
-  const result = yield* guarded(
-    Capabilities.Exec(executable),
-    `lint: ${command.join(" ")}`,
-    events,
-    process.run(command, workDir, {}).pipe(
-      Effect.mapError((cause) =>
-        ProcessError.make({
-          message: command.join(" "),
-          detail: cause.message
-        })
+  // A TOOL span (ADR 0026) and a Timed event: the command as configured and
+  // its exit code, never its output.
+  const result = yield* withKindSpan(
+    `gate ${command.join(" ")}`,
+    { kind: "TOOL", attributes: { [attr.gateCommand]: command.join(" ") } },
+    guarded(
+      Capabilities.Exec(executable),
+      `lint: ${command.join(" ")}`,
+      events,
+      process.run(command, workDir, {}).pipe(
+        Effect.mapError((cause) =>
+          ProcessError.make({
+            message: command.join(" "),
+            detail: cause.message
+          })
+        )
       )
-    )
+    ).pipe(Effect.tap((ran) => Effect.annotateCurrentSpan(attr.gateExit, ran.exitCode)))
   )
-  // The command as configured and its exit code: never its output.
   yield* events.publish(
     Timed.make({
       kind: "gate",