@llm4ts/runner 2.35.3 → 2.36.0

package/src/FlowRunner.ts

@@ -88,7 +88,8 @@ import {
   type ExecutorSpec,
   type Role,
   type RosterDocument,
-  type RosterShape
+  type RosterShape,
+  effortOf
 } from "@llm4ts/flow/Roster"
 import {
   makeHeldCoder,
@@ -349,19 +350,54 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
   // `capabilities.readOnlyEnforcement`.
   const announceReadOnlyGrade = (
     configuration: ConnectorConfig,
-    connector: { readonly capabilities: { readonly readOnlyEnforcement: string } }
-  ): Effect.Effect<void> =>
-    configuration instanceof CliConnectorConfig &&
-    configuration.readOnly &&
-    connector.capabilities.readOnlyEnforcement !== "enforced"
-      ? events.publish(
-          FlowEventsValues.CapabilityUnenforceable(
-            `readOnly requested from '${configuration.connectorId.value}', but its harness mapping ` +
-              `is ${connector.capabilities.readOnlyEnforcement} — the flag is a request, not a ` +
-              "capability removal"
-          )
-        )
-      : Effect.void
+    connector: {
+      readonly capabilities: {
+        readonly readOnlyEnforcement: string
+        readonly effort: string
+        readonly isolatedHeadless: string
+      }
+    }
+  ): Effect.Effect<void> => {
+    const name = configuration.connectorId.value
+    const notes: Array<string> = []
+    if (
+      configuration instanceof CliConnectorConfig &&
+      configuration.readOnly &&
+      connector.capabilities.readOnlyEnforcement !== "enforced"
+    ) {
+      notes.push(
+        `readOnly requested from '${name}', but its harness mapping ` +
+          `is ${connector.capabilities.readOnlyEnforcement} — the flag is a request, not a ` +
+          "capability removal"
+      )
+    }
+    // The same honesty for the ADR 0029 requests: an effort the harness
+    // has no flag for, an isolation the harness only partly provides.
+    if (configuration.effort !== undefined && connector.capabilities.effort !== "mapped") {
+      notes.push(
+        `effort '${configuration.effort}' requested from '${name}', but its harness has no ` +
+          "effort flag — the seat runs at the harness's own default"
+      )
+    }
+    if (
+      configuration instanceof CliConnectorConfig &&
+      configuration.isolated &&
+      connector.capabilities.isolatedHeadless !== "enforced"
+    ) {
+      notes.push(
+        `isolated requested from '${name}', but its harness isolation is ` +
+          `${connector.capabilities.isolatedHeadless} — ` +
+          (connector.capabilities.isolatedHeadless === "partial"
+            ? "the repository's own instruction file is still read"
+            : "nothing reaches the harness")
+      )
+    }
+    return Effect.forEach(
+      notes,
+      (detail) => events.publish(FlowEventsValues.CapabilityUnenforceable(detail)),
+      { discard: true }
+    )
+  }
   // Every seat is metered, so no run is silently free: a backend that reports
   // no token counts (`capabilities.usageReporting` false) is estimated from
   // character counts and labelled `estimated:<model>`, which is what makes
@@ -633,12 +669,15 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
       cacheLock.withPermit(
         Effect.gen(function* () {
           const readOnly = role !== "coder"
-          const key = `${executor.id}\u0000${readOnly ? "ro" : "rw"}\u0000${workDir}`
+          // One seat per executor, mode, directory and effort: two roles
+          // asking the same effort share a process configuration.
+          const effort = effortOf(executor, role) ?? ""
+          const key = `${executor.id}\u0000${readOnly ? "ro" : "rw"}\u0000${workDir}\u0000${effort}`
           const cached = seatCache.get(key)
           if (cached !== undefined) {
             return cached
           }
-          const configuration = executorConfig(executor, readOnly, runEnvironment)
+          const configuration = executorConfig(executor, readOnly, runEnvironment, role)
           if (configuration === undefined) {
             return yield* RosterInvalid.make({
               violations: [`executor '${executor.id}' has an unknown harness '${executor.harness}'`]
@@ -659,7 +698,7 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
       document.executors
         .filter((spec) => rolesOf(spec).includes("judge"))
         .flatMap((spec) => {
-          const configuration = executorConfig(spec, true, runEnvironment)
+          const configuration = executorConfig(spec, true, runEnvironment, "judge")
           return configuration === undefined ? [] : [configuration]
         })[0] ??
       reasoning

package/src/NodeGeminiCliExecutor.ts

@@ -8,7 +8,9 @@ import {
   GeminiCliExecutor,
   buildGeminiArgs,
   extractGeminiCliResponse,
+  geminiHarnessPolicyToml,
   geminiProcessEnv,
+  geminiSupportsAdminPolicy,
   geminiTurnLimitSettingsJson,
   parseGeminiCliStreamEvent,
   validateGeminiExitCode,
@@ -37,6 +39,40 @@ export const makeNodeGeminiCliExecutor = (
           ),
           (path) => geminiProcessEnv(context, path)
         )
+  // The harness policy (`geminiHarnessPolicyToml`) goes in with
+  // `--admin-policy` on a CLI that lists the flag: `gemini --help` is asked
+  // once, and an older CLI simply runs without it — an unknown flag would
+  // fail every turn.
+  let adminPolicy: boolean | undefined
+  const supportsAdminPolicy: Effect.Effect<boolean> = Effect.suspend(() =>
+    adminPolicy !== undefined
+      ? Effect.succeed(adminPolicy)
+      : executor.run(["gemini", "--help"], process.cwd(), {}).pipe(
+          Effect.map((result) =>
+            geminiSupportsAdminPolicy([...result.stdout, ...result.stderr].join("\n"))
+          ),
+          Effect.catch(() => Effect.succeed(false)),
+          Effect.tap((supported) =>
+            Effect.sync(() => {
+              adminPolicy = supported
+            })
+          )
+        )
+  )
+  /** The turn's arguments: the policy file, written for the turn, when the CLI takes it. */
+  const argsFor = (
+    config: Parameters<typeof buildGeminiArgs>[0],
+    context: GeminiCliExecutionContext,
+    outputFormat: string
+  ) =>
+    Effect.flatMap(supportsAdminPolicy, (supported) =>
+      supported
+        ? Effect.map(
+            temporaryFiles.write("gemini-policy-", ".toml", geminiHarnessPolicyToml),
+            (path) => buildGeminiArgs(config, context, outputFormat, path)
+          )
+        : Effect.succeed(buildGeminiArgs(config, context, outputFormat))
+    )
   return {
     checkGeminiInstalled: executor
       .run(["gemini", "--version"], process.cwd(), {})
@@ -51,8 +87,9 @@ export const makeNodeGeminiCliExecutor = (
       Effect.scoped(
         Effect.gen(function* () {
           const environment = yield* environmentFor(context)
+          const args = yield* argsFor(config, context, "json")
           const result = yield* executor.runWithStdin(
-            ["gemini", ...buildGeminiArgs(config, context, "json")],
+            ["gemini", ...args],
             context.cwd ?? process.cwd(),
             environment,
             prompt
@@ -69,15 +106,17 @@ export const makeNodeGeminiCliExecutor = (
     runGeminiProcessStream: (prompt, config, context) =>
       Stream.scoped(
         Stream.unwrap(
-          Effect.map(environmentFor(context), (environment) =>
-            executor
-              .runStreamingWithStdin(
-                ["gemini", ...buildGeminiArgs(config, context, "stream-json")],
-                context.cwd ?? process.cwd(),
-                environment,
-                prompt
-              )
-              .pipe(Stream.map(parseGeminiCliStreamEvent))
+          Effect.map(
+            Effect.all([environmentFor(context), argsFor(config, context, "stream-json")]),
+            ([environment, args]) =>
+              executor
+                .runStreamingWithStdin(
+                  ["gemini", ...args],
+                  context.cwd ?? process.cwd(),
+                  environment,
+                  prompt
+                )
+                .pipe(Stream.map(parseGeminiCliStreamEvent))
           )
         )
       )

package/src/NodeProcessExecutor.ts

@@ -32,6 +32,32 @@ const processFailure = (argv: ReadonlyArray<string>, detail: string): ProviderEr
       : `${commandName(argv)} failed: ${detail}`
   })
 
+/**
+ * The orchestrator's own variables: its `LLM4TS_*` knobs and `_`, the path
+ * of the binary the shell started. The standard `OTEL_*` variables stay:
+ * other OpenTelemetry-aware tools (a CLI's own telemetry) read them too.
+ */
+const orchestratorOwn = (name: string): boolean => name === "_" || name.startsWith("LLM4TS_")
+
+/**
+ * What a child process inherits: this process's environment without the
+ * orchestrator's own variables, then `envVars`. A coding agent, a gate or a
+ * setup command has no use for llm4ts's knobs, and an agent that lists its
+ * environment should not be pointed at the tool running it.
+ */
+export const childEnvironment = (
+  parent: Readonly<Record<string, string | undefined>>,
+  envVars: Readonly<Record<string, string>>
+): Record<string, string> => {
+  const inherited: Record<string, string> = {}
+  for (const [name, value] of Object.entries(parent)) {
+    if (value !== undefined && !orchestratorOwn(name)) {
+      inherited[name] = value
+    }
+  }
+  return { ...inherited, ...envVars }
+}
+
 const spawnChild = (
   argv: ReadonlyArray<string>,
   cwd: string,
@@ -58,10 +84,7 @@ const spawnChild = (
     try: () =>
       spawn(file, spawnArgs, {
         cwd,
-        env: {
-          ...process.env,
-          ...envVars
-        },
+        env: childEnvironment(process.env, envVars),
         stdio: "pipe",
         ...(windows === undefined ? {} : { windowsVerbatimArguments: windows.verbatim })
       }),

package/src/Profile.ts

@@ -9,9 +9,9 @@ export { duration }
 import { formatCount } from "./Terminal.ts"
 import {
   resolveTraceTarget,
+  WatchTargetMissing,
   type TraceChoice,
-  type TraceSources,
-  type WatchTargetMissing
+  type TraceSources
 } from "./Watch.ts"
 
 /**
@@ -1014,18 +1014,158 @@ export const renderProfile = (report: ProfileReport): string => {
   ].join("\n")
 }
 
+// ── delta ───────────────────────────────────────────────────────────────────
+
+/** One line of a before/after comparison (ADR 0029): a count or a duration. */
+export const DeltaRow = Schema.Struct({
+  label: Schema.String,
+  before: Schema.Number,
+  after: Schema.Number,
+  unit: Schema.Literals(["count", "ms"]),
+  /** Which direction is better, for the arrow; `none` for neutral facts. */
+  better: Schema.Literals(["lower", "higher", "none"])
+})
+export type DeltaRow = typeof DeltaRow.Type
+
+export const ProfileDelta = Schema.Struct({
+  rows: Schema.Array(DeltaRow)
+})
+export type ProfileDelta = typeof ProfileDelta.Type
+
+const roleCalls = (report: ProfileReport, role: string): number =>
+  sum(report.models.filter((row) => row.role === role).map((row) => row.calls))
+
+/**
+ * The harness-evals comparison (`docs/harness-evals/README.md`), in its
+ * order: how much the coders explored before editing, how often review and
+ * judgment were called, what the gates said, how honest the claims were,
+ * then where the time went. Tokens and cost are `llm4ts costs`' business.
+ */
+export const profileDelta = (before: ProfileReport, after: ProfileReport): ProfileDelta => {
+  const count = (
+    label: string,
+    pick: (report: ProfileReport) => number,
+    better: DeltaRow["better"] = "lower"
+  ): DeltaRow => ({
+    label,
+    before: pick(before),
+    after: pick(after),
+    unit: "count",
+    better
+  })
+  const time = (label: string, pick: (report: ProfileReport) => number): DeltaRow => ({
+    label,
+    before: pick(before),
+    after: pick(after),
+    unit: "ms",
+    better: "lower"
+  })
+  return {
+    rows: [
+      count("stories done", (r) => r.stories.filter((s) => s.status === "done").length, "higher"),
+      count("stories failed", (r) => r.stories.filter((s) => s.status === "failed").length),
+      count("explore calls before the first edit", (r) =>
+        sum(r.stories.map((s) => s.exploreBeforeEdit))
+      ),
+      count("coder turns", (r) => sum(r.stories.map((s) => s.turns.count))),
+      count("coder edits", (r) => sum(r.stories.map((s) => s.toolCalls.edit)), "none"),
+      count("coder test runs", (r) => sum(r.stories.map((s) => s.toolCalls.test)), "none"),
+      count("reviewer calls", (r) => roleCalls(r, "reviewer")),
+      count("judge calls", (r) => roleCalls(r, "judge")),
+      count("gate runs", (r) => sum(r.gates.map((g) => g.runs))),
+      count("gate failures", (r) => sum(r.gates.map((g) => g.failed))),
+      count("fabricated verification claims", (r) => sum(r.stories.map((s) => s.unverifiedClaims))),
+      count("low-confidence tasks", (r) => sum(r.stories.map((s) => s.lowConfidenceTasks))),
+      count(
+        "stories whose prompt grew over 3×",
+        (r) => r.stories.filter((s) => s.turns.promptGrowth).length
+      ),
+      time("wall", (r) => r.wallMs),
+      time("model time", (r) => r.totals.model),
+      time("coder tool time", (r) => r.totals.tools),
+      time("gate time", (r) => r.totals.gates),
+      time("waiting", (r) => r.totals.waiting)
+    ]
+  }
+}
+
+const deltaCell = (row: DeltaRow): string => {
+  const diff = row.after - row.before
+  if (diff === 0) {
+    return "="
+  }
+  const shown = row.unit === "ms" ? duration(Math.abs(diff)) : String(Math.abs(diff))
+  const sign = diff > 0 ? "+" : "-"
+  const verdict =
+    row.better === "none" ? "" : diff < 0 === (row.better === "lower") ? " better" : " worse"
+  const percent = row.before === 0 ? "" : ` (${Math.round((diff / row.before) * 100)}%)`
+  return `${sign}${shown}${percent}${verdict}`
+}
+
+export const renderProfileDelta = (
+  delta: ProfileDelta,
+  labels: { readonly before: string; readonly after: string }
+): string =>
+  [
+    `llm4ts profile · ${labels.before} → ${labels.after}`,
+    "",
+    ...table(
+      ["measure", "before", "after", "change"],
+      delta.rows.map((row) => [
+        row.label,
+        row.unit === "ms" ? duration(row.before) : String(row.before),
+        row.unit === "ms" ? duration(row.after) : String(row.after),
+        deltaCell(row)
+      ])
+    ),
+    "",
+    "* tokens and cost per role: llm4ts costs --repo <repo>"
+  ].join("\n")
+
 export interface ProfileOptions extends TraceChoice {
   readonly json?: boolean
+  /** A trace, or a `--json` report saved earlier, to compare the profiled run against (ADR 0029). */
+  readonly against?: string
 }
 
 const encodeReport = Schema.encodeSync(Schema.fromJsonString(ProfileReport))
+const decodeReport = Schema.decodeUnknownOption(Schema.fromJsonString(ProfileReport))
+const encodeDelta = Schema.encodeSync(Schema.fromJsonString(ProfileDelta))
 
-/** The report for a trace, an epic's latest run, or the newest trace: text, or JSON for comparing runs. */
+const reportAt = Effect.fn("@llm4ts/runner/Profile.reportAt")(function* (
+  files: TraceSources["files"],
+  path: string
+): Effect.fn.Return<ProfileReport, FlowError | WatchTargetMissing> {
+  if (path.endsWith(".json")) {
+    const text = yield* files.read(path)
+    const decoded = text === undefined ? undefined : decodeReport(text)
+    if (decoded === undefined || decoded._tag === "None") {
+      return yield* WatchTargetMissing.make({
+        message: `${path} is not a profile report (write one with llm4ts profile --json)`
+      })
+    }
+    return decoded.value
+  }
+  return profileOf(treeInputsOfTrace(yield* readTrace(files, path)))
+})
+
+/**
+ * The report for a trace, an epic's latest run, or the newest trace: text,
+ * or JSON for comparing runs; with `against`, the delta from that earlier
+ * run to this one.
+ */
 export const makeProfileProgram = Effect.fn("@llm4ts/runner/Profile.make")(function* (
   options: ProfileOptions,
   sources: TraceSources
 ): Effect.fn.Return<string, FlowError | WatchTargetMissing> {
   const target = yield* resolveTraceTarget(options, sources)
   const report = profileOf(treeInputsOfTrace(yield* readTrace(sources.files, target.tracePath)))
+  if (options.against !== undefined) {
+    const before = yield* reportAt(sources.files, options.against)
+    const delta = profileDelta(before, report)
+    return options.json === true
+      ? encodeDelta(delta)
+      : renderProfileDelta(delta, { before: options.against, after: target.tracePath })
+  }
   return options.json === true ? encodeReport(report) : renderProfile(report)
 })