@smthrs/std 0.0.0-stage → 1.0.0-rc.12

package/src/internal/CodexText.ts

@@ -0,0 +1,182 @@
+/**
+ * Faithful port of Codex CLI output truncation.
+ *
+ * Mirrors `codex-rs/utils/string/src/truncate.rs` and
+ * `codex-rs/utils/output-truncation/src/lib.rs`: middle truncation on UTF-8
+ * boundaries with a `…{n} tokens truncated…` marker and a 4-bytes-per-token
+ * estimate, so ChatGPT-family models see the exact shapes they were
+ * posttrained on.
+ *
+ * @since 1.0.0
+ */
+
+const APPROX_BYTES_PER_TOKEN = 4
+
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+
+/**
+ * UTF-8 byte length of a string.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const byteLength = (text: string): number => encoder.encode(text).byteLength
+
+/**
+ * Approximate token count: ceil(utf8Bytes / 4).
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxTokenCount = (text: string): number =>
+  Math.floor((byteLength(text) + APPROX_BYTES_PER_TOKEN - 1) / APPROX_BYTES_PER_TOKEN)
+
+/**
+ * Approximate byte budget for a token budget.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxBytesForTokens = (tokens: number): number => tokens * APPROX_BYTES_PER_TOKEN
+
+/**
+ * Approximate tokens represented by a byte count: ceil(bytes / 4).
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxTokensFromByteCount = (bytes: number): number =>
+  Math.floor((bytes + APPROX_BYTES_PER_TOKEN - 1) / APPROX_BYTES_PER_TOKEN)
+
+const formatTruncationMarker = (useTokens: boolean, removedCount: number): string =>
+  useTokens ? `…${removedCount} tokens truncated…` : `…${removedCount} chars truncated…`
+
+interface Split {
+  readonly removedChars: number
+  readonly before: string
+  readonly after: string
+}
+
+/** A UTF-8 continuation byte is `10xxxxxx`; every other byte starts a character. */
+const isContinuation = (byte: number): boolean => (byte & 0b1100_0000) === 0b1000_0000
+
+/** The last character boundary at or before `offset`. */
+const boundaryAtOrBefore = (bytes: Uint8Array, offset: number): number => {
+  let at = Math.min(offset, bytes.byteLength)
+  while (at > 0 && at < bytes.byteLength && isContinuation(bytes[at]!)) at--
+  return at
+}
+
+/** The first character boundary at or after `offset`. */
+const boundaryAtOrAfter = (bytes: Uint8Array, offset: number): number => {
+  let at = Math.max(offset, 0)
+  while (at < bytes.byteLength && isContinuation(bytes[at]!)) at++
+  return at
+}
+
+/**
+ * Splits `s` into the head that fits `beginningBytes` and the tail that fits
+ * `endBytes`, both cut on UTF-8 character boundaries, and counts the characters
+ * dropped between them.
+ *
+ * The boundaries are read off the encoded bytes rather than by re-encoding one
+ * character at a time: a per-character `TextEncoder.encode` costs about two
+ * microseconds, which is nineteen seconds for the eight megabytes a bounded
+ * capture can hand this function.
+ */
+const splitString = (s: string, beginningBytes: number, endBytes: number): Split => {
+  const bytes = encoder.encode(s)
+  const tailStartTarget = Math.max(bytes.byteLength - endBytes, 0)
+  const prefixEnd = boundaryAtOrBefore(bytes, beginningBytes)
+  const suffixStart = boundaryAtOrAfter(bytes, Math.max(tailStartTarget, prefixEnd))
+  let removedChars = 0
+  for (let at = prefixEnd; at < suffixStart; at++) if (!isContinuation(bytes[at]!)) removedChars++
+  return {
+    removedChars,
+    before: decoder.decode(bytes.subarray(0, prefixEnd)),
+    after: decoder.decode(bytes.subarray(suffixStart))
+  }
+}
+
+const truncateWithByteEstimate = (s: string, maxBytes: number, useTokens: boolean): string => {
+  if (s === "") return ""
+  const totalBytes = byteLength(s)
+  if (maxBytes === 0) {
+    return formatTruncationMarker(useTokens, useTokens ? approxTokensFromByteCount(totalBytes) : [...s].length)
+  }
+  if (totalBytes <= maxBytes) return s
+  const leftBudget = Math.floor(maxBytes / 2)
+  const rightBudget = maxBytes - leftBudget
+  const { after, before, removedChars } = splitString(s, leftBudget, rightBudget)
+  const marker = formatTruncationMarker(
+    useTokens,
+    useTokens ? approxTokensFromByteCount(Math.max(totalBytes - maxBytes, 0)) : removedChars
+  )
+  return `${before}${marker}${after}`
+}
+
+/**
+ * Middle-truncate to a byte budget with a char-count marker.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const truncateMiddleChars = (s: string, maxBytes: number): string => truncateWithByteEstimate(s, maxBytes, false)
+
+/**
+ * Middle-truncate to an approximate token budget, preserving the beginning
+ * and end. Returns the possibly truncated string and the original token
+ * count when truncation occurred.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const truncateMiddleWithTokenBudget = (
+  s: string,
+  maxTokens: number
+): { readonly text: string; readonly originalTokenCount: number | undefined } => {
+  if (s === "") return { text: "", originalTokenCount: undefined }
+  if (maxTokens > 0 && byteLength(s) <= approxBytesForTokens(maxTokens)) {
+    return { text: s, originalTokenCount: undefined }
+  }
+  const truncated = truncateWithByteEstimate(s, approxBytesForTokens(maxTokens), true)
+  if (truncated === s) return { text: s, originalTokenCount: undefined }
+  return { text: truncated, originalTokenCount: approxTokenCount(s) }
+}
+
+const countLines = (content: string): number => {
+  if (content === "") return 0
+  const parts = content.split("\n")
+  return parts[parts.length - 1] === "" ? parts.length - 1 : parts.length
+}
+
+/**
+ * Codex `format_exec_output_for_model`: exit code, wall time, optional line
+ * count disclosure, then the (possibly middle-truncated) output.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const formatExecOutputForModel = (options: {
+  readonly output: string
+  readonly exitCode: number
+  readonly durationSeconds: number
+  readonly maxOutputTokens: number
+  readonly timedOutAfterMs?: number | undefined
+}): string => {
+  const content = options.timedOutAfterMs === undefined
+    ? options.output
+    : `command timed out after ${options.timedOutAfterMs} milliseconds\n${options.output}`
+  const durationSeconds = Math.round(options.durationSeconds * 10) / 10
+  const totalLines = countLines(content)
+  const formatted = truncateMiddleWithTokenBudget(content, options.maxOutputTokens).text
+  const sections = [
+    `Exit code: ${options.exitCode}`,
+    `Wall time: ${durationSeconds} seconds`
+  ]
+  if (totalLines !== countLines(formatted)) sections.push(`Total output lines: ${totalLines}`)
+  sections.push("Output:")
+  sections.push(formatted)
+  return sections.join("\n")
+}

package/src/internal/Declaration.ts

@@ -0,0 +1,60 @@
+/**
+ * Shared declaration helpers for the standard flows.
+ *
+ * @since 1.0.0
+ */
+
+import type * as Capability from "@smthrs/capability/Capability"
+import * as Effects from "@smthrs/core/Effects"
+
+/**
+ * Options accepted by {@link envelope}.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface EnvelopeOptions {
+  readonly tier: "sealed" | "compensable" | "irreversible"
+  readonly mode: "hermetic" | "expected"
+  readonly reads: Iterable<string>
+  readonly writes: Iterable<string>
+}
+
+/**
+ * Builds a deterministic effect declaration for a standard flow.
+ *
+ * Every standard flow serializes on conflict: the library owns no lane
+ * partitioning of its own, so the conservative choice is the safe one.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const envelope = (options: EnvelopeOptions): Effects.Declaration =>
+  Effects.make({
+    reads: options.reads,
+    writes: options.writes,
+    mode: options.mode,
+    onConflict: "serialize",
+    tier: options.tier
+  })
+
+/**
+ * Formats a capability request as the `action:resource` string carried on a
+ * flow declaration.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const capability = (action: Capability.Action, resource: string): string => `${action}:${resource}`
+
+/**
+ * The read glob covering everything under one search root.
+ *
+ * `glob` and `grep` narrow their conservative `/**` declaration to the root the
+ * call actually names, and a declaration that disagreed between the two would
+ * make the same search reserve different paths depending on which flow ran it.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const rootSubtree = (root: string): string => root === "/" ? "/**" : `${root.replace(/\/+$/, "")}/**`

package/src/internal/Diagnostics.ts

@@ -0,0 +1,134 @@
+/**
+ * Keeps a bound language server in step with the files the mutation flows
+ * write, and reads back the errors an edit leaves.
+ *
+ * The server is optional: a host that binds no `LanguageServer` edits files
+ * exactly as before, and a server that fails or times out never fails the
+ * write that already happened.
+ *
+ * @since 1.0.0
+ */
+
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import * as Schema from "effect/Schema"
+import * as LanguageServer from "../LanguageServer.ts"
+
+/**
+ * Most errors attached to one result.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const MAX_ERRORS = 20
+
+/**
+ * One error-severity diagnostic, at its 1-based position.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const Problem = Schema.Struct({
+  line: Schema.Number.annotate({ description: "1-based line" }),
+  character: Schema.Number.annotate({ description: "1-based character" }),
+  message: Schema.String
+})
+
+/**
+ * A decoded {@link Problem}.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export type Problem = typeof Problem.Type
+
+const record = (value: unknown): Readonly<Record<string, unknown>> | undefined =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+    ? value as Readonly<Record<string, unknown>>
+    : undefined
+
+/**
+ * The error-severity items of a document diagnostic report, 1-based.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const errors = (report: unknown): ReadonlyArray<Problem> => {
+  const items = record(report)?.items
+  if (!Array.isArray(items)) return []
+  const found: Array<Problem> = []
+  for (const item of items) {
+    const diagnostic = record(item)
+    const start = record(record(diagnostic?.range)?.start)
+    if (
+      diagnostic?.severity !== 1 || typeof diagnostic.message !== "string" ||
+      typeof start?.line !== "number" || typeof start.character !== "number"
+    ) continue
+    found.push({ line: start.line + 1, character: start.character + 1, message: diagnostic.message })
+    if (found.length === MAX_ERRORS) break
+  }
+  return found
+}
+
+/**
+ * Sends a written file's new text to the bound server, if any. Succeeds with
+ * the server that took it.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const sync = (path: string, text: string): Effect.Effect<LanguageServer.LanguageServer | undefined> =>
+  Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+    Effect.flatMap(Option.match({
+      onNone: () => Effect.succeed(undefined),
+      onSome: (server) =>
+        server.sync(path, text).pipe(
+          Effect.as(server),
+          Effect.orElseSucceed(() => undefined)
+        )
+    }))
+  )
+
+/**
+ * Tells the bound server, if any, that a file is gone.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const close = (path: string): Effect.Effect<void> =>
+  Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+    Effect.flatMap(Option.match({
+      onNone: () => Effect.void,
+      onSome: (server) => server.close(path).pipe(Effect.ignore)
+    }))
+  )
+
+/**
+ * Tells the bound server, if any, to re-read every file it has open, after
+ * something other than the mutation flows may have changed them.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const refresh: Effect.Effect<void> = Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+  Effect.flatMap(Option.match({
+    onNone: () => Effect.void,
+    onSome: (server) => server.refresh.pipe(Effect.ignore)
+  }))
+)
+
+/**
+ * The errors `server` reports for `path`, or `undefined` when it could not
+ * answer.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const errorsOf = (
+  server: LanguageServer.LanguageServer,
+  path: string
+): Effect.Effect<ReadonlyArray<Problem> | undefined> =>
+  server.diagnostics(path).pipe(
+    Effect.map(errors),
+    Effect.orElseSucceed(() => undefined)
+  )