@agjs/tsforge 0.1.0

package/src/inference/inference.constants.ts

@@ -0,0 +1,34 @@
+/**
+ * Provider tuning. Defaults for the OpenAI-compatible client — kept with the
+ * inference domain (not a global bucket) so the provider's knobs live next to it.
+ */
+export const PROVIDER_LIMITS = {
+  /**
+   * Default model output token budget. Single source of truth (previously split
+   * between the CLI and the provider, which silently truncated large generations).
+   */
+  maxTokens: 16384,
+  /** Per-request timeout (ms): generous for slow local generations, bounded so a
+   *  hung server can't wedge an unattended run forever. */
+  requestTimeoutMs: 600_000,
+  /** Linear backoff base per retry attempt (ms): attempt N waits N * this. */
+  retryBackoffMs: 400,
+  /**
+   * Recommended vLLM repetition penalty IF you opt in via
+   * TSFORGE_REPETITION_PENALTY. OFF by default: applied globally it also
+   * penalizes the repetitive tool-call JSON tokens and pushes the model to
+   * narrate instead of emit tool calls (→ no files written). Degenerate
+   * repetition LOOPS are handled by the StreamGuard (inference/stream-guard.ts)
+   * instead, which only watches prose and can't affect tool-calling.
+   */
+  repetitionPenalty: 1.1,
+} as const;
+
+/**
+ * Default endpoint + model for the local provider — the single source of truth.
+ * Override per-run with TSFORGE_BASE_URL / TSFORGE_MODEL.
+ */
+export const PROVIDER_DEFAULTS = {
+  baseUrl: "http://localhost:8000/v1",
+  model: "qwen3.6-27b",
+} as const;

package/src/inference/inference.types.ts

@@ -0,0 +1,123 @@
+export type Role = "system" | "user" | "assistant" | "tool";
+
+export interface IChatMessage {
+  role: Role;
+  content: string;
+  /** Assistant only: the tool calls it emitted (kept in history so the model
+   *  sees what it asked for and the results that came back). */
+  toolCalls?: IToolCall[];
+  /** Tool messages only: the id of the call this message is the result of. */
+  toolCallId?: string;
+}
+
+/** A parsed tool call from the model (name + decoded JSON arguments). */
+export interface IToolCall {
+  /** Correlation id so a tool-result message can reference it. */
+  id?: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+/** Real token accounting from the server's `usage` block — the basis for the
+ *  status line's context gauge and (soon) auto-compaction triggering. */
+export interface ITokenUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+}
+
+export interface IModelResponse {
+  content: string;
+  toolCalls: IToolCall[];
+  /** Server-reported token usage for this call, when available. `promptTokens`
+   *  is the full context the model just saw — what auto-compaction will watch. */
+  usage?: ITokenUsage;
+  /** How many tool calls were SALVAGED from malformed content (server parser
+   *  left them as text). >0 signals the model emitted unparseable tool syntax. */
+  salvaged?: number;
+  /** Set when the stream was aborted because the model fell into a degenerate
+   *  repetition loop (same line/template until max_tokens). The loop driver
+   *  stops the turn instead of nudging into another loop. */
+  degenerated?: boolean;
+  /** Set when TTSR aborted the stream due to a rule match. Contains the rule name
+   *  and guidance to append to the corrective retry message. */
+  ttsrFired?: { ruleName: string; guidance: string };
+}
+
+export interface ICompleteOptions {
+  temperature?: number;
+  /** OpenAI-style tool schemas to advertise (opaque JSON). */
+  tools?: unknown[];
+  /**
+   * How hard to push the model to call a tool. `required` forces a tool call —
+   * which suppresses chat-style "here is my answer" prose the harness discards
+   * anyway. Defaults to `auto`. Ignored when no tools are advertised.
+   */
+  toolChoice?: "auto" | "required" | "none";
+  /** Per-request thinking toggle (Qwen `chat_template_kwargs.enable_thinking`).
+   *  Omitted = server default. Off for mechanical work, on for hard reasoning. */
+  enableThinking?: boolean;
+  /** Cap reasoning tokens before the model must answer (vLLM
+   *  `thinking_token_budget`). Omitted = unbounded. The lever for turn *time*. */
+  thinkingTokenBudget?: number;
+  /** When set, the request streams and each token is delivered here as it
+   *  arrives, tagged by channel: `reasoning` (the model's thinking) vs `content`
+   *  (its actual answer). Lets a UI dim the thinking and format the answer. */
+  onToken?: (text: string, channel: TokenChannel) => void;
+  /** Caller cancellation — aborting it stops the request (and any stream)
+   *  mid-flight. Combined with the per-request timeout. */
+  signal?: AbortSignal;
+  /** TTSR watcher for stream-interrupting rules (wired by the loop, not the provider). */
+  ttsrManager?: ITtsrWatcher;
+}
+
+/** Structural view of the loop's TtsrManager — keeps the inference layer free of
+ *  a hard dependency on loop internals while staying fully typed. */
+export interface ITtsrWatcher {
+  checkDelta(
+    text: string,
+    context: { source: "content" | "tool-args"; currentFile?: string }
+  ): { readonly name: string; readonly guidance: string } | null;
+}
+
+/** Which stream a token belongs to: the model's thinking (`reasoning`), its answer
+ *  (`content`), or the tool calls it is emitting (`tool` — the file it's writing,
+ *  streamed so a long tool-call generation isn't silent dead air). */
+export type TokenChannel = "reasoning" | "content" | "tool";
+
+/** The model seam. Implementations talk to a local server (vLLM/Ollama/...). */
+export interface IProvider {
+  complete(
+    messages: IChatMessage[],
+    opts?: ICompleteOptions
+  ): Promise<IModelResponse>;
+}
+
+export interface IOpenAICompatibleConfig {
+  /** Root of the OpenAI-compatible API, e.g. http://localhost:11434/v1 */
+  baseUrl: string;
+  /** Model id, e.g. qwen3.6-35b-a3b */
+  model: string;
+  apiKey?: string;
+  /**
+   * Abort a single request after this many ms (default LIMITS.requestTimeoutMs).
+   * Generous because local generations are slow, but bounded so a hung server
+   * can't wedge an unattended run forever.
+   */
+  timeoutMs?: number;
+  /**
+   * Hard cap on tokens per response (default LIMITS.maxTokens). Bounds a
+   * degenerate repetition loop so one runaway generation can't spew until the
+   * context limit. Generous enough for whole-file tool-call output.
+   */
+  maxTokens?: number;
+  /**
+   * vLLM repetition penalty (>1 discourages repeating tokens). The cure for the
+   * degenerate loops this local model falls into at temp 0 — where it repeats
+   * the same line/JSON until max_tokens. ~1.1 breaks loops without hurting
+   * correctness. Omitted (1.0 = off) by default; set it on code-gen providers.
+   */
+  repetitionPenalty?: number;
+  /** Injectable for tests; defaults to global fetch. */
+  fetch?: typeof fetch;
+}

package/src/inference/openai-compatible.ts

@@ -0,0 +1,113 @@
+import type {
+  IChatMessage,
+  ICompleteOptions,
+  IModelResponse,
+  IProvider,
+  IOpenAICompatibleConfig,
+} from "./inference.types";
+import { PROVIDER_LIMITS } from "./inference.constants";
+import { fetchWithRetry } from "./transport";
+import { toWire, parseResponse } from "./wire";
+import { streamResponse } from "./stream";
+
+export { salvageToolCalls } from "./wire";
+
+/**
+ * Talks to any OpenAI-compatible `/chat/completions` endpoint — which Ollama,
+ * vLLM, and llama.cpp all expose for a local Qwen3.6. Supports streaming: pass
+ * `onToken` to receive reasoning + content tokens as they arrive. The wire
+ * mapping lives in ./wire, the SSE assembly in ./stream, and connection retry in
+ * ./transport — this class just orchestrates one request.
+ */
+export class OpenAICompatibleProvider implements IProvider {
+  constructor(private cfg: IOpenAICompatibleConfig) {}
+
+  /** Hot-swap the endpoint/model/key (used by `/model` to switch live): the
+   *  running session keeps this provider reference and picks up the new config on
+   *  its next request — no restart. */
+  reconfigure(cfg: IOpenAICompatibleConfig): void {
+    this.cfg = cfg;
+  }
+
+  /** The current config — read by the CLI for the model/endpoint status line. */
+  get config(): IOpenAICompatibleConfig {
+    return this.cfg;
+  }
+
+  async complete(
+    messages: IChatMessage[],
+    opts: ICompleteOptions = {}
+  ): Promise<IModelResponse> {
+    const doFetch = this.cfg.fetch ?? fetch;
+    const streaming = opts.onToken !== undefined;
+    const headers: Record<string, string> = {
+      "content-type": "application/json",
+    };
+
+    if (this.cfg.apiKey !== undefined) {
+      headers.authorization = `Bearer ${this.cfg.apiKey}`;
+    }
+
+    const body = JSON.stringify({
+      model: this.cfg.model,
+      messages: messages.map(toWire),
+      max_tokens: this.cfg.maxTokens ?? PROVIDER_LIMITS.maxTokens,
+      temperature: opts.temperature,
+      ...(this.cfg.repetitionPenalty === undefined
+        ? {}
+        : { repetition_penalty: this.cfg.repetitionPenalty }),
+      ...(opts.tools === undefined
+        ? {}
+        : { tools: opts.tools, tool_choice: opts.toolChoice ?? "auto" }),
+      ...(opts.enableThinking === undefined
+        ? {}
+        : { chat_template_kwargs: { enable_thinking: opts.enableThinking } }),
+      ...(opts.thinkingTokenBudget === undefined
+        ? {}
+        : { thinking_token_budget: opts.thinkingTokenBudget }),
+      // include_usage → the stream emits a final chunk carrying token `usage`
+      // (otherwise a streamed response reports none). Non-stream replies carry it
+      // by default.
+      ...(streaming
+        ? { stream: true, stream_options: { include_usage: true } }
+        : {}),
+    });
+
+    // Retry transient CONNECTION blips (socket close / unable-to-connect) — the
+    // connect happens before any stream starts, so retrying is safe for both
+    // streaming and non-streaming. Essential for a long-running CLI; also stops
+    // a network hiccup from wrecking an eval run.
+    const res = await fetchWithRetry(
+      doFetch,
+      `${this.cfg.baseUrl}/chat/completions`,
+      headers,
+      body,
+      this.cfg.timeoutMs ?? PROVIDER_LIMITS.requestTimeoutMs,
+      opts.signal
+    );
+
+    if (!res.ok) {
+      const detail = await responseDetail(res);
+
+      throw new Error(
+        `model request failed: ${res.status}${detail.length > 0 ? ` ${detail}` : ""}`
+      );
+    }
+
+    if (opts.onToken !== undefined) {
+      return streamResponse(res, opts.onToken, opts.ttsrManager);
+    }
+
+    const data: unknown = await res.json();
+
+    return parseResponse(data);
+  }
+}
+
+async function responseDetail(res: Response): Promise<string> {
+  try {
+    return (await res.text()).trim().slice(0, 1000);
+  } catch {
+    return "";
+  }
+}

package/src/inference/stream-guard.ts

@@ -0,0 +1,161 @@
+import type { TokenChannel } from "./inference.types";
+
+/**
+ * Detects the degenerate repetition loops this local model falls into at
+ * temperature 0 — where it spews the same line (or a fixed template like
+ * "I will ensure it is X.") until it hits max_tokens, never emitting a tool
+ * call. The repetition penalty makes this rare; this guard makes it IMPOSSIBLE
+ * to hang the CLI: the stream is aborted the moment a loop is detected, instead
+ * of burning a full 16k-token generation (and then a nudge, and another loop).
+ *
+ * Watches the prose channels only (reasoning + content) — file content is
+ * carried in tool-call ARGUMENTS, never here, so code can't false-positive.
+ * Thresholds are deliberately high: real narration never repeats one short line
+ * 24 times, nor shares a 4-word prefix across 20 of 24 consecutive lines.
+ *
+ * The sliding-window checks only see repetition whose PERIOD fits inside WINDOW.
+ * A model that re-prints a large block (e.g. a ~30-line function + paragraphs of
+ * "wait, I think I see the issue…") loops with a period far bigger than 24, so
+ * every window slice looks distinct and they miss it entirely. The period-
+ * AGNOSTIC counter below catches that: any one long line emitted many times
+ * across the whole generation is a loop no matter how big the repeating block.
+ */
+const WINDOW = 24;
+/** Ignore trivial lines (blanks, lone braces, indentation) — only substantial
+ *  lines count toward a loop. */
+const MIN_LINE_LEN = 6;
+/** ≤ this many distinct lines across the window ⇒ near-exact repetition. */
+const MAX_DISTINCT = 3;
+/** ≥ this many lines sharing a 4-word prefix ⇒ templated repetition. */
+const PREFIX_MATCH = 20;
+const PREFIX_WORDS = 4;
+/** A line this long, repeated verbatim `GLOBAL_REPEAT_LIMIT` times anywhere in
+ *  the stream, is a loop — long lines don't recur exactly in real prose/code. */
+const LONG_LINE_LEN = 20;
+const GLOBAL_REPEAT_LIMIT = 5;
+
+/** Markers that the model has started emitting STRUCTURED tool calls into the
+ *  content channel — a server tool-call-parser mismatch (e.g. atlas-spark's
+ *  Qwen3.5-native `<function=…>` XML, which Atlas's parsers don't match) leaves
+ *  them in `content` instead of `tool_calls`. Once seen, content is no longer
+ *  prose, so the prose-loop guard must stand down for it (the leaked calls are
+ *  salvaged + deduped downstream). */
+const TOOL_MARKUP_RE = /<function=|<tool_call>/i;
+
+type ProseChannel = "reasoning" | "content";
+
+export class StreamGuard {
+  private readonly lines: Record<ProseChannel, string[]> = {
+    reasoning: [],
+    content: [],
+  };
+  private readonly partial: Record<ProseChannel, string> = {
+    reasoning: "",
+    content: "",
+  };
+  /** Per-channel count of every substantial line seen across the WHOLE stream —
+   *  backs the period-agnostic large-block loop check. */
+  private readonly counts: Record<ProseChannel, Map<string, number>> = {
+    reasoning: new Map(),
+    content: new Map(),
+  };
+  /** Set once tool-call markup leaks into the content channel — thereafter the
+   *  prose-loop guard stands down for content (see TOOL_MARKUP_RE). */
+  private contentIsToolMarkup = false;
+
+  /** Feed a streamed token; returns true once the channel has degenerated. Only
+   *  the prose channels are watched — tool-call output is structured, not a loop
+   *  we'd abort. */
+  observe(text: string, channel: TokenChannel): boolean {
+    if (channel === "tool") {
+      return false;
+    }
+
+    this.partial[channel] += text;
+
+    if (
+      channel === "content" &&
+      !this.contentIsToolMarkup &&
+      TOOL_MARKUP_RE.test(this.partial.content)
+    ) {
+      this.contentIsToolMarkup = true;
+    }
+
+    const segments = this.partial[channel].split("\n");
+
+    this.partial[channel] = segments.pop() ?? "";
+
+    // Content has become leaked tool-call markup, not prose — drain the buffer
+    // (so it stays bounded) but don't run the prose-loop checks on it.
+    if (channel === "content" && this.contentIsToolMarkup) {
+      return false;
+    }
+
+    for (const segment of segments) {
+      const trimmed = segment.trim();
+
+      if (trimmed.length < MIN_LINE_LEN) {
+        continue;
+      }
+
+      // Period-agnostic: a long line repeated many times anywhere in the stream
+      // is a loop even when the repeating BLOCK is larger than WINDOW (which the
+      // sliding-window checks below would miss).
+      if (trimmed.length >= LONG_LINE_LEN) {
+        const counts = this.counts[channel];
+        const seen = (counts.get(trimmed) ?? 0) + 1;
+
+        counts.set(trimmed, seen);
+
+        if (seen >= GLOBAL_REPEAT_LIMIT) {
+          return true;
+        }
+      }
+
+      const window = this.lines[channel];
+
+      window.push(trimmed);
+
+      if (window.length > WINDOW) {
+        window.shift();
+      }
+
+      if (window.length === WINDOW && isRepetitive(window)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}
+
+function isRepetitive(window: string[]): boolean {
+  const distinct = new Set(window).size;
+
+  if (distinct <= MAX_DISTINCT) {
+    return true;
+  }
+
+  // Block repetition: the model loops a multi-line block (e.g. re-printing the
+  // same "cat X / npx tsc / echo …" sequence). The lines vary within the block,
+  // so the exact-line check above misses it — but half-or-more of the window
+  // being duplicates is a loop no real prose produces.
+  if (distinct <= Math.floor(WINDOW / 2)) {
+    return true;
+  }
+
+  const prefixCounts = new Map<string, number>();
+
+  for (const line of window) {
+    const prefix = line.split(/\s+/).slice(0, PREFIX_WORDS).join(" ");
+    const next = (prefixCounts.get(prefix) ?? 0) + 1;
+
+    prefixCounts.set(prefix, next);
+
+    if (next >= PREFIX_MATCH) {
+      return true;
+    }
+  }
+
+  return false;
+}