loreli 0.0.0 → 2.0.0

package/packages/agent/src/trace.js

@@ -0,0 +1,186 @@
+/**
+ * Agent trace formatting and token parsing utilities.
+ *
+ * Produces marker-wrapped `<details>` blocks for embedding agent
+ * reasoning, terminal output, and token usage in PR bodies and
+ * review comments. Uses the loreli marker system for programmatic
+ * detection and stripping.
+ *
+ * @module loreli/agent/trace
+ */
+
+import { mark } from 'loreli/marker';
+
+/**
+ * Registry of regex patterns for extracting token usage from CLI output.
+ * Each entry has a `name`, a `pattern` regex, and an `extract` function
+ * that receives the match and returns `{ input, output }`.
+ *
+ * New backends or updated CLI versions can be supported by adding
+ * entries here without changing the `tokens()` function itself.
+ *
+ * @type {Array<{name: string, pattern: RegExp, extract: function}>}
+ */
+const TOKEN_PATTERNS = [
+  {
+    name: 'codex',
+    pattern: /tokens?\s+used\s*\n?\s*(\d[\d,]*)/i,
+    extract(m) {
+      const total = parseInt(m[1].replaceAll(',', ''), 10);
+      return { input: 0, output: total };
+    }
+  },
+  {
+    name: 'codex-split',
+    pattern: /input[:\s]+(\d[\d,]*)\s*(?:tokens?)?\s*[,|/]\s*output[:\s]+(\d[\d,]*)/i,
+    extract(m) {
+      return {
+        input: parseInt(m[1].replaceAll(',', ''), 10),
+        output: parseInt(m[2].replaceAll(',', ''), 10)
+      };
+    }
+  },
+  {
+    name: 'claude',
+    pattern: /total\s+tokens?[:\s]+input\s*=\s*(\d[\d,]*)\s+output\s*=\s*(\d[\d,]*)/i,
+    extract(m) {
+      return {
+        input: parseInt(m[1].replaceAll(',', ''), 10),
+        output: parseInt(m[2].replaceAll(',', ''), 10)
+      };
+    }
+  }
+];
+
+/**
+ * Parse token usage from CLI terminal output.
+ *
+ * Iterates the pattern registry and returns the first match.
+ * Returns `null` when no recognized pattern is found — this is
+ * expected for backends like cursor-agent that do not print
+ * token info.
+ *
+ * @param {string} text - Cleaned terminal output.
+ * @returns {{ input: number, output: number } | null}
+ */
+export function tokens(text) {
+  if (!text) return null;
+
+  for (const entry of TOKEN_PATTERNS) {
+    const m = text.match(entry.pattern);
+    if (m) return entry.extract(m);
+  }
+
+  return null;
+}
+
+/**
+ * Escape captured output so it cannot interfere with the loreli
+ * marker system or break markdown rendering.
+ *
+ * Strips:
+ * - HTML comments that look like loreli markers (`<!-- loreli:... -->`)
+ * - Null bytes and other ASCII control characters (except newline/tab)
+ *
+ * @param {string} text - Raw captured output.
+ * @returns {string} Sanitized text safe for embedding in code fences.
+ */
+export function escape(text) {
+  if (!text) return '';
+  return text
+    .replace(/<!-- loreli:[^>]*-->/g, '')
+    // eslint-disable-next-line no-control-regex
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
+}
+
+/**
+ * Determine the minimum number of backticks needed for a code fence
+ * that will not be broken by backtick sequences in the content.
+ *
+ * @param {string} text - Content to be fenced.
+ * @returns {number} Fence length (at least 3).
+ */
+function fenceLength(text) {
+  let max = 0;
+  const re = /`+/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    if (m[0].length > max) max = m[0].length;
+  }
+  return Math.max(3, max + 1);
+}
+
+/**
+ * Format a complete agent trace block.
+ *
+ * Produces a marker-delimited `<details>` section suitable for
+ * embedding in PR bodies or review comments. The block is wrapped
+ * with `loreli:trace` / `loreli:trace-end` markers so downstream
+ * consumers can strip it via `excise(body, 'trace')`.
+ *
+ * Sections are omitted when their data is absent:
+ * - `reasoning` — agent-provided summary of approach
+ * - `output` — captured terminal output (in a code fence)
+ * - `usage` — token counts from `tokens()`
+ *
+ * @param {string} name - Agent identity name (e.g. 'bumblebee-0').
+ * @param {object} data - Trace data.
+ * @param {string} [data.reasoning] - Agent-provided reasoning summary.
+ * @param {string} [data.output] - Captured terminal output.
+ * @param {{ input: number, output: number } | null} [data.usage] - Token usage.
+ * @param {string} [data.model] - Model identifier for the usage table.
+ * @param {number} [data.duration] - Duration in milliseconds.
+ * @returns {string} Complete trace block with markers.
+ */
+export function format(name, data = {}) {
+  const { reasoning, output: raw, usage, model, duration } = data;
+  const sections = [];
+
+  if (reasoning) {
+    sections.push('### Reasoning\n');
+    sections.push(reasoning);
+  }
+
+  if (raw) {
+    const cleaned = escape(raw);
+    const len = fenceLength(cleaned);
+    const fence = '`'.repeat(len);
+    sections.push('### Output\n');
+    sections.push(`${fence}\n${cleaned}\n${fence}`);
+  }
+
+  if (usage || model || duration != null) {
+    const rows = [];
+    if (usage?.input != null) rows.push(`| Input tokens | ${usage.input.toLocaleString()} |`);
+    if (usage?.output != null) rows.push(`| Output tokens | ${usage.output.toLocaleString()} |`);
+    if (model) rows.push(`| Model | ${model} |`);
+    if (duration != null) {
+      const secs = Math.round(duration / 1000);
+      const mins = Math.floor(secs / 60);
+      const rem = secs % 60;
+      const display = mins > 0 ? `${mins}m ${rem}s` : `${secs}s`;
+      rows.push(`| Duration | ${display} |`);
+    }
+    if (rows.length) {
+      sections.push('### Usage\n');
+      sections.push(`| Metric | Value |\n|--------|-------|\n${rows.join('\n')}`);
+    }
+  }
+
+  if (!sections.length) return '';
+
+  const body = sections.join('\n\n');
+  const startMarker = mark('trace', { agent: name });
+  const endMarker = `<!-- loreli:trace-end -->`;
+
+  return [
+    startMarker,
+    `<details>`,
+    `<summary>Agent Trace (${name})</summary>`,
+    '',
+    body,
+    '',
+    `</details>`,
+    endMarker
+  ].join('\n');
+}

package/packages/classify/README.md

@@ -0,0 +1,136 @@
+# loreli/classify
+
+Prompt-driven LLM classification. Loads a named Mustache template from disk, renders it with the provided content and variables, sends the result through `backends.oneshot()`, and returns the parsed JSON response. The prompt template defines the response shape — classify is generic plumbing.
+
+## Installation
+
+Part of the Loreli monorepo. Import via the package exports map:
+
+```js
+import { classify } from 'loreli/classify';
+```
+
+## Quick Start
+
+Classification requires a `BackendRegistry` with at least one available LLM backend and a prompt template in `packages/classify/prompts/`:
+
+```js
+import { classify } from 'loreli/classify';
+
+const result = await classify('pane-state', paneOutput, {
+  backends: backendRegistry
+});
+// => { category: 'option_dialog', reasoning: 'Trust dialog detected', confidence: 0.9 }
+```
+
+The first argument is the template name — it resolves to `prompts/<name>.md` inside the package. The second argument is the text to classify. The template defines what categories exist, what JSON shape to return, and how the LLM should reason about the input.
+
+## How It Works
+
+```text
+classify('pane-state', text, opts)
+  │
+  ├─ Load prompts/pane-state.md
+  ├─ Mustache.render(template, { content: text, ...opts.vars })
+  ├─ backends.oneshot(rendered, { model, timeout })
+  └─ Parse JSON from LLM response → return object
+```
+
+1. **Load** — Reads `prompts/<name>.md` from disk.
+2. **Render** — Runs Mustache templating. The content is available as `{{{content}}}` (triple-stache, unescaped). Extra variables via `opts.vars` are also available.
+3. **Send** — Calls `backends.oneshot()` with the rendered prompt.
+4. **Parse** — Extracts the first `{...}` JSON object from the response, handling markdown fences and preamble text.
+
+## Prompt Templates
+
+Templates are Markdown files in `packages/classify/prompts/`. Each template contains the full classification instructions for the LLM, including category definitions and the expected JSON output shape.
+
+### `pane-state.md` — orchestrator stall detection
+
+Used by the orchestrator's monitor loop and rapid-death detector to diagnose agent state. Returns `{category, reasoning, confidence}` with states: `working`, `waiting_for_input`, `option_dialog`, `error_loop`, `idle`, `fatal`, `dead`. The `dead` state identifies agents whose process exited or crashed — the orchestrator uses it in the rapid-death window (first 15s after spawn) to classify *why* an agent died, replacing the old binary alive/dead check with diagnostic context.
+
+### `feedback.md` — knowledge feedback categorization
+
+Used by `loreli/knowledge` to classify review feedback. Returns `{category, reasoning, confidence}` with categories: `naming`, `architecture`, `testing`, `documentation`, `performance`, `security`.
+
+### `blocker.md` — knowledge per-ref blocker detection
+
+Used by `loreli/knowledge` to classify issue/PR references as blockers or informational. Receives `{{{content}}}` (joined discussion text) and `{{refs}}` (formatted ref list). Returns `{blockers: [number], references: [number]}`.
+
+### Writing a new template
+
+Create a new `.md` file in `prompts/`. Use `{{{content}}}` for the text to classify (triple-stache prevents HTML escaping). Use `{{varName}}` for additional variables passed via `opts.vars`. End the template with the expected JSON shape so the LLM knows what to return.
+
+The following example shows how a severity template might look:
+
+```markdown
+Classify this error message by severity.
+
+Levels:
+- critical: System is down or data loss is occurring
+- warning: Something is wrong but the system is functional
+- info: Normal operational message
+
+Respond with ONLY a JSON object.
+{"level": "<severity>", "reasoning": "<one sentence>"}
+
+{{{content}}}
+```
+
+Then call it:
+
+```js
+const result = await classify('severity', errorMessage, { backends });
+// => { level: 'critical', reasoning: 'Database connection lost' }
+```
+
+## API Reference
+
+### `classify(name, content, opts)`
+
+Run a named classification prompt against content via LLM.
+
+**Parameters:**
+
+| Name | Type | Default | Description |
+|------|------|---------|-------------|
+| `name` | `string` | — | Template name — resolves to `prompts/<name>.md`. |
+| `content` | `string` | — | Text to classify — injected as `{{{content}}}`. |
+| `opts.backends` | `BackendRegistry` | — | **Required.** Backend registry with `oneshot()` method. |
+| `opts.config` | `Config` | `undefined` | Config instance for model/timeout resolution. |
+| `opts.model` | `string` | `'fast'` | Model alias override. Falls back to `config.classify.model`, then `'fast'`. |
+| `opts.timeout` | `number` | `30000` | Timeout in ms. Falls back to `config.classify.timeout`, then `30000`. |
+| `opts.vars` | `object` | `{}` | Extra Mustache variables beyond `content`. |
+
+**Returns:** `Promise<object>` — Parsed JSON from the LLM. Shape is defined by the prompt template, not enforced by classify.
+
+**Throws:**
+
+| Error | Cause |
+|-------|-------|
+| `classify() requires a backends instance` | `opts.backends` is missing or falsy. |
+| `ENOENT` | Template file `prompts/<name>.md` does not exist. |
+| LLM error (propagated) | `backends.oneshot()` threw (timeout, network, etc.). |
+| `classify: LLM response contains no JSON object` | Response had no `{...}` block. |
+| `classify: failed to parse JSON from LLM response` | Found `{...}` but it was not valid JSON. |
+
+## Configuration
+
+The classify package reads configuration from the `classify` section in `loreli.yml`:
+
+```yaml
+classify:
+  model: fast         # Model alias — resolves via backends
+  maxLines: 100       # Lines of pane output to capture (used by orchestrator)
+  timeout: 30s        # Timeout for the oneshot CLI call
+  maxRetries: 5       # Consecutive failures before safety-net kill
+```
+
+## Error Reference
+
+| Error | Cause | Resolution |
+|-------|-------|------------|
+| `classify() requires a backends instance` | No `backends` option provided. | Pass a `BackendRegistry` instance in `opts.backends`. |
+| Template `ENOENT` | Named template does not exist in `prompts/`. | Create the template file or check the name for typos. |
+| `LLM response contains no JSON object` | The LLM returned prose without any JSON. | Check the template instructions — they should explicitly ask for JSON-only output. |
+| `failed to parse JSON from LLM response` | JSON was found but was malformed. | Usually a transient LLM issue. Retry or use a more capable model. |

package/packages/classify/prompts/blocker.md

@@ -0,0 +1,12 @@
+Analyze this discussion text and classify each referenced issue/PR number as either a blocking dependency or an informational reference.
+
+Referenced issues: {{refs}}
+
+A reference is a **blocker** if the text indicates it must be resolved, merged, or closed before this work can proceed (e.g., "blocked by", "depends on", "needs #N merged first", "waiting for #N").
+
+A reference is **informational** if it is mentioned for context, background, or related reading (e.g., "see #N for context", "related to #N", "similar to #N").
+
+Respond with ONLY a JSON object. Do not wrap in markdown. Do not add any other text.
+{"blockers": [<number>, ...], "references": [<number>, ...]}
+
+{{{content}}}

package/packages/classify/prompts/feedback.md

@@ -0,0 +1,14 @@
+Classify this code review feedback into exactly one category.
+
+Categories:
+- naming: Feedback about naming conventions, variable names, or renaming suggestions
+- architecture: Feedback about code structure, module organization, or refactoring
+- testing: Feedback about test coverage, assertions, or testing practices
+- documentation: Feedback about documentation, README, JSDoc, or code comments
+- performance: Feedback about performance optimization, memory, or caching
+- security: Feedback about security, secrets, authentication, or vulnerabilities
+
+Respond with ONLY a JSON object. Do not wrap in markdown. Do not add any other text.
+{"category": "<name>", "reasoning": "<one sentence explanation>", "confidence": <0.0 to 1.0>}
+
+{{{content}}}

package/packages/classify/prompts/pane-state.md

@@ -0,0 +1,20 @@
+Classify this terminal output from an AI coding agent into exactly one state.
+{{#model}}
+The agent was launched with model `{{model}}` on the `{{backend}}` backend (role: {{role}}).
+{{/model}}
+
+States:
+- working: Agent is mid-task, output is progressing normally
+- waiting_for_input: Agent at a prompt waiting for user input
+- option_dialog: Agent showing a Y/N or selection dialog that needs a keystroke
+- error_loop: Agent repeating the same error without making progress
+- idle: Agent finished all tasks or has no pending work
+- fatal: Agent hit a fatal infrastructure error (rate limit, auth failure, budget exhaustion, invalid model)
+- dead: Agent process exited or crashed — output shows exit code, stack trace, or abrupt termination
+
+For option_dialog, include the tmux key names needed to dismiss the dialog in `remedy` (e.g. "Enter", "Down Enter", "Escape"). For all other states, set remedy to null.
+
+Respond with ONLY a JSON object. Do not wrap in markdown. Do not add any other text.
+{"category": "<state>", "reasoning": "<one sentence explanation>", "confidence": <0.0 to 1.0>, "remedy": "<tmux keys or null>"}
+
+{{{content}}}

package/packages/classify/src/index.js

@@ -0,0 +1,81 @@
+/**
+ * Prompt-driven LLM classification.
+ *
+ * Loads a named Mustache template from disk, renders it with the provided
+ * content and variables, sends the result through `backends.oneshot()`,
+ * and returns the parsed JSON response. The prompt template defines the
+ * response shape — classify is generic plumbing.
+ *
+ * @module loreli/classify
+ */
+
+import { readFile } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import Mustache from 'mustache';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const prompts = join(__dirname, '..', 'prompts');
+
+/**
+ * Extract a JSON object from LLM response text.
+ *
+ * LLMs sometimes wrap JSON in markdown fences or add preamble.
+ * This extracts the first `{...}` block from the response.
+ *
+ * @param {string} raw - Raw LLM response.
+ * @returns {object} Parsed JSON object.
+ * @throws {Error} When no valid JSON is found in the response.
+ */
+function extract(raw) {
+  const fenced = raw.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const json = fenced ? fenced[1].trim() : raw.trim();
+
+  const start = json.indexOf('{');
+  const end = json.lastIndexOf('}');
+  if (start === -1 || end === -1) {
+    throw new Error('classify: LLM response contains no JSON object');
+  }
+
+  try {
+    return JSON.parse(json.slice(start, end + 1));
+  } catch (err) {
+    throw new Error(`classify: failed to parse JSON from LLM response — ${err.message}`);
+  }
+}
+
+/**
+ * Run a named classification prompt against content via LLM.
+ *
+ * Loads `prompts/<name>.md`, renders it with Mustache using `content`
+ * and any extra `vars`, sends the rendered prompt through
+ * `backends.oneshot()`, and returns the parsed JSON from the response.
+ *
+ * @param {string} name - Prompt template name (resolves to `prompts/<name>.md`).
+ * @param {string} content - Text to classify — injected as `{{{content}}}`.
+ * @param {object} opts - Options.
+ * @param {object} opts.backends - BackendRegistry instance with a `oneshot()` method. Required.
+ * @param {object} [opts.config] - Config instance for model/timeout resolution.
+ * @param {string} [opts.model] - Model alias override.
+ * @param {number} [opts.timeout] - Timeout for the oneshot call in ms.
+ * @param {object} [opts.vars] - Extra Mustache variables beyond `content`.
+ * @returns {Promise<object>} Parsed JSON from the LLM response. Shape is prompt-defined.
+ * @throws {Error} When backends is missing, template not found, oneshot fails, or response has no valid JSON.
+ */
+export async function classify(name, content, opts = {}) {
+  const { backends, config, model, timeout, vars } = opts;
+
+  if (!backends) throw new Error('classify() requires a backends instance');
+
+  const path = join(prompts, `${name}.md`);
+  const template = await readFile(path, 'utf8');
+  const rendered = Mustache.render(template, { content, ...vars });
+
+  const raw = await backends.oneshot(rendered, {
+    model: model ?? config?.get?.('classify.model') ?? 'fast',
+    config,
+    timeout: timeout ?? config?.get?.('classify.timeout') ?? 60000
+  });
+
+  return extract(raw);
+}