llm-cli-gateway 2.14.0-rc.1 → 2.14.1

package/.agents/skills/retrospective-walk/SKILL.md

@@ -0,0 +1,294 @@
+---
+name: retrospective-walk
+description: Walk a human or agent through a diff, worktree, commit range, gateway job, or episode reference as a structured retrospective. Use after implement-review-fix or multi-LLM review cycles, when reviewing prior jobs or uncommitted work, or when durable evidence is needed for what changed, why it changed, who/when contributed, and captured human or model comments.
+---
+
+# Retrospective Walk
+
+Guide a change review as a narrative with evidence. Prefer this when the user
+needs the story of a change set, not just hunk-level inspection.
+
+## Inputs
+
+Accept these fields when the user supplies them, and infer conservative
+defaults when they do not:
+
+```json
+{
+  "scope": "diff | commit-range | worktree | job-id | episode-ref",
+  "target": "diff text, HEAD~5..HEAD, path, job id, or episode reference",
+  "mode": "guided | agent-driven | summary-only",
+  "capture_comments": true,
+  "models_for_why": [],
+  "include_prior_evidence": true
+}
+```
+
+Default `mode` to `guided` for a human in the loop, `agent-driven` for automated
+review, and `summary-only` only when explicitly requested or when interaction is
+not possible. Default `capture_comments` to true.
+
+## Scope Resolution
+
+Resolve the target into immutable evidence before analysis:
+
+- `diff`: use the supplied diff text. If the target is empty, capture
+  `git diff --no-ext-diff` and `git diff --cached --no-ext-diff`.
+- `commit-range`: capture `git diff --stat <range>`, `git diff --name-status
+  <range>`, and per-commit metadata from `git log --format=fuller <range>`.
+- `worktree`: capture status, dirty file list, staged diff, and unstaged diff.
+  Do not stash, reset, or mutate the target worktree. If deeper model/tool
+  analysis may write files, create a disposable isolated worktree or analyze a
+  read-only evidence packet instead.
+- `job-id`: fetch `llm_job_status` and `llm_job_result`; include `cli`,
+  `correlationId`, `sessionId`, timestamps, exit status, and output digests.
+- `episode-ref`: resolve through the caller's available episode/DAG context. If
+  no episode tool is available, record the unresolved reference and continue
+  from any linked diff, job, receipt, or review artifacts the user supplied.
+
+If `include_prior_evidence` is true, collect linked validation runs, receipts,
+review reports, approval records, job ids, and session/cache state that are
+available through gateway tools or local artifacts. Treat summaries as claims;
+prefer raw diffs, job outputs, receipt ids, commands, and file:line evidence.
+
+## Change Units
+
+Group hunks and files into logical change units before narrating them. Favor
+semantic intent over file boundaries, but keep units small enough to review.
+
+Use these grouping signals:
+
+- Shared feature, bug, invariant, or user-facing behavior.
+- Production code plus its tests and docs.
+- Config/schema/migration changes that must be deployed together.
+- Review-only or evidence-only changes, such as verification reports.
+
+Assign stable ids: `cu_001`, `cu_002`, etc. For each unit, record:
+
+```json
+{
+  "change_unit_id": "cu_001",
+  "title": "Short noun phrase",
+  "files": ["src/example.ts"],
+  "diff_refs": ["commit abc123", "hunk @@ -42,7 +42,9 @@"],
+  "what": "Semantic summary of the delta",
+  "why": "Intent, rationale, alternatives, and unresolved assumptions",
+  "who_when": [
+    {
+      "actor_type": "human | model | agent | git-author | unknown",
+      "identifier": "name, model key, job id, or commit author",
+      "timestamp": "RFC3339 if known",
+      "provenance": "commit, job, session, receipt, or user statement"
+    }
+  ],
+  "evidence": ["file:line", "job id", "validation receipt id", "command digest"],
+  "comments": []
+}
+```
+
+## What, Why, Who
+
+For each change unit:
+
+1. Extract **what** directly from the diff, file reads, tests, and docs.
+2. Derive **why** from commit messages, issue/DAG context, review findings,
+   prior receipts, job prompts/results, and nearby code intent.
+3. Attribute **who / when** using git author/committer metadata, gateway job
+   metadata, session ids, model keys, review outputs, and explicit user
+   statements.
+4. Mark inference boundaries. Use `inferred:` when intent or authorship is
+   reasoned from evidence rather than directly stated.
+5. Surface missing evidence as an open question instead of filling the gap with
+   confidence.
+
+Use `multi-llm-review` or direct gateway model calls only when why-synthesis is
+ambiguous, high stakes, or explicitly requested. Omit `model` unless the user
+requested specific variants; use `approvalStrategy:"mcp_managed"` and async job
+handling from `async-job-orchestration` for longer analysis. Ask models for
+intent/rationale synthesis, not for unverified approval.
+
+## Comment Capture
+
+When `capture_comments` is true, comments are evidence. Capture them at the
+change-unit level and, when possible, at the file/hunk/line location.
+
+In `guided` mode, walk one unit at a time:
+
+1. Present the unit's what/why/who and evidence.
+2. Ask for comments, concerns, approvals, suggestions, or intent corrections.
+3. Normalize each answer into the schema below.
+4. Continue only after comments are captured or the user skips the unit.
+
+In `agent-driven` mode, create agent/model comments only when they add evidence,
+concern, approval, or intent clarification beyond the narrative. In
+`summary-only` mode, do not stop for comments; include an empty comments array
+unless comments were supplied up front.
+
+Comment schema:
+
+```json
+{
+  "comment_id": "cmt_<ulid>",
+  "retrospective_id": "retro_<ulid>",
+  "change_unit_id": "cu_001",
+  "author": {
+    "author_type": "human | model",
+    "identifier": "werner or claude-4-opus",
+    "model_key": "optional; only for model comments"
+  },
+  "timestamp": "RFC3339",
+  "type": "note | concern | approval | suggestion | intent_clarification",
+  "text": "Comment body",
+  "location": {
+    "file": "optional path",
+    "hunk_range": "optional @@ range",
+    "semantic_unit": "optional semantic code-search node or future semantic id",
+    "line_start": 1,
+    "line_end": 2
+  },
+  "linked_receipts": ["validation receipt ids"],
+  "linked_jobs": ["async job ids"],
+  "metadata": {
+    "namespace.key": "freeform extension values"
+  }
+}
+```
+
+Generate ids in a stable, sortable form when the runtime has a ULID helper;
+otherwise use `retro_<timestamp>_<short-hash>` and
+`cmt_<timestamp>_<sequence>`.
+
+## Output Contract
+
+Emit both a readable retrospective and a machine-readable block. Keep the
+machine block fenced as `json` and make it complete enough to persist.
+
+Markdown structure:
+
+```markdown
+# Retrospective Walk: <target>
+
+Retrospective ID: retro_<id>
+Scope: <scope>
+Mode: <mode>
+Evidence baseline: <commands, commits, jobs, receipts>
+
+## Narrative
+<overall story: what happened, why, and remaining uncertainty>
+
+## Change Units
+### cu_001 - <title>
+What: ...
+Why: ...
+Who / when: ...
+Evidence: ...
+Comments: ...
+
+## Open Questions
+- ...
+
+## Evidence Package
+- validation_receipt: <id or unavailable: reason>
+- linked_jobs: [...]
+- linked_receipts: [...]
+```
+
+Machine-readable shape:
+
+```json
+{
+  "retrospective_id": "retro_<id>",
+  "schema_version": "retrospective-walk.v0.1.0",
+  "scope": "commit-range",
+  "target": "HEAD~5..HEAD",
+  "mode": "guided",
+  "created_at": "RFC3339",
+  "provenance_context": {
+    "commands": [],
+    "commits": [],
+    "jobs": [],
+    "receipts": [],
+    "episodes": []
+  },
+  "change_units": [],
+  "comments": [],
+  "narrative": "",
+  "open_questions": [],
+  "validation_receipt": {
+    "status": "minted | linked | unavailable",
+    "validation_id": "optional",
+    "receipt_id": "optional",
+    "reason": "optional"
+  }
+}
+```
+
+## Validation Receipts
+
+Use existing gateway validation receipt machinery when the retrospective is
+backed by a terminal validation run:
+
+- For prior validation ids, call `validation_receipt` and link the receipt.
+- For review or why-synthesis model jobs launched during the retrospective,
+  preserve job ids and retrieve receipts if those jobs are part of a validation
+  run that can mint one.
+- If no receipt-capable validation run exists, set
+  `validation_receipt.status` to `unavailable` and include a reason. Do not
+  claim a receipt was minted when the current gateway surface only produced a
+  retrospective artifact.
+
+## DAG Plan
+
+Use this plan shape when embedding the retrospective in a DAG or episode:
+
+```toml
+[plan]
+name = "retrospective-walk"
+version = "0.1.0"
+description = "Guided retrospective over changes with what/why/who and comment capture as evidence"
+owner = "verivus-oss"
+
+[[node]]
+id = "resolve-scope"
+type = "data"
+outputs = ["change_units", "provenance_context"]
+
+[[node]]
+id = "analyze-changes"
+type = "compute"
+inputs = ["change_units", "provenance_context", "models_for_why"]
+outputs = ["analyzed_units"]
+
+[[node]]
+id = "capture-comments"
+type = "human-or-agent"
+inputs = ["analyzed_units"]
+outputs = ["comments"]
+
+[[node]]
+id = "synthesize-retrospective"
+type = "compute"
+inputs = ["analyzed_units", "comments", "provenance_context"]
+outputs = ["retrospective_markdown", "retrospective_json"]
+
+[[node]]
+id = "emit-evidence"
+type = "side-effect"
+inputs = ["retrospective_json", "comments", "analyzed_units"]
+outputs = ["validation_receipt_id"]
+```
+
+Only update a DAG or episode when the caller provided a parent context and an
+available tool supports that write. Otherwise emit the retrospective and the
+evidence package without side effects.
+
+## Security And Isolation
+
+- Prefer read-only evidence capture. Do not mutate user worktrees while walking
+  changes.
+- Use a disposable isolated git worktree for model/tool operations that may edit
+  files, especially for `scope = worktree`.
+- Keep secrets, local account names, and machine-specific paths out of
+  persisted comments and receipts unless the user explicitly requires them.
+- Treat model-generated comments as model evidence, not human approval.
+- Preserve provenance for every comment, job, receipt, and inferred rationale.

package/CHANGELOG.md

@@ -2,6 +2,58 @@
 
 All notable changes to the llm-cli-gateway project.
 
+## [Unreleased]
+
+## [2.14.1] - 2026-07-03
+
+### Added
+
+- **`retrospective-walk` workflow skill.** Bundles a guided retrospective skill
+  for walking a human or agent through a diff, worktree, commit range, gateway
+  job, or episode reference with structured what/why/who analysis, comment
+  capture as evidence, machine-readable output, and validation-receipt links
+  when an existing receipt-capable validation run is available.
+
+## [2.14.0] - 2026-07-03
+
+### Fixed
+
+- **Durable instance-lease orphan recovery (#139).** The blanket
+  `markOrphanedOnStartup` sweep in the `AsyncJobManager` constructor rewrote
+  every `running` row to `orphaned`, so on a SHARED store (`backend =
+  "postgres"`) a fresh instance (especially an ephemeral stdio spawn)
+  transiently orphaned other live instances' in-flight jobs (a `running` ->
+  `orphaned` -> `completed` flap a poller can trip on). The sweep is now a
+  per-job fencing lease: each instance registers a lease and advances a
+  `jobs.lease_deadline` on every heartbeat, and the sweep orphans a
+  `queued`/`running` job only when its own `lease_deadline` has expired, never
+  because a different live instance started. Because heartbeat and sweep are
+  both `UPDATE`s on the same `jobs` rows, they serialize on the row lock
+  (Postgres READ COMMITTED) and are trivially serial under single-writer sqlite,
+  so no job whose owner heartbeated within the lease TTL is ever swept. A
+  genuinely stale-then-reviving owner self-heals to the correct terminal state
+  via the guarded `recordComplete` and a flight-recorder reconcile. The fix is
+  transport-aware (an extra `httpJobGrace` for no-pid http jobs, an advisory
+  never-vetoing `kill(pid,0)` for same-host process jobs), durable-admission is
+  fail-closed (a failed `recordStart`/`registerInstance` fails the request and
+  releases the limiter permit), and graceful shutdown drains in-flight terminal
+  writes before deregistering the lease. Additive schema (`jobs.owner_instance`,
+  `jobs.lease_deadline`, a `gateway_instances` table), auto-created on both
+  sqlite and postgres, no data migration.
+
+### Changed
+
+- **`[persistence].ownsOrphanRecovery` is deprecated (#139).** The interim gate
+  (PR #140) that let one designated `postgres` instance own the blanket startup
+  sweep is superseded by the durable lease above, which is safe to run from
+  every instance. The flag is still parsed (so existing configs do not error)
+  and now emits a one-time deprecation warning; it no longer changes behaviour
+  and will be removed in a later release. New `[persistence]` knobs tune the
+  lease: `instanceHeartbeatMs` (15000), `instanceLeaseTtlMs` (90000),
+  `httpJobGraceMs` (300000), `orphanSweepIntervalMs` (30000), `instanceGcMs`
+  (3600000), validated so `instanceLeaseTtlMs >= 2 * instanceHeartbeatMs` and
+  `httpJobGraceMs >= instanceLeaseTtlMs`.
+
 ## [2.14.0-rc.1] - 2026-07-03: full-featured provider integration + security-review hardening
 
 Release candidate. Every non-Cursor provider (Claude, Codex, Gemini, Grok,

package/README.md

@@ -9,11 +9,11 @@
 > _"Without consultation, plans are frustrated, but with many counselors they succeed."_
 > — Proverbs 15:22 (LSB)
 
-**Cross-model review without rebuilding your agent stack.**
+**Secure local control plane for AI coding agents.**
 
-`llm-cli-gateway` gives any MCP client one local-first gateway for Claude Code, Codex, Gemini/Antigravity, Grok Build, Mistral Vibe, Cognition Devin, Cursor Agent, and configured HTTP API providers, while preserving native CLI sessions, local credentials, durable async jobs, validation receipts, and review workflows.
+`llm-cli-gateway` lets supported MCP clients operate Claude Code, Codex, Gemini/Antigravity, Grok Build, Mistral Vibe, Cognition Devin, Cursor Agent, and configured HTTP API providers through one user-owned gateway while preserving native CLI sessions, local credentials, durable async jobs, validation receipts, and review workflows.
 
-**Why developers try it:** any connected client can ask other models for a second opinion, run implementation/review loops, collect durable job results, and route API-token LLMs without turning local coding agents into a generic chat proxy.
+**Why developers try it:** use the client you are already in to delegate work to local coding agents, scope remote execution to registered workspaces, gate risky actions, survive disconnects, and collect auditable review evidence without turning those agents into a generic chat proxy.
 
 **Current signals:** CI and security workflows pass on `main`, OpenSSF Scorecard is published, OpenSSF Best Practices is passing, releases use Sigstore signing, and the package is MIT licensed.
 
@@ -38,7 +38,7 @@ Or use directly with `npx` from an MCP client:
 
 ## What It Provides Today
 
-`llm-cli-gateway` is a single-user MCP gateway for cross-LLM validation and multi-agent coding workflows. It is more than a thin CLI wrapper:
+`llm-cli-gateway` is a single-user MCP control plane for operating AI coding agents from supported local or remote clients. It is more than a thin CLI wrapper:
 
 - Runs registered provider CLIs and configured HTTP API providers through consistent sync and async MCP tools.
 - Persists long-running jobs, supports restart-safe result collection, deduplication, cancellation, and sync-to-async deferral.
@@ -51,7 +51,7 @@ Or use directly with `npx` from an MCP client:
 
 ## Workflow Assets
 
-The repo ships agent-ready workflow skills under [`.agents/skills`](.agents/skills) for async orchestration, session continuity, multi-LLM review, implement-review-fix loops, and secure approval-gated dispatch. Six caller-facing skills are bundled in the published npm package: `async-job-orchestration`, `multi-llm-review`, `session-workflow`, `secure-orchestration`, `implement-review-fix`, and `public-demo-session`. Machine-readable DAG-TOML plans live under [`docs/plans`](docs/plans) and [`setup/install-plan.dag.toml`](setup/install-plan.dag.toml) for workflows that need deterministic sequencing and verification gates.
+The repo ships agent-ready workflow skills under [`.agents/skills`](.agents/skills) for async orchestration, session continuity, multi-LLM review, implement-review-fix loops, retrospective evidence walks, and secure approval-gated dispatch. Seven caller-facing skills are bundled in the published npm package: `async-job-orchestration`, `multi-llm-review`, `session-workflow`, `secure-orchestration`, `implement-review-fix`, `retrospective-walk`, and `public-demo-session`. Machine-readable DAG-TOML plans live under [`docs/plans`](docs/plans) and [`setup/install-plan.dag.toml`](setup/install-plan.dag.toml) for workflows that need deterministic sequencing and verification gates.
 
 The next documentation focus is provider-specific skill and DAG-TOML pairs for each outbound CLI and API-provider family: Claude, Codex, Gemini/Antigravity, Grok, Mistral Vibe, Devin, Cursor Agent, OpenAI-compatible endpoints, Anthropic Messages, and xAI Responses. The implementation plan is tracked in [`docs/plans/provider-workflow-assets.dag.toml`](docs/plans/provider-workflow-assets.dag.toml), with each provider asset expected to cover install/login checks or token-env checks, session behavior, approval modes, cache/telemetry surfaces, failure modes, and a smoke-test gate.
 
@@ -69,7 +69,7 @@ The next documentation focus is provider-specific skill and DAG-TOML pairs for e
 
 ## Personal MCP Appliance
 
-The personal-appliance contract keeps that surface intentionally narrow: one trusted user runs the gateway on a machine or volume they own, connects one MCP endpoint, and asks any connected client for cross-LLM validation.
+The personal-appliance contract keeps that surface intentionally narrow: one trusted user runs the gateway on a machine or volume they own, connects one MCP endpoint, and lets supported clients operate local coding agents through workspace-scoped, approval-gated, auditable requests.
 
 The product contract is documented in [docs/personal-mcp/PRODUCT_CONTRACT.md](docs/personal-mcp/PRODUCT_CONTRACT.md). It defines the single-user scope, security posture, target support matrix, and provider-support verification gates. Public setup guides must not claim ChatGPT, Claude web, Claude Desktop, Codex, Gemini CLI, Gemini web, or Grok inbound support until the corresponding provider/client path has been verified.
 
@@ -659,7 +659,7 @@ Every async job is persisted to a job store as it transitions through running
 
 - **Re-issuing a request is safe.** Identical `*_request` / `*_request_async` calls within the dedup window (default 1 hour) short-circuit onto the existing running or completed job — the caller gets back the same job ID instead of starting a duplicate run. This directly fixes the "agent times out polling, re-issues, and the whole job starts over" failure mode.
 - **`llm_job_status` and `llm_job_result` work across gateway restarts.** Job rows live for 30 days by default; callers can collect results long after the in-memory cache has evicted them.
-- **Jobs running at shutdown are marked `orphaned`** on the next gateway boot (the detached child can't be reattached to). Their captured partial output remains readable.
+- **A job is marked `orphaned` only when its owning gateway instance is provably gone**, never because another instance restarted. Each instance holds a periodic heartbeat lease and stamps every job it owns; the recovery sweep orphans a `queued`/`running` job only when that job's own lease has expired. On a shared store (`backend = "postgres"`) this means a fresh instance never orphans another live instance's in-flight jobs. The captured partial output of a genuinely orphaned job remains readable, and a stale-then-reviving owner that later finishes self-heals to the correct terminal state (issue #139).
 - **Pass `forceRefresh: true`** on any request tool to bypass dedup and force a fresh CLI run.
 
 ##### Persistence configuration
@@ -674,6 +674,18 @@ path = "~/.llm-cli-gateway/logs.db"         # for sqlite
 retentionDays = 30
 dedupWindowMs = 3600000
 acknowledgeEphemeral = false                # required to enable async tools with memory backend
+
+# Issue #139 durable orphan-recovery lease (defaults shown). Each instance
+# advances a per-job lease on every heartbeat; the sweep orphans a job only
+# after its own lease expires, so a fresh instance never orphans another live
+# instance's jobs on a shared store. Validated: leaseTtl >= 2*heartbeat and
+# httpJobGrace >= leaseTtl.
+instanceHeartbeatMs = 15000                 # heartbeat cadence
+instanceLeaseTtlMs = 90000                  # per-job lease TTL (6x heartbeat)
+httpJobGraceMs = 300000                     # extra grace for no-pid http jobs (5 min)
+orphanSweepIntervalMs = 30000               # reaper cadence
+instanceGcMs = 3600000                      # gateway_instances GC horizon
+# ownsOrphanRecovery = false                # DEPRECATED (#139): superseded by the lease; parsed + warned, no longer used
 ```
 
 Backends:

package/dist/async-job-manager.d.ts

@@ -5,6 +5,14 @@ import { type FlightRecorderLike } from "./flight-recorder.js";
 import type { ValidationRunStore } from "./job-store.js";
 import { type ApiProvider, type ApiRequest, type ApiUsage } from "./api-provider.js";
 import { type JobLimitsConfig } from "./config.js";
+export interface LeaseRuntimeConfig {
+    instanceHeartbeatMs: number;
+    instanceLeaseTtlMs: number;
+    httpJobGraceMs: number;
+    orphanSweepIntervalMs: number;
+    instanceGcMs: number;
+    role?: string;
+}
 export declare function extractApiHttpStatus(error: unknown): number | null;
 export declare function extractApiErrorBody(error: unknown): string | undefined;
 export type LlmCli = "claude" | "codex" | "gemini" | "grok" | "mistral" | "devin" | "cursor";
@@ -102,7 +110,34 @@ export declare class AsyncJobManager {
     private readonly limiter;
     private readonly completedJobMemoryTtlMs;
     private readonly maxJobOutputBytes;
-    constructor(logger?: Logger, onJobComplete?: ((cli: JobProvider, durationMs: number, success: boolean) => void) | undefined, store?: JobStore | null, flightRecorder?: FlightRecorderLike, limits?: JobLimitsConfig);
+    private readonly instanceId;
+    private readonly hostname;
+    private readonly instancePid;
+    private readonly lease;
+    private heartbeatTimer;
+    private sweepTimer;
+    private durableAdmission;
+    private consecutiveHeartbeatFailures;
+    private skipSweepThisCycle;
+    private nextHeartbeatExpectedAt;
+    private disposed;
+    private readonly pendingWrites;
+    constructor(logger?: Logger, onJobComplete?: ((cli: JobProvider, durationMs: number, success: boolean) => void) | undefined, store?: JobStore | null, flightRecorder?: FlightRecorderLike, limits?: JobLimitsConfig, _deprecatedOwnsOrphanRecovery?: boolean, leaseConfig?: LeaseRuntimeConfig);
+    canAdmitDurableJobs(): boolean;
+    private assertDurableAdmission;
+    private trackPendingWrite;
+    dispose(opts?: {
+        timeoutMs?: number;
+    }): Promise<void>;
+    getInstanceId(): string;
+    private startHeartbeat;
+    private onHeartbeatTick;
+    runOrphanSweepNow(): void;
+    private startReaper;
+    private runOrphanSweep;
+    private confirmLiveProcessCandidates;
+    private recordStartOrFailClosed;
+    private markRunningDurable;
     private buildOrphanFlightResult;
     checkStalledJobs(now?: number): void;
     hasStore(): boolean;