@galaxyproject/orbit 0.0.0 → 0.9.0

package/extensions/orbit/lessons/pi-event-contract.ts

@@ -0,0 +1,105 @@
+/**
+ * The one place pi's tool-result shape is named.
+ *
+ * The lesson matcher needs the CALL ARGUMENTS, not just the result text: a
+ * host in a URL, an extension in a path, a Galaxy tool id in `tool_id` are all
+ * triggers. Only `tool_result` carries them. `message_end` -- which the
+ * existing model-facing hints in `skill-triggers.ts` use -- carries just
+ * `{ message }`, so a hint hosted there could never match on arguments. The
+ * assertions below make that a compile error rather than a silently dead
+ * trigger table if upstream ever moves a field.
+ *
+ * Handlers chain inside one extension in registration order, and pi feeds each
+ * one the previous handler's `content`. So every handler registered after
+ * `registerLessonHint` sees the hint, which is why the hint is a block of its
+ * own, and why this module remembers which blocks it appended: a consumer that
+ * drops "the hint" drops only those, never a tool's own block that happens to
+ * open with the marker.
+ */
+
+import type { MessageEndEvent, ToolResultEvent } from "@earendil-works/pi-coding-agent";
+import { isLessonHintBlock } from "../../../shared/lesson-hint-marker.js";
+
+// The exact text of every hint this process appended. A tool can open a block
+// of its own with the marker, but it cannot put that block in here, so a
+// hostile output that dresses its error up as a lesson is still read as the
+// tool's error by the observation collector and the activity log. One entry
+// per lesson that fired; cleared with the armed set on session_start.
+const appendedHints = new Set<string>();
+
+/** The content array pi hands a `tool_result` handler, and accepts back. */
+export type LessonToolResultContent = ToolResultEvent["content"];
+
+/** Everything lesson matching reads off a `tool_result` event. */
+export interface LessonToolResultFields {
+  toolName: string;
+  toolCallId: string;
+  input: Record<string, unknown>;
+  content: LessonToolResultContent;
+  /**
+   * Carried for completeness, and deliberately NOT a matcher input:
+   * `invocation-failure-hint.ts` finds failures inside successful results, so
+   * gating a lesson on `isError` would miss the cases that matter.
+   */
+  isError: boolean;
+}
+
+// Compile-time proof that pi's event really carries all of them.
+const toolResultFields: (e: ToolResultEvent) => LessonToolResultFields = (e) => ({
+  toolName: e.toolName,
+  toolCallId: e.toolCallId,
+  input: e.input,
+  content: e.content,
+  isError: e.isError,
+});
+void toolResultFields;
+
+// ...and that `message_end` carries nothing but the message, which is why the
+// hint cannot live there.
+type MessageEndExtraKeys = Exclude<keyof MessageEndEvent, "type" | "message">;
+const messageEndHasOnlyMessage: [MessageEndExtraKeys] extends [never] ? true : false = true;
+void messageEndHasOnlyMessage;
+
+/** Every text block of a result, joined -- the signature-matching haystack. */
+export function resultTextOf(content: LessonToolResultContent): string {
+  const parts: string[] = [];
+  for (const block of content) {
+    if (block.type === "text") parts.push(block.text);
+  }
+  return parts.join("\n");
+}
+
+/**
+ * Add the hint as a text block of its own at the end. Not folded into the
+ * tool's last block: a later handler reading the result as the tool's output
+ * has to be able to tell the two apart, and a failed call with no text of its
+ * own would otherwise be described by the hint. By the time this runs the
+ * oversized-output recovery has already replaced a huge result with its
+ * preview (`index.ts` registers that first), so the end of the content is
+ * still the end of what the model reads. Returns a copy.
+ */
+export function appendHintToContent(
+  content: LessonToolResultContent,
+  hint: string,
+): LessonToolResultContent {
+  appendedHints.add(hint);
+  return [...content, { type: "text", text: hint }];
+}
+
+/** A block Orbit itself appended -- not merely one that opens with the marker. */
+export function isAppendedLessonHint(
+  block: { type: string; text?: string } | null | undefined,
+): boolean {
+  return isLessonHintBlock(block) && appendedHints.has(block!.text as string);
+}
+
+/** The content as the tool produced it: every hint Orbit appended removed. */
+export function withoutAppendedLessonHints<T extends { type: string; text?: string }>(
+  content: readonly T[],
+): T[] {
+  return content.filter((block) => !isAppendedLessonHint(block));
+}
+
+export function resetAppendedLessonHints(): void {
+  appendedHints.clear();
+}

package/extensions/orbit/lessons/propose-replay.ts

@@ -0,0 +1,103 @@
+/**
+ * Replaying a planted lesson proposal, for the model-free eval.
+ *
+ * What is worth pinning deterministically is that a hostile draft is REJECTED
+ * and recorded as such. Reaching that through a model is unreliable by
+ * construction: a model that understood the attack declines to propose at all,
+ * which is the better behaviour and leaves no row to assert on. So the eval
+ * plants the proposal and drives the real core.
+ *
+ * The same shape as submission-replay.ts, with its two constraints:
+ *
+ * - the file must resolve inside the session directory (its resolveReplayPath,
+ *   symlinks included), so the env var alone cannot point the replay anywhere
+ *   else on the machine;
+ * - every replay writes a `lesson.replay` row FIRST, so its outcome is never
+ *   indistinguishable from a proposal a user made.
+ *
+ * It does NOT forge an approval. It arms a proposal and calls proposeLesson
+ * with the session's real ctx, so the approval step still runs and still
+ * refuses when there is nobody to ask. A seam that could write a lesson
+ * without a human would be a hole, not a fixture.
+ *
+ * It also keeps the session_start ctx past the handler's return, which pi
+ * asks extensions not to do: the approval dialog is left pending (see below)
+ * and its answer comes back through that ctx. If the session is replaced
+ * while the dialog is open (/new, a fork, a resume), pi's RPC mode still
+ * delivers the answer to the old dialog, so a Save can still write the lesson
+ * -- and its lesson.saved row lands in the new session's activity log with no
+ * lesson.replay row ahead of it. Only a ctx.ui call made after the replacement
+ * throws (the stale-ctx error), and the .catch swallows that. Tolerable only
+ * because this runs behind ORBIT_LESSON_PROPOSAL_REPLAY (or its LOOM_ alias)
+ * in evals, where nothing replaces the session; nothing user-facing should
+ * copy it.
+ */
+
+import * as fs from "fs";
+import { isLessonsEnabled } from "./enabled";
+import * as path from "path";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { readEnv } from "../../../shared/orbit-env.js";
+import { appendActivityEvent } from "../activity.js";
+import { getNotebookPath } from "../state.js";
+import { resolveReplayPath } from "../submission-replay.js";
+import type { LessonProposalInput } from "./compose.js";
+import { armLessonProposal, disarmLessonProposal, proposeLesson } from "./propose.js";
+
+export function isLessonProposalReplayEnabled(): boolean {
+  return !!readEnv("LESSON_PROPOSAL_REPLAY")?.trim();
+}
+
+export function registerLessonProposalReplay(pi: ExtensionAPI): void {
+  pi.on("session_start", async (_event, ctx: ExtensionContext) => {
+    const configured = readEnv("LESSON_PROPOSAL_REPLAY")?.trim();
+    if (!configured) return;
+    // Same answer the live surface gives: nothing read, written or armed while off.
+    if (!isLessonsEnabled()) return;
+
+    const notebookPath = getNotebookPath();
+    if (!notebookPath) return;
+    const sessionDir = path.dirname(notebookPath);
+
+    const file = resolveReplayPath(sessionDir, configured);
+    if (!file || !fs.existsSync(file)) return;
+
+    appendActivityEvent(sessionDir, {
+      timestamp: new Date().toISOString(),
+      kind: "lesson.replay",
+      source: "lesson-propose-replay",
+      // Relative to the realpath'd directory, the way resolveReplayPath
+      // compared it: against the spelled path a macOS /var -> /private/var hop
+      // turns this into a ../-climb through the machine's directory names.
+      payload: { file: path.relative(fs.realpathSync(sessionDir), file) },
+    });
+
+    let proposal: LessonProposalInput;
+    try {
+      proposal = JSON.parse(fs.readFileSync(file, "utf-8")) as LessonProposalInput;
+    } catch {
+      return;
+    }
+
+    // Live straight away: there is no agent run here to start it, and the
+    // replay stands in for a proposal made inside one.
+    const arming = armLessonProposal("explicit", { live: true });
+    // Not awaited. pi's RPC mode emits session_start before it starts reading
+    // stdin, so holding this handler open on the approval dialog would wait
+    // for an answer that can never arrive -- and hang the whole session.
+    // Everything up to the dialog (validation, the activity rows) still runs
+    // synchronously here.
+    void proposeLesson(proposal, {
+      hasUI: ctx.hasUI,
+      ui: {
+        notify: (message, level) => ctx.ui.notify(message, level),
+        select: (title, options) => ctx.ui.select(title, options),
+        confirm: (title, message) => ctx.ui.confirm(title, message),
+      },
+    })
+      .catch(() => undefined)
+      // A rejected draft keeps its retry; the replay's arming must not outlive
+      // it -- but a /lesson typed while the dialog was open is not ours to clear.
+      .finally(() => disarmLessonProposal(arming));
+  });
+}

package/extensions/orbit/lessons/propose-tool.ts

@@ -0,0 +1,222 @@
+/**
+ * `lesson_propose` -- the agent's only way to put a lesson on disk.
+ *
+ * A tool rather than a parse of the agent's next message:
+ *
+ * - pi validates arguments against this schema before execute runs, so a
+ *   wrong-shaped proposal never reaches our code and the model gets an error
+ *   it can act on. A fence parser would be a hand-written tokenizer attacked by
+ *   exactly the content it extracts -- the body rules forbid code fences, so a
+ *   hostile draft is the input that contains one.
+ * - structured fields are what let the brain build the document and own
+ *   type/status/generated/verified by construction. A whole file from the
+ *   model could only be allowlisted by re-parsing and re-emitting it, which is
+ *   the scrubbing pass the threat model rules out.
+ * - the validator's errors come back as a tool result, so a rejection is
+ *   correctable in the same turn.
+ * - execute gets a real ExtensionContext (ui + hasUI) in TUI, RPC and json mode.
+ *
+ * This schema is a convenience for the model, NOT the security boundary. Every
+ * cap here is re-enforced by validateLessonMarkdown on the composed bytes; a
+ * disagreement is a bug in this schema, never a reason to relax the validator.
+ *
+ * No promptSnippet: nothing about lessons goes in the cached system prompt.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "@sinclair/typebox";
+import {
+  EVIDENCE_CAUSE,
+  EVIDENCE_OUTCOME,
+  EVIDENCE_SYMPTOM,
+  LESSON_KINDS,
+  LESSON_STAGES,
+  LIMITS,
+  PROPOSABLE_NAMESPACES,
+} from "../../../shared/lesson-rules.js";
+import type { LessonProposalInput } from "./compose.js";
+import { proposeLesson, type ProposeUiContext } from "./propose.js";
+
+export const LESSON_PROPOSE_TOOL_NAME = "lesson_propose";
+
+const literals = (values: readonly string[]) => Type.Union(values.map((v) => Type.Literal(v)));
+
+const stringList = (maxLength: number, description: string) =>
+  Type.Optional(
+    Type.Array(Type.String({ maxLength }), { maxItems: LIMITS.listItems, description }),
+  );
+
+const PARAMETERS = Type.Object({
+  namespace: literals(PROPOSABLE_NAMESPACES),
+  slug: Type.String({
+    maxLength: LIMITS.slug,
+    description:
+      "Lowercase words joined by hyphens naming the SITUATION, not the fix, e.g. " +
+      "'na-coerced-to-zero-in-filters'.",
+  }),
+  title: Type.String({
+    maxLength: LIMITS.title,
+    description: "One line describing the situation, not the fix.",
+  }),
+  description: Type.String({
+    maxLength: LIMITS.description,
+    description: "One line, one sentence.",
+  }),
+  kind: literals(LESSON_KINDS),
+  stage: Type.Array(literals(LESSON_STAGES), {
+    minItems: 1,
+    maxItems: LESSON_STAGES.length,
+    description: "Which stage(s) of an analysis this bites in.",
+  }),
+  tags: stringList(LIMITS.tag, "Short keywords."),
+  stale_after: Type.String({
+    maxLength: 10,
+    description:
+      "YYYY-MM-DD after which this should be re-checked. Galaxy ships quarterly, so pick 6-12 " +
+      "months out, and sooner for anything from a forum answer.",
+  }),
+  cues: Type.String({
+    maxLength: LIMITS.cues,
+    description: "When this applies, in a sentence a person reads. One line.",
+  }),
+  applies_to: Type.Object({
+    versions: Type.String({
+      maxLength: LIMITS.appliesTo,
+      description: "Versions it holds for, or 'any'.",
+    }),
+    tested: Type.String({
+      maxLength: LIMITS.appliesTo,
+      description: "What it was actually seen against.",
+    }),
+  }),
+  evidence: Type.Object({
+    symptom: literals(EVIDENCE_SYMPTOM),
+    cause: literals(EVIDENCE_CAUSE),
+    outcome: literals(EVIDENCE_OUTCOME),
+    method: Type.String({
+      maxLength: LIMITS.method,
+      description: "How you know. 'hypothesized' and 'unvalidated' are fine answers.",
+    }),
+  }),
+  trigger: Type.Optional(
+    Type.Object({
+      signatures: stringList(
+        LIMITS.signature,
+        "The distinctive first line of the error, at least 8 characters, with URLs, paths, " +
+          "emails, long hex ids and 5+ digit numbers already replaced by <url>, <path>, <email>, " +
+          "<id>, <n>.",
+      ),
+      tools: stringList(LIMITS.tool, "Short Galaxy tool ids or families, e.g. 'deseq2'. No '/'."),
+      mcp_tools: stringList(LIMITS.mcpTool, "galaxy_* tool names."),
+      formats: stringList(LIMITS.format, "Lowercase Galaxy datatypes, e.g. 'tabular'."),
+      hosts: stringList(LIMITS.host, "Public hostnames seen in commands, e.g. 'ncbi.nlm.nih.gov'."),
+      extensions: stringList(LIMITS.extension, "File extensions seen in arguments, e.g. '.gtf'."),
+      step_keywords: stringList(
+        LIMITS.stepKeyword,
+        "Lowercase words matched against plan-step text.",
+      ),
+    }),
+  ),
+  sources: Type.Optional(
+    Type.Array(
+      Type.Object({
+        id: Type.String({ maxLength: LIMITS.sourceId }),
+        resource: Type.Optional(Type.String({ maxLength: LIMITS.sourceText })),
+        title: Type.Optional(Type.String({ maxLength: LIMITS.sourceText })),
+      }),
+      {
+        maxItems: LIMITS.listItems,
+        description:
+          "Required when this came from GTN or the forum, whose content is attribution-licensed. " +
+          "resource, if given, is an https link.",
+      },
+    ),
+  ),
+  sections: Type.Object(
+    {
+      symptom: Type.String({ maxLength: LIMITS.section }),
+      cause: Type.Optional(Type.String({ maxLength: LIMITS.section })),
+      check_first: Type.String({ maxLength: LIMITS.section }),
+      intervention: Type.String({ maxLength: LIMITS.section }),
+      validate: Type.String({ maxLength: LIMITS.section }),
+      not_when: Type.String({ maxLength: LIMITS.section }),
+    },
+    {
+      description:
+        "Prose only: no code fences, URLs, markdown links, headings or HTML. Up to 600 characters " +
+        "each. Omit `cause` if you do not actually know it.",
+    },
+  ),
+});
+
+const DESCRIPTION = `Propose ONE lesson drawn from what happened in this session, for the user to
+approve. The user always sees the draft and decides; nothing is written without that.
+
+Only call this when the user asked -- they typed /lesson, or Orbit told you they corrected you and
+they then said yes to writing it down. Never call it on your own initiative: an unasked call is
+refused and recorded.
+
+A lesson is for what you only catch by knowing the science or the data: a silently wrong result,
+a sanity range that makes a silent failure visible, a choice and when it flips, a data-source
+quirk, or what it actually takes to reproduce a published analysis. If a validator, a schema or a
+better error message could catch it, it belongs upstream instead -- say so and do not call this.
+
+Nothing identifying goes in a lesson: no paths, URLs, hostnames of private servers, ids, emails or
+copied data values. If the draft fails the schema you get the violations back and may fix them
+and call once more.`;
+
+function uiOf(ctx: ExtensionContext): ProposeUiContext {
+  return {
+    hasUI: ctx.hasUI,
+    ui: {
+      notify: (message, level) => ctx.ui.notify(message, level),
+      select: (title, options) => ctx.ui.select(title, options),
+      confirm: (title, message) => ctx.ui.confirm(title, message),
+    },
+  };
+}
+
+export function registerLessonProposeTool(pi: ExtensionAPI): void {
+  pi.registerTool({
+    name: LESSON_PROPOSE_TOOL_NAME,
+    label: "Propose a Lesson",
+    description: DESCRIPTION,
+    parameters: PARAMETERS,
+    async execute(_toolCallId, params, _signal, _onUpdate, ctx: ExtensionContext) {
+      // Never throw out of execute: the model always gets an answer it can act
+      // on, and a thrown error would carry a stack instead.
+      try {
+        const outcome = await proposeLesson(params as unknown as LessonProposalInput, uiOf(ctx));
+        if (outcome.ok) {
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text:
+                  `The user approved it. Lesson saved as ${outcome.id}. Tell them it is saved, ` +
+                  `as a local draft that stays on this machine.`,
+              },
+            ],
+            details: { id: outcome.id, saved: true },
+          };
+        }
+        return {
+          content: [{ type: "text" as const, text: outcome.message }],
+          details: { error: true, reason: outcome.reason },
+        };
+      } catch {
+        return {
+          content: [{ type: "text" as const, text: "lesson_propose failed; nothing was saved." }],
+          details: { error: true, reason: "exception" },
+        };
+      }
+    },
+    renderResult: (result) => {
+      const details = result.details as
+        { saved?: boolean; id?: string; reason?: string } | undefined;
+      if (details?.saved) return new Text(`Lesson saved: ${details.id}`);
+      return new Text(`Lesson not saved (${details?.reason ?? "rejected"})`);
+    },
+  });
+}