@exulu/backend 2.3.0 → 3.1.0

package/ee/agentic-retrieval/pipeline/routing.ts

@@ -1,6 +1,5 @@
-import { generateText, Output } from "ai";
 import { z } from "zod";
-import { withRetry } from "@SRC/utils/with-retry";
+import { microCall } from "./micro-call";
 import { fuzzyPrefilter } from "./prefilter";
 import { normalizeFileName } from "./text-utils";
 import type { RoutingRule } from "./config";
@@ -103,25 +102,17 @@ export async function runRoutingPhase(opts: {
   const [docPageRaw, explicitKBRaw] = await Promise.all([
     (async () => {
       try {
-        return await withRetry(
-          () =>
-            generateText({
-              model,
-              temperature: 0,
-              system: buildDocPagePrompt(knownIdentifiers),
-              messages: [{ role: "user", content: question }],
-              output: Output.object({
-                schema: z.object({
-                  hasFilenameHint: z.boolean(),
-                  filenameHints: z.array(z.string()).optional(),
-                  hasPageHint: z.boolean(),
-                  pageNumber: z.number().int().nullable().optional(),
-                }),
-              }),
-              maxOutputTokens: 300,
-            }),
-          3,
-        );
+        return await microCall({
+          model,
+          system: buildDocPagePrompt(knownIdentifiers),
+          messages: [{ role: "user", content: question }],
+          schema: z.object({
+            hasFilenameHint: z.boolean(),
+            filenameHints: z.array(z.string()).optional(),
+            hasPageHint: z.boolean(),
+            pageNumber: z.number().int().nullable().optional(),
+          }),
+        });
       } catch (err) {
         steps.push({ text: "Doc/page detection failed — skipping filename and page hints." });
         return {
@@ -136,24 +127,16 @@ export async function runRoutingPhase(opts: {
     })(),
     (async () => {
       try {
-        return await withRetry(
-          () =>
-            generateText({
-              model,
-              temperature: 0,
-              system: kbSystemPrompt,
-              output: Output.object({
-                schema: z.object({
-                  explicitlyRequestedKnowledgeBases: z.array(
-                    z.enum(enabledContexts.map((c) => c.id) as [string, ...string[]]),
-                  ),
-                }),
-              }),
-              messages: [{ role: "user", content: question }],
-              maxOutputTokens: 200,
-            }),
-          3,
-        );
+        return await microCall({
+          model,
+          system: kbSystemPrompt,
+          schema: z.object({
+            explicitlyRequestedKnowledgeBases: z.array(
+              z.enum(enabledContexts.map((c) => c.id) as [string, ...string[]]),
+            ),
+          }),
+          messages: [{ role: "user", content: question }],
+        });
       } catch (err) {
         return { output: { explicitlyRequestedKnowledgeBases: [] as string[] } };
       }
@@ -267,23 +250,15 @@ export async function runRoutingPhase(opts: {
     }
 
     try {
-      const { output: classified } = await withRetry(
-        () =>
-          generateText({
-            model,
-            temperature: 0,
-            system: classifyPrompt,
-            messages: [{ role: "user", content: question }],
-            output: Output.object({
-              schema: z.object({
-                ruleId: z.enum(ruleIds as [string, ...string[]]),
-                reason: z.string(),
-              }),
-            }),
-            maxOutputTokens: 200,
-          }),
-        3,
-      );
+      const { output: classified } = await microCall({
+        model,
+        system: classifyPrompt,
+        messages: [{ role: "user", content: question }],
+        schema: z.object({
+          ruleId: z.enum(ruleIds as [string, ...string[]]),
+          reason: z.string(),
+        }),
+      });
 
       const matchedRule = routingRules.find((r) => r.id === classified.ruleId);
       if (matchedRule) {

package/ee/queues/decorator.ts

@@ -60,6 +60,17 @@ export type BullMqJobData = {
   evaluation?: string;
   item?: string;
   context?: string;
+  // Email-triggered routines (spec 2026-07-15): session-backed workflow runs.
+  /** agent_sessions id to run in; when absent the worker creates one. */
+  session?: string;
+  /** Existing job_results row to UPDATE instead of INSERT (continuation/retry/email intake). */
+  jobResultId?: string;
+  /** Skip steps before this index (resume after approval pause / retry-from-step). */
+  resumeFromIndex?: number;
+  /** Persisted to job_results.trigger — run provenance for the runs views. */
+  triggerSource?: "email" | "schedule" | "manual" | "api";
+  /** Persisted to job_results.trigger_metadata (email: from/subject/message_id; schedule: cron). */
+  triggerMetadata?: Record<string, unknown>;
 };
 
 export const bullmqDecorator = async ({

package/ee/queues/prune-job-results.test.ts

@@ -0,0 +1,41 @@
+import { TERMINAL_JOB_STATES } from "@EXULU_TYPES/enums/jobs";
+import { maybePruneJobResults } from "./prune-job-results";
+
+describe("TERMINAL_JOB_STATES", () => {
+  it("is the single source of truth for prunable terminal states", () => {
+    expect(TERMINAL_JOB_STATES).toEqual(["completed", "failed", "filtered", "cancelled"]);
+  });
+
+  it("never contains live or paused states", () => {
+    for (const state of ["waiting", "active", "delayed", "paused", "waiting_approval", "stuck"]) {
+      expect(TERMINAL_JOB_STATES).not.toContain(state);
+    }
+  });
+});
+
+describe("maybePruneJobResults", () => {
+  it("prunes only TERMINAL_JOB_STATES rows (every 100th call)", async () => {
+    const whereInCalls: any[][] = [];
+    const builder: any = {
+      whereIn: (...args: any[]) => {
+        whereInCalls.push(args);
+        return builder;
+      },
+      orderBy: () => builder,
+      offset: () => builder,
+      limit: () => builder,
+      first: async () => undefined, // under cap: nothing to delete
+      where: () => builder,
+      del: async () => 0,
+    };
+    const db: any = jest.fn(() => builder);
+
+    // The module-level counter only reaches the prune body every 100th call.
+    for (let i = 0; i < 100; i++) {
+      await maybePruneJobResults(db);
+    }
+
+    expect(whereInCalls.length).toBeGreaterThanOrEqual(1);
+    expect(whereInCalls[0]).toEqual(["state", TERMINAL_JOB_STATES]);
+  });
+});

package/ee/queues/prune-job-results.ts

@@ -3,7 +3,7 @@
  *
  * We now write a job_results row at enqueue time, so the table grows faster.
  * To bound it, every PRUNE_EVERY-th call we delete the oldest terminal rows
- * (state failed/completed) beyond the newest MAX_TERMINAL — keeping a rolling
+ * (states in TERMINAL_JOB_STATES) beyond the newest MAX_TERMINAL — keeping a rolling
  * window of recent finished jobs. Waiting/active/delayed rows are never
  * pruned (they're still live).
  *
@@ -13,9 +13,10 @@
  * overlapping runs.
  */
 
+import { TERMINAL_JOB_STATES } from "@EXULU_TYPES/enums/jobs";
+
 const MAX_TERMINAL = 10_000;
 const PRUNE_EVERY = 100;
-const TERMINAL_STATES = ["failed", "completed"];
 
 let sinceLastPrune = 0;
 let pruning = false;
@@ -30,7 +31,7 @@ export async function maybePruneJobResults(db: any): Promise<void> {
     // and everything older. Dialect-agnostic (knex offset/limit) so it works on
     // both Postgres and MySQL.
     const boundary = await db("job_results")
-      .whereIn("state", TERMINAL_STATES)
+      .whereIn("state", TERMINAL_JOB_STATES)
       .orderBy("createdAt", "desc")
       .offset(MAX_TERMINAL)
       .limit(1)
@@ -38,7 +39,7 @@ export async function maybePruneJobResults(db: any): Promise<void> {
 
     if (boundary?.createdAt) {
       const deleted = await db("job_results")
-        .whereIn("state", TERMINAL_STATES)
+        .whereIn("state", TERMINAL_JOB_STATES)
         .where("createdAt", "<=", boundary.createdAt)
         .del();
       if (deleted) {

package/ee/schemas.ts

@@ -260,6 +260,26 @@ export const jobResultsSchema: ExuluTableDefinition = {
             name: "type",
             type: "text",
         },
+        // Email-triggered routines (spec 2026-07-15 §3.3): run provenance +
+        // session cross-link. `workflow` replaces label-substring filtering
+        // (indexed via the composite index created in init-exulu-db.ts).
+        // Pre-migration rows keep trigger = NULL (displayed as "—").
+        {
+            name: "trigger",
+            type: "text",
+        },
+        {
+            name: "trigger_metadata",
+            type: "json",
+        },
+        {
+            name: "session",
+            type: "text",
+        },
+        {
+            name: "workflow",
+            type: "text",
+        },
     ],
 };
 
@@ -392,5 +412,80 @@ export const workflowTemplatesSchema: ExuluTableDefinition = {
         type: "json",
         required: true,
       },
+      // Escape hatch for the approval behavior change (spec §5.2): when true
+      // the run keeps the legacy blanket tool pre-approval and never pauses.
+      {
+        name: "auto_approve_tools",
+        type: "boolean",
+        default: false,
+      },
     ],
-  };
+  };
+
+  // Email-triggered routines (spec §3.1): one inbound trigger per routine.
+  // RBAC is false — access is checked via the parent workflow_templates row
+  // (routine read for listing, routine write + workflows:write role for CRUD),
+  // resolved explicitly in the custom GraphQL resolvers. graphql: false keeps
+  // the auto-CRUD generator away from this table; the API surface is the
+  // custom workflowTriggers / upsertWorkflowEmailTrigger / deleteWorkflowTrigger
+  // resolvers only.
+  export const workflowTriggersSchema: ExuluTableDefinition = {
+      type: "workflow_triggers",
+      name: {
+        plural: "workflow_triggers",
+        singular: "workflow_trigger",
+      },
+      RBAC: false,
+      graphql: false,
+      fields: [
+        {
+          name: "workflow",
+          type: "uuid",
+          required: true,
+        },
+        {
+          // 'email' for now; extensible ('webhook' later).
+          name: "type",
+          type: "text",
+          required: true,
+        },
+        {
+          name: "enabled",
+          type: "boolean",
+          default: false,
+        },
+        {
+          // Generated server-side: {routine-slug}-{8 hex}@{inbound_domain}.
+          // Real UNIQUE column (not JSON) because the webhook resolves
+          // triggers by recipient address.
+          name: "address",
+          type: "text",
+          required: true,
+          unique: true,
+          index: true,
+        },
+        {
+          // allowed_senders / filters / filtered_run_retention /
+          // rate_limit_per_hour / sender_rate_limit_per_hour (spec §3.1).
+          name: "config",
+          type: "json",
+          required: true,
+        },
+        {
+          // Captured from the admin who saves the trigger; email runs execute
+          // under this identity (same principle as cron).
+          name: "run_as_user",
+          type: "number",
+        },
+        {
+          name: "run_as_role",
+          type: "uuid",
+        },
+        {
+          // RBAC:false means addCoreFields does not add created_by; add it
+          // explicitly (audit trail, spec §3.1 core fields).
+          name: "created_by",
+          type: "number",
+        },
+      ],
+    };

package/ee/workers.flow.test.ts

@@ -0,0 +1,236 @@
+import type { UIMessage } from "ai";
+
+// ee/workers.ts pulls in the whole worker runtime; mock everything with
+// side effects / heavy transitive imports. Specifiers match workers.ts's
+// own import strings (moduleNameMapper resolves both aliased forms).
+jest.mock("@SRC/postgres/client", () => ({
+  postgresClient: jest.fn(async () => ({ db: jest.fn() })),
+}));
+jest.mock("@SRC/utils/enabled-tools.ts", () => ({
+  getEnabledTools: jest.fn(async () => []),
+}));
+jest.mock("@SRC/exulu/resolve-model.ts", () => ({
+  resolveModel: jest.fn(async () => ({ apiKey: undefined, languageModel: {} })),
+}));
+jest.mock("@SRC/exulu/statistics", () => ({
+  updateStatistic: jest.fn(async () => undefined),
+}));
+jest.mock("@SRC/exulu/storage.ts", () => ({ ExuluStorage: class {} }));
+jest.mock("@SRC/exulu/context.ts", () => ({ getTableName: jest.fn() }));
+jest.mock("@SRC/exulu/app/singleton", () => ({ exuluApp: { get: jest.fn() } }));
+jest.mock("@SRC/exulu/provider.ts", () => ({
+  saveChat: jest.fn(async () => undefined),
+  getAgentMessages: jest.fn(async () => []),
+}));
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const providerModule = require("@SRC/exulu/provider.ts") as {
+  saveChat: jest.Mock;
+  getAgentMessages: jest.Mock;
+};
+
+import { FlowStepError, processUiMessagesFlow } from "./workers";
+
+const step = (id: string, text: string): UIMessage =>
+  ({ id, role: "user", parts: [{ type: "text", text }] }) as UIMessage;
+
+const assistant = (id: string, parts: any[]): UIMessage =>
+  ({ id, role: "assistant", parts }) as UIMessage;
+
+const approvalPart = {
+  type: "tool-create_offer",
+  state: "approval-requested",
+  approval: { id: "appr-1" },
+};
+
+/**
+ * Stub ExuluProvider: generateStream returns a fake AI-SDK stream whose
+ * toUIMessageStream immediately finishes with [history + step + response].
+ * `responses[n]` = assistant messages appended by the n-th generateStream call.
+ * A response of `null` makes that call's stream error (onError + reject).
+ */
+const makeStubProvider = (responses: (UIMessage[] | null)[]) => {
+  let call = 0;
+  const generateStream = jest.fn(async (opts: any) => {
+    const index = call++;
+    const original: UIMessage[] = [...(opts.previousMessages ?? []), opts.message];
+    return {
+      originalMessages: original,
+      previousMessages: opts.previousMessages ?? [],
+      stream: {
+        toUIMessageStream: (streamOpts: any) => ({
+          async *[Symbol.asyncIterator]() {
+            const response = responses[index];
+            if (response === null) {
+              streamOpts.onError(new Error("provider exploded"));
+              return;
+            }
+            await streamOpts.onFinish({ messages: [...original, ...(response ?? [])] });
+          },
+        }),
+      },
+    };
+  });
+  return { provider: { generateStream } as any, generateStream };
+};
+
+const baseArgs = (provider: any) => ({
+  providers: [] as any[],
+  agent: { id: "agent-1", name: "Agent", model: "model-1", tools: [], instructions: "do" } as any,
+  provider,
+  contexts: [] as any[],
+  user: { id: 7, role: { id: "role-1" } } as any,
+  tools: [{ name: "Create Offer" }] as any[],
+  config: {} as any,
+});
+
+afterEach(() => jest.clearAllMocks());
+
+describe("processUiMessagesFlow (headless — unchanged legacy behavior)", () => {
+  it("passes session undefined + blanket approvedTools and never persists", async () => {
+    const { provider, generateStream } = makeStubProvider([[assistant("a1", [{ type: "text", text: "ok" }])]]);
+    const result = await processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "hello")],
+    });
+    expect(generateStream).toHaveBeenCalledTimes(1);
+    const opts = generateStream.mock.calls[0][0];
+    expect(opts.session).toBeUndefined();
+    expect(Array.isArray(opts.approvedTools)).toBe(true);
+    expect(providerModule.saveChat).not.toHaveBeenCalled();
+    expect(result.pausedAtStepIndex).toBeUndefined();
+    expect(result.messages.map((m) => m.id)).toContain("a1");
+  });
+});
+
+describe("processUiMessagesFlow (session-backed)", () => {
+  it("passes the session, rewrites step ids, and persists at each step boundary", async () => {
+    const { provider, generateStream } = makeStubProvider([
+      [assistant("a1", [{ type: "text", text: "one" }])],
+      [assistant("a2", [{ type: "text", text: "two" }])],
+    ]);
+    await processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "first"), step("s2", "second")],
+      sessionId: "sess-1",
+    });
+    expect(generateStream).toHaveBeenCalledTimes(2);
+    for (const call of generateStream.mock.calls) {
+      expect(call[0].session).toBe("sess-1");
+      // steps_json ids repeat across runs — persisted ids must be fresh:
+      expect(call[0].message.id).toMatch(/^wfmsg-/);
+    }
+    expect(providerModule.saveChat).toHaveBeenCalledTimes(2);
+    expect(providerModule.saveChat.mock.calls[0][0]).toMatchObject({ session: "sess-1", user: 7 });
+  });
+
+  it("drops the blanket approvedTools when respectToolApprovals is set", async () => {
+    const { provider, generateStream } = makeStubProvider([[assistant("a1", [{ type: "text", text: "ok" }])]]);
+    await processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "x")],
+      sessionId: "sess-1",
+      respectToolApprovals: true,
+    });
+    expect(generateStream.mock.calls[0][0].approvedTools).toBeUndefined();
+  });
+
+  it("pauses at the step whose final message requests approval and skips later steps", async () => {
+    const { provider, generateStream } = makeStubProvider([
+      [assistant("a1", [approvalPart])],
+      [assistant("a2", [{ type: "text", text: "never reached" }])],
+    ]);
+    const result = await processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "gated"), step("s2", "after")],
+      sessionId: "sess-1",
+      respectToolApprovals: true,
+    });
+    expect(result.pausedAtStepIndex).toBe(0);
+    expect(generateStream).toHaveBeenCalledTimes(1);
+    // The paused transcript was persisted before returning:
+    expect(providerModule.saveChat).toHaveBeenCalledTimes(1);
+  });
+
+  it("resumeFromIndex skips completed steps and reloads history from agent_messages", async () => {
+    providerModule.getAgentMessages.mockResolvedValueOnce([
+      { content: JSON.stringify(step("old-1", "first")) },
+      { content: JSON.stringify(assistant("old-a1", [{ type: "text", text: "done" }])) },
+    ]);
+    const { provider, generateStream } = makeStubProvider([
+      [assistant("a2", [{ type: "text", text: "resumed" }])],
+    ]);
+    const result = await processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "first"), step("s2", "second")],
+      sessionId: "sess-1",
+      resumeFromIndex: 1,
+    });
+    expect(providerModule.getAgentMessages).toHaveBeenCalledWith({
+      session: "sess-1",
+      includeAllUsers: true,
+    });
+    expect(generateStream).toHaveBeenCalledTimes(1); // only step index 1
+    expect(generateStream.mock.calls[0][0].previousMessages.map((m: UIMessage) => m.id)).toEqual([
+      "old-1",
+      "old-a1",
+    ]);
+    expect(result.messages.map((m) => m.id)).toContain("a2");
+  });
+
+  it("wraps step failures in FlowStepError carrying the failing step index", async () => {
+    const { provider } = makeStubProvider([
+      [assistant("a1", [{ type: "text", text: "ok" }])],
+      null, // step 1 explodes
+    ]);
+    const promise = processUiMessagesFlow({
+      ...baseArgs(provider),
+      inputMessages: [step("s1", "one"), step("s2", "two")],
+      sessionId: "sess-1",
+    });
+    await expect(promise).rejects.toThrow("provider exploded");
+    await promise.catch((error: unknown) => {
+      expect(error).toBeInstanceOf(FlowStepError);
+      expect((error as FlowStepError).stepIndex).toBe(1);
+    });
+  });
+
+  it("a rerun after a step-1 failure persists only steps >= 1 — no duplicate messages (spec §5.4/§9)", async () => {
+    // First run: step 0 succeeds (one boundary persist), step 1 explodes.
+    const first = makeStubProvider([
+      [assistant("a1", [{ type: "text", text: "one" }])],
+      null, // step 1 explodes
+    ]);
+    await expect(
+      processUiMessagesFlow({
+        ...baseArgs(first.provider),
+        inputMessages: [step("s1", "one"), step("s2", "two")],
+        sessionId: "sess-1",
+      }),
+    ).rejects.toThrow("provider exploded");
+    expect(providerModule.saveChat).toHaveBeenCalledTimes(1); // step 0 only
+
+    // Rerun from the failed step (what the worker's retry loop does with
+    // FlowStepError.stepIndex): prior history reloads from agent_messages;
+    // step 0 must NOT run or persist again.
+    providerModule.saveChat.mockClear();
+    providerModule.getAgentMessages.mockResolvedValueOnce([
+      { content: JSON.stringify(step("old-s1", "one")) },
+      { content: JSON.stringify(assistant("a1", [{ type: "text", text: "one" }])) },
+    ]);
+    const second = makeStubProvider([[assistant("a2", [{ type: "text", text: "two" }])]]);
+    await processUiMessagesFlow({
+      ...baseArgs(second.provider),
+      inputMessages: [step("s1", "one"), step("s2", "two")],
+      sessionId: "sess-1",
+      resumeFromIndex: 1,
+    });
+    expect(second.generateStream).toHaveBeenCalledTimes(1); // only step index 1
+    expect(providerModule.saveChat).toHaveBeenCalledTimes(1); // only the step-1 boundary
+    const persisted = providerModule.saveChat.mock.calls[0][0].messages as UIMessage[];
+    expect(persisted.map((m) => m.id)).toContain("a2");
+    // Step 0's message reaches saveChat only via the reloaded history (same
+    // ids — saveChat's message_id merge keeps it a no-op), never as a re-run.
+    expect(persisted.filter((m) => m.id === "a1")).toHaveLength(1);
+  });
+});