@agent-native/dispatch 0.10.3 → 0.11.0

package/src/actions/provider-api-catalog.ts

@@ -20,7 +20,7 @@ export default defineAction({
     return {
       providers,
       guidance:
-        "Workspace integrations and grants are not capability limits. When a provider can answer a question through its HTTP API, inspect docs/spec URLs here and call provider-api-request with the exact provider API method/path/query/body instead of adding a rigid one-off action. Custom providers registered via provider-api-register also appear here.",
+        "Workspace integrations and grants are not capability limits. When a provider can answer a question through its HTTP API, inspect docs/spec URLs here and call provider-api-request with the exact provider API method/path/query/body instead of adding a rigid one-off action. For broad searches, joins, classification, or absence claims, stage or save the full bounded corpus and reduce it with query-staged-dataset or run-code. Custom providers registered via provider-api-register also appear here.",
     };
   },
 });

package/src/actions/provider-api-docs.ts

@@ -2,6 +2,22 @@ import { defineAction } from "@agent-native/core";
 import { z } from "zod";
 import { fetchProviderApiDocs } from "../server/lib/provider-api.js";
 
+const BooleanFromQuerySchema = z.preprocess(
+  (value) => (typeof value === "string" ? value === "true" : value),
+  z.boolean(),
+);
+const WebContentSearchSchema = z.object({
+  query: z.union([z.string(), z.array(z.string())]).optional(),
+  queries: z.array(z.string()).optional(),
+  terms: z.array(z.string()).optional(),
+  regex: z.string().optional(),
+  regexFlags: z.string().optional(),
+  source: z.enum(["extracted", "raw"]).optional(),
+  maxMatches: z.coerce.number().int().min(1).max(500).optional(),
+  contextChars: z.coerce.number().int().min(0).max(1_000).optional(),
+  caseSensitive: BooleanFromQuerySchema.optional(),
+});
+
 export default defineAction({
   description:
     "Inspect provider API docs/spec metadata, or fetch ANY public API documentation page, OpenAPI spec, changelog, or web page. Registered docs/spec URLs from provider-api-catalog are curated starting points, but any public https/http URL is allowed. Use web-search to find documentation URLs first when uncertain, then fetch them here. SSRF guard still applies — private/internal addresses are blocked.",
@@ -26,6 +42,29 @@ export default defineAction({
       .max(4 * 1024 * 1024)
       .optional()
       .describe("Maximum response bytes to read. Default 1MB, max 4MB."),
+    maxChars: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .max(200_000)
+      .optional()
+      .describe("Maximum extracted content characters to return."),
+    responseMode: z
+      .enum(["auto", "raw", "text", "markdown", "links", "metadata", "matches"])
+      .optional()
+      .describe(
+        "How to return fetched docs. Default auto extracts HTML to markdown; use matches with search for compact snippets.",
+      ),
+    extract: z
+      .enum(["readability", "all-visible", "none"])
+      .optional()
+      .describe("HTML extraction strategy. Default readability."),
+    includeLinks: BooleanFromQuerySchema.optional().describe(
+      "Include compact links from extracted HTML. Default true.",
+    ),
+    search: WebContentSearchSchema.optional().describe(
+      "Optional post-fetch search over extracted content by default. Supports query, queries, terms, regex, source, maxMatches, and contextChars.",
+    ),
   }),
   http: { method: "GET" },
   readOnly: true,

package/src/actions/provider-api-request.ts

@@ -1,9 +1,69 @@
 import { defineAction } from "@agent-native/core";
+import { stagingExecuteRequest } from "@agent-native/core/provider-api/staging";
+import { getCredentialContext } from "@agent-native/core/server/request-context";
 import { z } from "zod";
-import { executeProviderApiRequest } from "../server/lib/provider-api.js";
+import {
+  DISPATCH_APP_ID,
+  executeProviderApiRequest,
+} from "../server/lib/provider-api.js";
 
 const MethodSchema = z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"]);
 
+const PaginationSchema = z
+  .object({
+    nextCursorPath: z
+      .string()
+      .optional()
+      .describe(
+        "Dot-path in the response JSON where the next cursor/token lives, e.g. 'next_cursor', 'meta.next', or 'nextPageToken'.",
+      ),
+    cursorParam: z
+      .string()
+      .optional()
+      .describe(
+        "Query parameter name to inject the cursor into the next request. Use cursorBodyPath for APIs that page through POST bodies.",
+      ),
+    cursorBodyPath: z
+      .string()
+      .optional()
+      .describe(
+        "Dot-path in the JSON request body to set to the next cursor. Use this for POST-body pagination.",
+      ),
+    pageParam: z
+      .string()
+      .optional()
+      .describe(
+        "Use page-number mode: this query param is incremented on each page.",
+      ),
+    startPage: z.coerce
+      .number()
+      .int()
+      .optional()
+      .describe("Starting page number for pageParam mode (default 1)."),
+    offsetParam: z
+      .string()
+      .optional()
+      .describe(
+        "Use offset mode: this query param is incremented by pageSize on each request.",
+      ),
+    pageSize: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .optional()
+      .describe(
+        "Expected page size for offset increments. Defaults to the actual item count of the first page.",
+      ),
+    maxPages: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .max(200)
+      .optional()
+      .describe("Maximum pages to fetch server-side (default 50, max 200)."),
+  })
+  .optional();
+
 export default defineAction({
   description:
     "Make an arbitrary authenticated HTTP request to a shared workspace integration, configured provider API, or custom provider registered via provider-api-register. Use this as the flexible escape hatch when Dispatch needs a provider endpoint, filter, pagination mode, payload, or API version that no canned action models. The request is constrained to the provider host, uses configured credentials automatically, blocks private/internal URLs, and redacts secrets from responses.",
@@ -77,8 +137,24 @@ export default defineAction({
       .string()
       .optional()
       .describe(
-        "Workspace file path to save the full response body to instead of returning it in context (e.g. 'analysis/hubspot-deals.json'). When set, returns only a compact summary {savedTo, status, bytes, preview} and allows up to 20MB response. Useful for large datasets that would overflow context.",
+        "Workspace file path to save the full response body to instead of returning it in context (e.g. 'analysis/provider-response.json'). When set, returns only a compact summary {savedTo, status, bytes, preview} and allows up to 20MB response. Useful for large datasets that would overflow context.",
+      ),
+    stageAs: z
+      .string()
+      .min(1)
+      .optional()
+      .describe(
+        "When set, parse the response as an array of records and write them into a staged dataset with this name. Returns a compact summary instead of the raw body. Re-staging the same name replaces the previous dataset.",
+      ),
+    itemsPath: z
+      .string()
+      .optional()
+      .describe(
+        "Dot-path to the items array in the response JSON, e.g. 'items', 'results', or 'data'. Omit for auto-detection.",
       ),
+    pagination: PaginationSchema.describe(
+      "Pagination config for server-side fetchAll when stageAs is set. Supports cursor (nextCursorPath + cursorParam or cursorBodyPath), page, and offset modes.",
+    ),
     fetchAllPages: z
       .object({
         cursorPath: z
@@ -88,8 +164,15 @@ export default defineAction({
           ),
         cursorParam: z
           .string()
+          .optional()
+          .describe(
+            "Query parameter name to pass the cursor on subsequent pages, e.g. 'cursor' or 'page_token'. Use cursorBodyPath instead for APIs that put cursors in POST bodies.",
+          ),
+        cursorBodyPath: z
+          .string()
+          .optional()
           .describe(
-            "Query parameter name to pass the cursor on subsequent pages, e.g. 'cursor' or 'page_token'.",
+            "Dot-path in the JSON request body to set to the next cursor. Use for POST-body pagination.",
           ),
         itemsPath: z
           .string()
@@ -109,9 +192,39 @@ export default defineAction({
       })
       .optional()
       .describe(
-        "Enable cursor-based pagination. After each response, reads cursorPath from the JSON body and re-issues the request with cursorParam set, accumulating items from itemsPath (or whole bodies) until cursor is empty or maxPages is reached. Combine with saveToFile to write the full dataset to a workspace file.",
+        "Enable cursor-based pagination. After each response, reads cursorPath from the JSON body and re-issues the request with cursorParam or cursorBodyPath set, accumulating items from itemsPath (or whole bodies) until cursor is empty or maxPages is reached. Combine with saveToFile to write the full dataset to a workspace file.",
       ),
   }),
   http: false,
-  run: async (args) => executeProviderApiRequest(args),
+  run: async (args) => {
+    if (args.stageAs) {
+      const ctx = getCredentialContext();
+      if (!ctx) {
+        throw new Error("No authenticated context for provider API staging.");
+      }
+      return stagingExecuteRequest(
+        {
+          provider: args.provider,
+          method: args.method,
+          path: args.path,
+          query: args.query,
+          headers: args.headers,
+          body: args.body,
+          auth: args.auth,
+          connectionId: args.connectionId,
+          accountId: args.accountId,
+          timeoutMs: args.timeoutMs,
+          maxBytes: args.maxBytes,
+          stageAs: args.stageAs,
+          itemsPath: args.itemsPath,
+          pagination: args.pagination,
+        },
+        (reqArgs) => executeProviderApiRequest(reqArgs),
+        { appId: DISPATCH_APP_ID, ownerEmail: ctx.userEmail },
+      );
+    }
+    return executeProviderApiRequest(
+      args as unknown as Parameters<typeof executeProviderApiRequest>[0],
+    );
+  },
 });

package/src/actions/query-staged-dataset.ts

@@ -0,0 +1,125 @@
+import { defineAction } from "@agent-native/core";
+import { runAggregateQuery } from "@agent-native/core/provider-api/staged-datasets-aggregate";
+import {
+  getStagedDatasetMeta,
+  getStagedDatasetRows,
+} from "@agent-native/core/provider-api/staged-datasets-store";
+import { getCredentialContext } from "@agent-native/core/server/request-context";
+import { z } from "zod";
+import { DISPATCH_APP_ID } from "../server/lib/provider-api.js";
+
+const WhereSchema = z.object({
+  column: z.string().min(1),
+  op: z.enum([
+    "equals",
+    "not_equals",
+    "contains",
+    "not_contains",
+    "gt",
+    "gte",
+    "lt",
+    "lte",
+    "exists",
+    "not_exists",
+  ]),
+  value: z.unknown().optional(),
+});
+
+const AggregateFieldSchema = z.object({
+  column: z.string().min(1).describe("Column to aggregate."),
+  op: z
+    .enum(["sum", "avg", "count", "min", "max", "count_distinct"])
+    .describe("Aggregation function."),
+  as: z
+    .string()
+    .optional()
+    .describe("Output column name. Default: {op}_{column}."),
+});
+
+export default defineAction({
+  description:
+    "Run a filter/aggregate/project query over a staged dataset previously written by provider-api-request (stageAs). Use after staging provider records, messages, documents, issues, events, or search results to count, group, filter, or project rows without re-fetching provider APIs.",
+  schema: z.object({
+    datasetId: z
+      .string()
+      .min(1)
+      .describe(
+        "Dataset id from provider-api-request stageAs result, or from list-staged-datasets.",
+      ),
+    where: z
+      .array(WhereSchema)
+      .optional()
+      .describe(
+        "Row-level filters (AND). Ops: equals, not_equals, contains, not_contains, gt, gte, lt, lte, exists, not_exists.",
+      ),
+    groupBy: z
+      .array(z.string().min(1))
+      .optional()
+      .describe(
+        "Column(s) to group by. Omit for a single aggregate over all rows.",
+      ),
+    aggregate: z
+      .array(AggregateFieldSchema)
+      .optional()
+      .describe(
+        "Aggregation operations. When set, non-group columns are aggregated. Omit to return raw rows.",
+      ),
+    select: z
+      .array(z.string().min(1))
+      .optional()
+      .describe("Column projection when aggregate is empty."),
+    orderBy: z.string().optional().describe("Sort output by this column."),
+    orderDir: z
+      .enum(["asc", "desc"])
+      .optional()
+      .describe("Sort direction (default asc)."),
+    limit: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .max(10_000)
+      .optional()
+      .describe("Maximum rows to return (default all, max 10000)."),
+  }),
+  http: false,
+  readOnly: true,
+  run: async (args) => {
+    const ctx = getCredentialContext();
+    if (!ctx) {
+      throw new Error("No authenticated context for query-staged-dataset.");
+    }
+
+    const meta = await getStagedDatasetMeta({
+      id: args.datasetId,
+      appId: DISPATCH_APP_ID,
+      ownerEmail: ctx.userEmail,
+    });
+    if (!meta) {
+      throw new Error(
+        `Dataset ${args.datasetId} not found (or belongs to a different owner/app).`,
+      );
+    }
+
+    const rows = await getStagedDatasetRows({
+      id: args.datasetId,
+      appId: DISPATCH_APP_ID,
+      ownerEmail: ctx.userEmail,
+    });
+
+    const result = runAggregateQuery(rows, {
+      where: args.where,
+      groupBy: args.groupBy,
+      aggregate: args.aggregate,
+      select: args.select,
+      orderBy: args.orderBy,
+      orderDir: args.orderDir,
+      limit: args.limit,
+    });
+
+    return {
+      dataset: { id: meta.id, name: meta.name, totalRows: meta.rowCount },
+      rowCount: result.length,
+      rows: result,
+    };
+  },
+});

package/src/server/lib/provider-api.ts

@@ -1,6 +1,7 @@
 import {
   PROVIDER_API_IDS,
   createProviderApiRuntime,
+  type ProviderApiDocsOptions,
   listCustomProviders,
   type ProviderApiId,
   type ProviderApiMethod,
@@ -8,6 +9,7 @@ import {
 } from "@agent-native/core/provider-api";
 import { getCredentialContext } from "@agent-native/core/server";
 
+export const DISPATCH_APP_ID = "dispatch";
 export const DISPATCH_PROVIDER_API_IDS = PROVIDER_API_IDS;
 export type DispatchProviderApiId = ProviderApiId;
 export type { ProviderApiMethod, ProviderApiRequestArgs };
@@ -23,7 +25,7 @@ function requireCtx(action: string) {
 }
 
 const runtime = createProviderApiRuntime({
-  appId: "dispatch",
+  appId: DISPATCH_APP_ID,
   localCredentialSource: "dispatch_local",
   getCredentialContext: () => requireCtx("requests"),
   getCustomProviders: async () => {
@@ -56,11 +58,9 @@ export function listProviderApiCatalog(provider?: string) {
   return runtime.listCatalog(provider);
 }
 
-export function fetchProviderApiDocs(options: {
-  provider: string;
-  url?: string;
-  maxBytes?: number;
-}) {
+export function fetchProviderApiDocs(
+  options: ProviderApiDocsOptions & { provider: string },
+) {
   return runtime.fetchDocs(options);
 }
 

package/src/server/plugins/agent-chat.ts

@@ -15,6 +15,7 @@ export default createAgentChatPlugin({
   // a build-time-generated `.generated/actions-registry.ts` (the latter is a
   // template-only construct that the Vite plugin emits next to actions/).
   actions: dispatchActions,
+  codeExecution: { production: "sandboxed" },
   systemPrompt: `You are the central dispatch for this workspace.
 
 Default posture:
@@ -33,6 +34,7 @@ Use the standard workspace primitives:
 - If the starter template is used, treat it as scaffolding only: the finished app must be branded as the requested app with its own home screen/navigation/package metadata/manifest, and must not leave visible "Starter", "Blank app", or "New app" UI behind.
 - Treat first-party apps such as Mail, Calendar, Analytics, Brain, Assets, and Dispatch as existing hosted/connected neighbors available through links and A2A/default connected agents. Do not create wrapper apps, child apps, nested routes, or cloned template copies just to give a new app access to them; build only the genuinely new workflow and delegate cross-app work to those existing apps.
 - Integration grants are not provider capability limits. For ad hoc provider inspection, querying, reporting, or troubleshooting, call provider-api-catalog/provider-api-docs, then provider-api-request against the provider's real HTTP API. Use connectionId for a specific shared grant and accountId for a specific OAuth account. Never expose secret values or silently widen app access while doing this.
+- For broad provider searches, joins, classification, corpus counts, or absence claims, fetch every relevant page or an explicitly bounded cohort, stage/save large responses with stageAs/saveToFile/fetchAllPages, and reduce them with query-staged-dataset or run-code. Report source, filters, row counts, pagination, truncation, failed pages, and uncovered gaps.
 
 When a user asks for something like a digest, reminder, routing rule, or saved behavior:
 - First decide whether it should be a resource, a recurring job, a destination, or a delegated task.