la-machina-engine 0.15.1 → 0.16.0

package/dist/index.cjs

@@ -1261,6 +1261,130 @@ var ApiAuthSchema = import_zod.z.discriminatedUnion("type", [
   }).strict(),
   import_zod.z.object({ type: import_zod.z.literal("custom"), id: import_zod.z.string().min(1) }).strict()
 ]);
+var ApiPaginationModeEnum = import_zod.z.enum(["offset", "page", "cursor", "link-header"]);
+var ApiPaginationStopEnum = import_zod.z.enum([
+  "empty-page",
+  "total-reached",
+  "missing-cursor",
+  "missing-next-link"
+]);
+var PAGINATION_MAX_LIMIT = 1e3;
+var ApiPaginationSchema = import_zod.z.object({
+  mode: ApiPaginationModeEnum,
+  request: import_zod.z.object({
+    limitParam: import_zod.z.string().min(1).optional(),
+    offsetParam: import_zod.z.string().min(1).optional(),
+    pageParam: import_zod.z.string().min(1).optional(),
+    cursorParam: import_zod.z.string().min(1).optional(),
+    maxLimit: import_zod.z.number().int().positive().max(PAGINATION_MAX_LIMIT).optional(),
+    firstPage: import_zod.z.number().int().nonnegative().optional()
+  }).strict().optional(),
+  response: import_zod.z.object({
+    itemsPath: import_zod.z.string().min(1).optional(),
+    totalPath: import_zod.z.string().min(1).optional(),
+    nextCursorPath: import_zod.z.string().min(1).optional(),
+    nextLinkPath: import_zod.z.string().min(1).optional()
+  }).strict().optional(),
+  stop: import_zod.z.object({
+    strategy: ApiPaginationStopEnum
+  }).strict().optional()
+}).strict().superRefine((p, ctx) => {
+  const req2 = p.request ?? {};
+  const res = p.response ?? {};
+  if (p.mode === "offset") {
+    if (req2.limitParam === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=offset requires request.limitParam",
+        path: ["request", "limitParam"]
+      });
+    }
+    if (req2.offsetParam === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=offset requires request.offsetParam",
+        path: ["request", "offsetParam"]
+      });
+    }
+    if (req2.maxLimit === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=offset requires request.maxLimit",
+        path: ["request", "maxLimit"]
+      });
+    }
+    if (res.itemsPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=offset requires response.itemsPath",
+        path: ["response", "itemsPath"]
+      });
+    }
+  } else if (p.mode === "page") {
+    if (req2.pageParam === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=page requires request.pageParam",
+        path: ["request", "pageParam"]
+      });
+    }
+    if (req2.maxLimit === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=page requires request.maxLimit",
+        path: ["request", "maxLimit"]
+      });
+    }
+    if (res.itemsPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=page requires response.itemsPath",
+        path: ["response", "itemsPath"]
+      });
+    }
+  } else if (p.mode === "cursor") {
+    if (req2.cursorParam === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=cursor requires request.cursorParam",
+        path: ["request", "cursorParam"]
+      });
+    }
+    if (res.nextCursorPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=cursor requires response.nextCursorPath",
+        path: ["response", "nextCursorPath"]
+      });
+    }
+    if (res.itemsPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=cursor requires response.itemsPath",
+        path: ["response", "itemsPath"]
+      });
+    }
+  } else if (p.mode === "link-header") {
+    if (res.nextLinkPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=link-header requires response.nextLinkPath (HTTP Link header parsing is deferred)",
+        path: ["response", "nextLinkPath"]
+      });
+    }
+    if (res.itemsPath === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        message: "pagination.mode=link-header requires response.itemsPath",
+        path: ["response", "itemsPath"]
+      });
+    }
+  }
+});
+var ApiResponseMappingSchema = import_zod.z.object({
+  itemsPath: import_zod.z.string().min(1).optional(),
+  totalPath: import_zod.z.string().min(1).optional()
+}).strict();
 var ApiEndpointSchema = import_zod.z.object({
   method: ApiHttpMethodEnum,
   path: import_zod.z.string().regex(/^\//, "path must start with /"),
@@ -1269,7 +1393,10 @@ var ApiEndpointSchema = import_zod.z.object({
   // time by higher-level tooling (nikaido's yaml compiler), not
   // here.
   inputSchema: import_zod.z.unknown().optional(),
-  outputHint: import_zod.z.string().optional()
+  outputHint: import_zod.z.string().optional(),
+  // Plan 050 — declarative pagination + response extraction.
+  pagination: ApiPaginationSchema.optional(),
+  response: ApiResponseMappingSchema.optional()
 }).strict();
 var HEADER_NAME_RE = /^[!#$%&'*+\-.0-9A-Z^_`a-z|~]+$/;
 var RESERVED_HEADER_NAMES = /* @__PURE__ */ new Set([
@@ -8160,6 +8287,11 @@ function getApiServicesSection(opts) {
       "Configured external HTTP APIs. Use the `ApiCall` tool to invoke \u2014 auth is injected automatically. Do not pass credentials via headers."
     );
     lines.push("");
+    appendStrictApiToolRules(
+      lines,
+      /* hasLazy */
+      false
+    );
     for (const svc of withEndpoints) {
       lines.push(`## ${svc.name}${svc.description ? ` \u2014 ${svc.description}` : ""}`);
       lines.push(`baseUrl: ${svc.baseUrl}`);
@@ -8188,6 +8320,11 @@ function getApiServicesSection(opts) {
       "Configured external HTTP APIs. Use `ApiCall` to invoke, but first call `DescribeService(service)` to fetch that service's endpoint catalog. Auth is injected automatically."
     );
     lines.push("");
+    appendStrictApiToolRules(
+      lines,
+      /* hasLazy */
+      true
+    );
     for (const svc of withEndpoints) {
       const count = svc.endpoints.length;
       const suffix = svc.description ? ` \u2014 ${svc.description}` : "";
@@ -8207,6 +8344,24 @@ function getApiServicesSection(opts) {
   }
   return lines.join("\n").trimEnd();
 }
+function appendStrictApiToolRules(lines, hasLazy) {
+  lines.push("Rules:");
+  if (hasLazy) {
+    lines.push(
+      "- Call `DescribeService(name)` before `ApiCall` for any service shown above. The catalog gives you method, path, input schema, and (when declared) pagination + response-extraction metadata."
+    );
+  }
+  lines.push(
+    "- Auth headers (Authorization, X-API-Key, secret headers) are injected by the host. Never pass credentials via `ApiCall.headers`; the runtime drops them and your call may fail without them."
+  );
+  lines.push(
+    "- For endpoints whose catalog entry declares `pagination`, prefer `ApiCall` with `pagination.auto: true` over manually issuing one call per page. The engine then handles request mutation, item extraction, and stop conditions deterministically."
+  );
+  lines.push(
+    "- Tool results may be returned as opaque references when payloads are large. Inspect or otherwise read the shaped fields before deciding next steps. Never finalize using a raw reference id as data, and never include reference ids in user-visible output unless explicitly asked for debugging."
+  );
+  lines.push("");
+}
 function resolveEffectiveMode(services, requested, threshold) {
   if (requested === "eager" || requested === "lazy") return requested;
   let total = 0;
@@ -8436,6 +8591,10 @@ init_contract();
 var ALL_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"];
 var DEFAULT_MAX_BODY_BYTES = 256 * 1024;
 var DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024;
+var DEFAULT_MAX_PAGES = 5;
+var MAX_PAGES_HARD_CAP = 50;
+var DEFAULT_MAX_ITEMS = 500;
+var MAX_ITEMS_HARD_CAP = 1e4;
 function createApiCallTool(opts) {
   if (opts.services.length === 0) {
     throw new Error("createApiCallTool: services list must be non-empty");
@@ -8482,7 +8641,21 @@ function createApiCallTool(opts) {
      *           ingestion, image push, etc.).
      */
     bodyEncoding: import_zod23.z.enum(["json", "raw"]).optional(),
-    headers: import_zod23.z.record(import_zod23.z.string(), import_zod23.z.string()).optional()
+    headers: import_zod23.z.record(import_zod23.z.string(), import_zod23.z.string()).optional(),
+    /**
+     * Plan 050 — opt-in auto-pagination. When `auto: true`, the
+     * engine looks up the endpoint's pagination metadata from the
+     * service catalog and loops pages deterministically. Without
+     * `auto`, ApiCall behaves as a single-shot HTTP call (pre-050).
+     *
+     * `maxPages` defaults to 5, hard cap 50.
+     * `maxItems` defaults to 500, hard cap 10000.
+     */
+    pagination: import_zod23.z.object({
+      auto: import_zod23.z.boolean().optional().default(false),
+      maxPages: import_zod23.z.number().int().min(1).max(MAX_PAGES_HARD_CAP).optional(),
+      maxItems: import_zod23.z.number().int().min(1).max(MAX_ITEMS_HARD_CAP).optional()
+    }).optional()
   });
   const description = opts.toolDescription ?? `Call a configured external API. Services: ${serviceNames.join(", ")}. Auth is injected automatically \u2014 do not pass credentials via headers.`;
   return defineTool({
@@ -8504,6 +8677,18 @@ function createApiCallTool(opts) {
       if (!pathAllowed(input.path, effectivePaths)) {
         return errResult(`ERR_API_PATH_NOT_ALLOWED: ${input.path} for service ${svc.name}`);
       }
+      if (input.pagination?.auto === true) {
+        return executeAutoPaginated({
+          svc,
+          input,
+          fetchFn,
+          maxResponseBytes,
+          env: opts.env,
+          resolveAuth: opts.resolveAuth,
+          onRequest: opts.onRequest,
+          onResponse: opts.onResponse
+        });
+      }
       let bodyText;
       let defaultContentType;
       if (input.body !== void 0) {
@@ -8702,6 +8887,287 @@ async function invokeHook(hook, event) {
   } catch {
   }
 }
+function matchEndpoint(endpoints, method, path) {
+  if (endpoints === void 0 || endpoints.length === 0) return { error: "NO_MATCH" };
+  const exact = [];
+  const templated = [];
+  for (const ep of endpoints) {
+    if (ep.method !== method) continue;
+    if (ep.path === path) {
+      exact.push(ep);
+      continue;
+    }
+    if (ep.path.includes("{") && pathMatchesTemplate(ep.path, path)) {
+      templated.push(ep);
+    }
+  }
+  if (exact.length === 1) return { endpoint: exact[0] };
+  if (exact.length > 1) return { error: "AMBIGUOUS" };
+  if (templated.length === 1) return { endpoint: templated[0] };
+  if (templated.length > 1) return { error: "AMBIGUOUS" };
+  return { error: "NO_MATCH" };
+}
+function pathMatchesTemplate(template, concrete) {
+  const pattern = "^" + template.split(/(\{[^}]+\})/).map(
+    (seg) => seg.startsWith("{") && seg.endsWith("}") ? "[^/]+" : seg.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+  ).join("") + "$";
+  return new RegExp(pattern).test(concrete);
+}
+function extractByDotPath(value, path) {
+  if (path.length === 0) return value;
+  let cur = value;
+  for (const seg of path.split(".")) {
+    if (cur === null || typeof cur !== "object") return void 0;
+    cur = cur[seg];
+  }
+  return cur;
+}
+function asItemsArray(value) {
+  return Array.isArray(value) ? value : null;
+}
+function asTotalCount(value) {
+  if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+    return Math.floor(value);
+  }
+  if (typeof value === "string") {
+    const n = Number(value);
+    if (Number.isFinite(n) && n >= 0) return Math.floor(n);
+  }
+  return null;
+}
+async function executeAutoPaginated(args) {
+  const { svc, input, fetchFn, maxResponseBytes } = args;
+  const matched = matchEndpoint(svc.endpoints, input.method, input.path);
+  if ("error" in matched) {
+    if (matched.error === "NO_MATCH") {
+      return errResult(
+        `ERR_API_PAGINATION_NO_METADATA: no catalog entry for ${input.method} ${input.path} on service ${svc.name} \u2014 auto-pagination requires endpoints[].pagination`
+      );
+    }
+    return errResult(
+      `ERR_API_ENDPOINT_AMBIGUOUS: ${input.method} ${input.path} matched multiple catalog entries on service ${svc.name}`
+    );
+  }
+  const endpoint = matched.endpoint;
+  const p = endpoint.pagination;
+  if (p === void 0) {
+    return errResult(
+      `ERR_API_PAGINATION_NO_METADATA: endpoint ${input.method} ${input.path} on service ${svc.name} has no pagination metadata; remove pagination.auto or declare it in the catalog`
+    );
+  }
+  const itemsPath = p.response?.itemsPath ?? endpoint.response?.itemsPath;
+  if (itemsPath === void 0) {
+    return errResult(
+      `ERR_API_PAGINATION_NO_METADATA: endpoint ${input.method} ${input.path} declares pagination but no response.itemsPath; cannot extract items`
+    );
+  }
+  const maxPages = Math.min(input.pagination?.maxPages ?? DEFAULT_MAX_PAGES, MAX_PAGES_HARD_CAP);
+  const maxItems = Math.min(input.pagination?.maxItems ?? DEFAULT_MAX_ITEMS, MAX_ITEMS_HARD_CAP);
+  const secretHeaderRefs = svc.secretHeaders !== void 0 && Object.keys(svc.secretHeaders).length > 0 ? svc.secretHeaders : void 0;
+  let authHeaders;
+  try {
+    authHeaders = await resolveAuth({
+      auth: svc.auth ?? { type: "none" },
+      env: args.env,
+      resolver: args.resolveAuth,
+      ctx: {
+        serviceName: svc.name,
+        method: input.method,
+        path: input.path,
+        ...secretHeaderRefs !== void 0 ? { secretHeaderRefs } : {}
+      },
+      forceResolve: secretHeaderRefs !== void 0
+    });
+  } catch (err) {
+    const raw = err instanceof Error ? err.message : String(err);
+    const truncated = raw.length > 200 ? raw.slice(0, 200) + "\u2026" : raw;
+    return errResult(`ERR_API_RESOLVER_FAILED: ${truncated}`);
+  }
+  const userHeaders = sanitizeHeaders(input.headers ?? {}, authHeaders);
+  const aggregated = [];
+  let pagesFetched = 0;
+  let stoppedBy = "unknown";
+  let lastStatus = 0;
+  let nextOffset = 0;
+  let nextPage = p.request?.firstPage ?? 1;
+  let nextCursor = null;
+  let nextLink = null;
+  while (pagesFetched < maxPages && aggregated.length < maxItems) {
+    const pageQuery = { ...input.query ?? {} };
+    let pageUrl = null;
+    if (p.mode === "offset") {
+      const limit = p.request?.maxLimit ?? 100;
+      if (p.request?.limitParam) pageQuery[p.request.limitParam] = String(limit);
+      if (p.request?.offsetParam) pageQuery[p.request.offsetParam] = String(nextOffset);
+    } else if (p.mode === "page") {
+      if (p.request?.maxLimit !== void 0 && p.request?.limitParam) {
+        pageQuery[p.request.limitParam] = String(p.request.maxLimit);
+      }
+      if (p.request?.pageParam) pageQuery[p.request.pageParam] = String(nextPage);
+    } else if (p.mode === "cursor") {
+      if (p.request?.maxLimit !== void 0 && p.request?.limitParam) {
+        pageQuery[p.request.limitParam] = String(p.request.maxLimit);
+      }
+      if (nextCursor !== null && p.request?.cursorParam) {
+        pageQuery[p.request.cursorParam] = nextCursor;
+      }
+    } else if (p.mode === "link-header") {
+      if (nextLink !== null) pageUrl = nextLink;
+    }
+    const url = pageUrl ?? buildUrl(svc.baseUrl, input.path, pageQuery);
+    await invokeHook(args.onRequest, {
+      service: svc.name,
+      method: input.method,
+      path: input.path
+    });
+    const started = Date.now();
+    let res;
+    try {
+      res = await fetchFn(url, {
+        method: input.method,
+        headers: {
+          ...svc.defaultHeaders ?? {},
+          ...userHeaders,
+          ...authHeaders
+        }
+      });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return paginationErr({
+        code: "ERR_API_PAGINATION_PAGE_FAILED",
+        message: `${input.method} ${input.path} page ${pagesFetched + 1} network error: ${msg}`,
+        pagesFetched,
+        itemsFetched: aggregated.length,
+        partialItems: aggregated
+      });
+    }
+    lastStatus = res.status;
+    const raw = await res.text();
+    const captured = raw.length > maxResponseBytes ? raw.slice(0, maxResponseBytes) + "\n\u2026[TRUNCATED]" : raw;
+    await invokeHook(args.onResponse, {
+      service: svc.name,
+      method: input.method,
+      path: input.path,
+      status: res.status,
+      latencyMs: Date.now() - started,
+      bytesIn: raw.length
+    });
+    if (!res.ok) {
+      return paginationErr({
+        code: "ERR_API_PAGINATION_PAGE_FAILED",
+        message: `${input.method} ${input.path} page ${pagesFetched + 1} returned status ${String(res.status)}`,
+        pagesFetched,
+        itemsFetched: aggregated.length,
+        partialItems: aggregated,
+        responsePreview: captured
+      });
+    }
+    let body;
+    try {
+      body = JSON.parse(captured);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return paginationErr({
+        code: "ERR_API_PAGINATION_PAGE_FAILED",
+        message: `${input.method} ${input.path} page ${pagesFetched + 1} returned non-JSON: ${msg}`,
+        pagesFetched,
+        itemsFetched: aggregated.length,
+        partialItems: aggregated
+      });
+    }
+    const itemsRaw = extractByDotPath(body, itemsPath);
+    const items = asItemsArray(itemsRaw);
+    if (items === null) {
+      return errResult(
+        `ERR_API_ITEMS_PATH_INVALID: ${input.method} ${input.path} response.${itemsPath} did not resolve to an array`
+      );
+    }
+    pagesFetched += 1;
+    for (const it of items) {
+      if (aggregated.length >= maxItems) break;
+      aggregated.push(it);
+    }
+    const stop = p.stop?.strategy;
+    if (stop === "empty-page" && items.length === 0) {
+      stoppedBy = "empty-page";
+      break;
+    }
+    if (stop === "total-reached" && p.response?.totalPath !== void 0) {
+      const total = asTotalCount(extractByDotPath(body, p.response.totalPath));
+      if (total !== null && aggregated.length >= total) {
+        stoppedBy = "total-reached";
+        break;
+      }
+    }
+    if (p.mode === "cursor") {
+      const cursorRaw = p.response?.nextCursorPath ? extractByDotPath(body, p.response.nextCursorPath) : void 0;
+      const cursor = typeof cursorRaw === "string" && cursorRaw.length > 0 ? cursorRaw : null;
+      if (cursor === null) {
+        stoppedBy = "missing-cursor";
+        break;
+      }
+      nextCursor = cursor;
+    } else if (p.mode === "link-header") {
+      const linkRaw = p.response?.nextLinkPath ? extractByDotPath(body, p.response.nextLinkPath) : void 0;
+      const link = typeof linkRaw === "string" && linkRaw.length > 0 ? linkRaw : null;
+      if (link === null) {
+        stoppedBy = "missing-next-link";
+        break;
+      }
+      nextLink = link;
+    } else if (p.mode === "offset") {
+      if (items.length === 0) {
+        stoppedBy = "empty-page-fallback";
+        break;
+      }
+      nextOffset += items.length;
+    } else if (p.mode === "page") {
+      if (items.length === 0) {
+        stoppedBy = "empty-page-fallback";
+        break;
+      }
+      nextPage += 1;
+    }
+  }
+  if (stoppedBy === "unknown") {
+    if (aggregated.length >= maxItems) stoppedBy = "max-items";
+    else if (pagesFetched >= maxPages) stoppedBy = "max-pages";
+  }
+  const payload = {
+    ok: true,
+    status: lastStatus || 200,
+    items: aggregated,
+    pagination: {
+      mode: p.mode,
+      pagesFetched,
+      itemsFetched: aggregated.length,
+      stoppedBy
+    }
+  };
+  return {
+    content: JSON.stringify(payload),
+    metadata: {
+      status: lastStatus || 200,
+      service: svc.name,
+      pagesFetched,
+      itemsFetched: aggregated.length,
+      stoppedBy
+    }
+  };
+}
+function paginationErr(args) {
+  const payload = {
+    ok: false,
+    error: { code: args.code, message: args.message },
+    pagination: {
+      pagesFetched: args.pagesFetched,
+      itemsFetched: args.itemsFetched
+    },
+    partialItems: args.partialItems,
+    ...args.responsePreview !== void 0 ? { responsePreview: args.responsePreview.slice(0, 4096) } : {}
+  };
+  return { content: JSON.stringify(payload), isError: true };
+}
 
 // src/tools/describeService.ts
 init_cjs_shims();
@@ -8714,7 +9180,12 @@ function describe(svc) {
       path: ep.path,
       description: ep.description,
       ...ep.inputSchema !== void 0 ? { inputSchema: ep.inputSchema } : {},
-      ...ep.outputHint !== void 0 ? { outputHint: ep.outputHint } : {}
+      ...ep.outputHint !== void 0 ? { outputHint: ep.outputHint } : {},
+      // Plan 050 — surface pagination + response extraction metadata
+      // so the model knows it can request `pagination.auto: true` and
+      // sees the declared items / total / cursor paths inline.
+      ...ep.pagination !== void 0 ? { pagination: ep.pagination } : {},
+      ...ep.response !== void 0 ? { response: ep.response } : {}
     })
   );
   return {
@@ -8741,7 +9212,7 @@ function createDescribeServiceTool(opts) {
   const inputSchema19 = import_zod24.z.object({
     service: import_zod24.z.enum(serviceNames)
   });
-  const description = `Look up the endpoint catalog for one configured API service. Returns every endpoint's method, path, description, and input schema. Call this before invoking \`ApiCall\` when the service has lazy endpoints. Services: ${serviceNames.join(", ")}.`;
+  const description = `Look up the endpoint catalog for one configured API service. Returns every endpoint's method, path, description, input schema, and (when declared) pagination + response-extraction metadata. Call this before invoking \`ApiCall\` when the service has lazy endpoints; for paginated list endpoints, prefer \`ApiCall\` with \`pagination.auto: true\` over manually issuing one call per page. Services: ${serviceNames.join(", ")}.`;
   return defineTool({
     name: opts.toolName ?? "DescribeService",
     description,