@executor-js/plugin-openapi 1.5.16 → 1.5.18

package/dist/sdk/backing.d.ts

@@ -6,6 +6,7 @@ import { type OpenApiIntegrationConfig } from "./config";
 import { OpenApiExtractionError, OpenApiParseError } from "./errors";
 import { type ToolDefinition } from "./definitions";
 import { type ParsedDocument } from "./parse";
+import { type KeepPathItem, type SpecStructure } from "./split";
 import { type OpenapiStore, type StoredOperation } from "./store";
 export declare const extractOpenApiUpstreamMessage: (body: unknown, status: number) => string;
 /** Rewrite OpenAPI `#/components/schemas/X` refs to standard `#/$defs/X`. */
@@ -20,8 +21,97 @@ export declare const compileOpenApiDocument: (doc: ParsedDocument) => Effect.Eff
 export declare const compileOpenApiSpec: (specText: string) => Effect.Effect<CompiledOpenApiSpec, OpenApiParseError | OpenApiExtractionError>;
 export declare const openApiToolDefsFromCompiled: (compiled: CompiledOpenApiSpec) => readonly ToolDef[];
 export declare const openApiStoredOperationsFromCompiled: (integration: string, compiled: CompiledOpenApiSpec) => readonly StoredOperation[];
+/**
+ * Serialize a document's `components.schemas` into the content-addressed defs
+ * blob JSON (`{ "<Name>": <normalized schema>, ... }`), one schema at a time.
+ * Normalizing + stringifying per entry keeps the whole normalized definition
+ * tree from ever being co-resident with the parsed document, so the streaming
+ * add path's peak stays near parse level. The serve path JSON-parses this blob
+ * to rebuild the shared `definitions` instead of re-parsing the spec.
+ */
+export declare const buildDefsJson: (doc: ParsedDocument) => string;
+/**
+ * Streaming twin of `buildDefsJson`: serialize the content-addressed defs blob
+ * from a `SpecStructure` by parsing each `components.schemas` entry in isolation
+ * (indent-4 range) rather than from a whole-document parse. Used by the fully
+ * streaming add path so the 37MB Microsoft Graph spec never builds its
+ * ~300MB tree. The blob carries *all* source schemas (it is shared across every
+ * tenant/selection on the same spec hash); extra unreferenced `$defs` are
+ * harmless. Like `buildDefsJson`, normalizing + stringifying per entry keeps the
+ * whole normalized tree from being co-resident with any parsed schema, and the
+ * ConsString accumulation avoids the join-doubling of an array build.
+ */
+export declare const buildDefsJsonStreaming: (structure: SpecStructure) => string;
+export interface OpenApiPersistResult {
+    readonly toolCount: number;
+    readonly toolNames: readonly string[];
+}
+/**
+ * Compile a parsed document straight to persisted operation bindings, streaming
+ * in bounded chunks so a huge spec's bindings are never all co-resident with
+ * the parsed tree. This is the memory-safe replacement for
+ * `compileOpenApiDocument` + `openApiStoredOperationsFromCompiled` + `putOperations`
+ * on the add/update path: it skips per-op input/output schema assembly (the
+ * serve path rebuilds those on demand from the bindings). Clears existing
+ * operations first, then appends each chunk. When `specHash` is given, also
+ * stream-serializes the document's `#/$defs/*` into the content-addressed defs
+ * blob so the serve path can resolve the shared `definitions` without
+ * re-parsing the spec.
+ */
+export declare const compileAndPersistOpenApiOperations: ({ doc, integration, storage, specHash, chunkSize, }: {
+    readonly doc: ParsedDocument;
+    readonly integration: string;
+    readonly storage: OpenapiStore;
+    readonly specHash?: string;
+    readonly chunkSize?: number;
+}) => Effect.Effect<OpenApiPersistResult, OpenApiExtractionError | StorageFailure>;
+/** Parse spec text, then stream-compile + persist its bindings (and, when
+ *  `specHash` is given, the content-addressed defs blob). */
+export declare const compileAndPersistOpenApiSpec: ({ specText, integration, storage, specHash, chunkSize, }: {
+    readonly specText: string;
+    readonly integration: string;
+    readonly storage: OpenapiStore;
+    readonly specHash?: string;
+    readonly chunkSize?: number;
+}) => Effect.Effect<OpenApiPersistResult, OpenApiParseError | OpenApiExtractionError | StorageFailure>;
+/**
+ * Fully streaming add/update path: compile + persist operation bindings (and the
+ * content-addressed defs blob) straight from spec *text*, without ever parsing
+ * the whole document. The text is structurally split, then each path-item and
+ * each schema is parsed in isolation and discarded, so peak memory stays near
+ * one item rather than the ~300MB whole-tree parse that OOMs a 128MB Workers
+ * isolate on the 37MB Microsoft Graph spec.
+ *
+ * There is deliberately no whole-parse fallback: a spec that does not present
+ * the streamable block-YAML profile (no top-level `paths:` block) is a hard
+ * `OpenApiExtractionError`, because the fallback is exactly the OOM this path
+ * exists to avoid. `keepPathItem` optionally filters/trims path-items (the
+ * Microsoft Graph scope selection), so the same primitive serves a full-spec
+ * compile and a selection identically.
+ */
+export declare const compileAndPersistOpenApiSpecStreaming: ({ specText, integration, storage, specHash, chunkSize, keepPathItem, }: {
+    readonly specText: string;
+    readonly integration: string;
+    readonly storage: OpenapiStore;
+    readonly specHash?: string;
+    readonly chunkSize?: number;
+    readonly keepPathItem?: KeepPathItem;
+}) => Effect.Effect<OpenApiPersistResult, OpenApiExtractionError | StorageFailure>;
 export declare const loadOpenApiSpecText: (storage: OpenapiStore, config: OpenApiIntegrationConfig) => Effect.Effect<string | null, StorageFailure>;
-export declare const resolveOpenApiBackedTools: ({ config, storage, }: {
+/**
+ * Resolve the tool defs + shared definitions for a connection refresh
+ * (`tools/list`). Fast path: serve from the persisted operation bindings plus
+ * the content-addressed defs blob, rebuilding each tool's input/output schema
+ * on demand, so a 37MB spec is never re-parsed (the 2nd OOM site). The defs
+ * blob is global per `specHash`, so the heavy normalize work is done once at
+ * add time and shared across every tenant on the same spec. Falls back to the
+ * spec re-parse for legacy rows persisted before the defs blob existed (or if
+ * the blob is missing/corrupt).
+ */
+export declare const resolveOpenApiBackedTools: ({ integration, config, storage, }: {
+    readonly integration: {
+        readonly slug: string;
+    };
     readonly config: unknown;
     readonly storage: OpenapiStore;
 }) => Effect.Effect<ResolveToolsResult, StorageFailure>;

package/dist/sdk/definitions.d.ts

@@ -18,8 +18,40 @@ export interface ToolDefinition {
     /** The original operation */
     readonly operation: ExtractedOperation;
 }
+/**
+ * The minimal per-operation metadata the tool-path planner needs. Kept
+ * deliberately light (no schemas) so the streaming compile path can plan paths
+ * for tens of thousands of operations without holding the full extracted
+ * operations in memory.
+ */
+export interface OperationPathInput {
+    readonly operationId: string;
+    readonly explicitToolPath: string | undefined;
+    readonly method: string;
+    readonly pathTemplate: string;
+    /** The first non-empty tag, used to seed the group segment. */
+    readonly tag0: string | undefined;
+}
+/** A resolved tool path plus its index back into the input operations array. */
+export interface PlannedToolPath {
+    readonly toolPath: string;
+    readonly group: string;
+    readonly leaf: string;
+    readonly operationIndex: number;
+}
+/**
+ * Plan structured `group.leaf` tool paths from lightweight per-operation
+ * metadata. Path derivation + collision resolution need a global view of every
+ * operation, but only its name/method/path/tag, never its schemas. Keeping this
+ * pass schema-free lets the streaming compile path plan paths for a 16k-op spec
+ * without materializing the full extracted operations.
+ *
+ * The returned `operationIndex` points back into the `inputs` array.
+ */
+export declare const planToolPaths: (inputs: readonly OperationPathInput[]) => PlannedToolPath[];
 /**
  * Compile extracted operations into tool definitions with structured
- * `group.leaf` paths suitable for tree rendering.
+ * `group.leaf` paths suitable for tree rendering. Thin wrapper over
+ * `planToolPaths` that re-attaches each operation by index.
  */
 export declare const compileToolDefinitions: (operations: readonly ExtractedOperation[]) => ToolDefinition[];

package/dist/sdk/extract.d.ts

@@ -1,6 +1,9 @@
 import { Effect, Option } from "effect";
 import { OpenApiExtractionError } from "./errors";
 import type { ParsedDocument } from "./parse";
+import { type KeepPathItem, type SpecStructure } from "./split";
+import { OperationBinding, OperationParameter, OperationRequestBody, ServerInfo } from "./types";
+export declare const buildInputSchema: (parameters: readonly OperationParameter[], requestBody: OperationRequestBody | undefined, servers: readonly ServerInfo[]) => Record<string, unknown> | undefined;
 /** Extract all operations from a bundled OpenAPI 3.x document */
 export declare const extract: (doc: ParsedDocument) => Effect.Effect<{
     readonly version: Option.Option<string>;
@@ -79,3 +82,66 @@ export declare const extract: (doc: ParsedDocument) => Effect.Effect<{
         }>;
     }[];
 }, OpenApiExtractionError, never>;
+/** One persisted invocation binding plus the tool name and description it
+ *  backs. The description is the resolved operation description / summary /
+ *  method+path fallback, persisted so the serve path needs no re-parse. */
+export interface OperationBindingChunk {
+    readonly toolName: string;
+    readonly description: string;
+    readonly binding: OperationBinding;
+}
+/**
+ * Stream invocation bindings out of a parsed document in bounded chunks,
+ * persisting each chunk via `onChunk` before building the next.
+ *
+ * This is the memory-safe compile path for huge specs (e.g. Microsoft Graph,
+ * 16.5k operations / 37MB). It differs from `extract` + `compileToolDefinitions`
+ * in two ways that keep peak memory at parse level rather than ~doubling it:
+ *
+ *   1. It never builds `hoistedDefs` or per-operation `inputSchema`/`outputSchema`
+ *      (the add path only needs invocation bindings, which carry `$ref`s, not
+ *      inlined schemas).
+ *   2. It never holds all bindings at once. Tool-path planning needs a global
+ *      view, but only of lightweight metadata (`planToolPaths`, schema-free);
+ *      the heavy per-operation bindings are built, flushed, and dropped one
+ *      chunk at a time.
+ *
+ * Bindings reference subtrees of the parsed document rather than copying them,
+ * so `onChunk` must sever those references (its storage layer JSON-serializes
+ * the binding) before the chunk is dropped. Returns the resolved tool names in
+ * sorted order, matching `compileToolDefinitions`.
+ */
+export declare const streamOperationBindings: <E, R>(doc: ParsedDocument, chunkSize: number, onChunk: (chunk: readonly OperationBindingChunk[]) => Effect.Effect<void, E, R>) => Effect.Effect<{
+    readonly toolCount: number;
+    readonly toolNames: readonly string[];
+}, OpenApiExtractionError | E, R>;
+/**
+ * Stream invocation bindings straight from a `SpecStructure` (the structural
+ * split of a large spec) without ever materializing the whole-document tree.
+ *
+ * This is the fully-streaming compile path: it never parses the spec whole.
+ * Each path-item is parsed in isolation twice (pass 1 for light tool-path
+ * planning metadata, pass 2 to build the heavy binding) and discarded, so peak
+ * memory stays near the size of a single path-item plus the raw text, even for
+ * the 37MB / 16.5k-operation Microsoft Graph spec that OOMs a whole-tree parse.
+ * Re-parsing in pass 2 (rather than holding pass-1 tree references) is the
+ * deliberate CPU-for-memory trade that keeps peak at one-path-item level.
+ *
+ * `keepPathItem`, when given, filters (and may trim) each path-item, so the same
+ * primitive serves both a full-spec compile (no filter) and a selection (e.g.
+ * the Microsoft Graph scope filter) with identical streaming guarantees. It is
+ * applied identically in both passes so the per-operation index stays aligned.
+ *
+ * Schemas are never resolved here: parameter / requestBody / response component
+ * `$ref`s resolve against the small (schema-free) components built from the
+ * structure; `#/components/schemas/X` refs stay as strings in the binding and
+ * are normalized + served from the content-addressed defs blob. Returns the
+ * resolved tool names in sorted order, matching `compileToolDefinitions`.
+ */
+export declare const streamOperationBindingsFromStructure: <E, R>(structure: SpecStructure, options: {
+    readonly chunkSize: number;
+    readonly keepPathItem?: KeepPathItem;
+}, onChunk: (chunk: readonly OperationBindingChunk[]) => Effect.Effect<void, E, R>) => Effect.Effect<{
+    readonly toolCount: number;
+    readonly toolNames: readonly string[];
+}, E, R>;

package/dist/sdk/index.d.ts

@@ -1,7 +1,8 @@
 export { parse, resolveSpecText, fetchSpecText } from "./parse";
-export { extract } from "./extract";
+export { extract, streamOperationBindingsFromStructure } from "./extract";
+export { structuralSplit, isStreamableSpec, indexSchemas, collectReferencedSchemas, parseEntry, parseHead, parseSmallComponents, type SpecStructure, type ByteRange, type KeepPathItem, } from "./split";
 export { invoke, invokeWithLayer, annotationsForOperation } from "./invoke";
-export { compileOpenApiDocument, compileOpenApiSpec, extractOpenApiUpstreamMessage, invokeOpenApiBackedTool, loadOpenApiSpecText, normalizeOpenApiRefs, openApiStoredOperationsFromCompiled, openApiToolDefsFromCompiled, resolveOpenApiBackedAnnotations, resolveOpenApiBackedTools, type CompiledOpenApiSpec, } from "./backing";
+export { buildDefsJsonStreaming, compileAndPersistOpenApiOperations, compileAndPersistOpenApiSpec, compileAndPersistOpenApiSpecStreaming, compileOpenApiDocument, compileOpenApiSpec, extractOpenApiUpstreamMessage, invokeOpenApiBackedTool, loadOpenApiSpecText, normalizeOpenApiRefs, openApiStoredOperationsFromCompiled, openApiToolDefsFromCompiled, resolveOpenApiBackedAnnotations, resolveOpenApiBackedTools, type CompiledOpenApiSpec, type OpenApiPersistResult, } from "./backing";
 export type { ParsedDocument } from "./parse";
 export { openApiPlugin, type OpenApiSpecConfig, type OpenApiConfigureInput, type OpenApiSpecInput, type OpenApiPreviewInput, type OpenApiPluginExtension, type OpenApiPluginOptions, } from "./plugin";
 export { type OpenapiStore, type StoredOperation, makeDefaultOpenapiStore } from "./store";

package/dist/sdk/request-user-agent.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/sdk/split.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Structural splitter for large OpenAPI documents written in clean block YAML.
+ *
+ * The whole-document parse of a 37MB spec (Microsoft Graph: 16.5k operations,
+ * 8.2k schemas) builds a ~300MB JS tree that OOMs the 128MB Cloudflare Workers
+ * isolate. This splitter avoids ever holding that tree: it scans the text once,
+ * recording the byte range of each top-level key, each path-item (the indent-2
+ * entries under `paths:`), and each schema (the indent-4 entries under
+ * `components.schemas:`). The streaming compile then slices one item at a time,
+ * de-indents it back to column 0, hands the isolated fragment to a real YAML
+ * parser, and discards the result before moving on. Peak memory stays near the
+ * size of the largest single item plus the raw text, not the parsed whole.
+ *
+ * This is not a YAML reimplementation: it extracts safe byte ranges from a
+ * constrained document shape, then defers every actual parse to `js-yaml`. It
+ * is only valid for the block-YAML profile `isStreamableSpec` accepts (2-space
+ * block maps, top-level keys at column 0, no anchors/aliases/merge keys). Block
+ * scalars (`|` / `>`) in descriptions are tracked so their indented content is
+ * never mistaken for structure.
+ */
+export interface ByteRange {
+    readonly start: number;
+    readonly end: number;
+}
+export interface SpecStructure {
+    /** Raw spec text. Every range below indexes into this string. */
+    readonly text: string;
+    /** Byte ranges of the top-level keys that are not `paths` / `components`
+     *  (openapi, info, servers, tags, security, ...). Concatenated and parsed as
+     *  one small document for the head (servers, info). */
+    readonly headRanges: readonly ByteRange[];
+    /** One range per path-item: the indent-2 entries under `paths:`. */
+    readonly pathItems: readonly ByteRange[];
+    /** One range per schema entry: the indent-4 entries under
+     *  `components.schemas:`. */
+    readonly schemas: readonly ByteRange[];
+    /** Ranges of the indent-2 `components` subkeys we keep whole because they are
+     *  small and may be `$ref`'d by a kept operation (parameters / requestBodies /
+     *  responses / headers / links / securitySchemes). Excludes the huge `schemas`
+     *  (streamed and pruned separately) and `examples` (never referenced by a
+     *  binding). */
+    readonly smallComponentRanges: readonly ByteRange[];
+}
+/**
+ * Scan a document into its structural ranges. Pure and synchronous; never
+ * parses. Returns null when the document does not present the expected shape
+ * (no `paths:` block), so the caller can fall back to a whole-document parse.
+ */
+export declare const structuralSplit: (text: string) => SpecStructure | null;
+/**
+ * Parse a single indent-N entry range (a path-item or a schema) in isolation.
+ * Returns `[name, value]` where `name` is the entry's key and `value` its
+ * parsed body, or null when the fragment does not parse to a single-key map.
+ */
+export declare const parseEntry: (text: string, range: ByteRange, indent: number) => readonly [string, unknown] | null;
+/** Parse the concatenation of the head ranges into the document head (openapi,
+ *  info, servers, tags). Small; safe to materialize whole. */
+export declare const parseHead: (structure: SpecStructure) => Record<string, unknown>;
+/** Parse the small component subkeys (parameters / requestBodies / responses)
+ *  into a `components`-shaped object for `$ref` resolution. */
+export declare const parseSmallComponents: (structure: SpecStructure) => Record<string, unknown>;
+/**
+ * Accept only the block-YAML profile the splitter can safely slice: no tabs, no
+ * anchors/aliases, no merge keys, and a `paths:` block present. Block scalars
+ * are allowed (tracked during the scan). Conservative by design: a false
+ * negative just routes a spec through the whole-document parse.
+ */
+export declare const isStreamableSpec: (text: string) => boolean;
+/** Map each `components.schemas` entry name to its byte range, reading only the
+ *  key line (never parsing the schema body). The schema name is the raw YAML
+ *  key, which matches the trailing segment of a `#/components/schemas/<name>`
+ *  reference. */
+export declare const indexSchemas: (structure: SpecStructure) => ReadonlyMap<string, ByteRange>;
+/**
+ * Resolve the transitive closure of schemas referenced by `roots`, parsing each
+ * referenced schema once from its byte range (BFS over `$ref`s). Schemas not
+ * reachable from `roots` are never parsed, so peak memory tracks the kept
+ * subset rather than the full `components.schemas` map.
+ */
+export declare const collectReferencedSchemas: (structure: SpecStructure, index: ReadonlyMap<string, ByteRange>, roots: readonly unknown[]) => Record<string, unknown>;
+/** A path-item filter for the streaming compile: given a parsed path-item,
+ *  return the (possibly trimmed) value to keep, or null to drop the path
+ *  entirely. Applied per path-item by `streamOperationBindingsFromStructure`. */
+export type KeepPathItem = (path: string, pathItem: Record<string, unknown>) => Record<string, unknown> | null;

package/dist/sdk/store.d.ts

@@ -7,13 +7,25 @@ export interface StoredOperation {
     /** The tool name (the `<tool>` address segment) this operation backs. */
     readonly toolName: string;
     readonly binding: OperationBinding;
+    /** Resolved tool description, persisted alongside the binding so the serve
+     *  path can rebuild the tool def without re-parsing the spec. */
+    readonly description?: string;
 }
 /** Blob key for a spec's content hash. Content-addressed so re-puts are
  *  idempotent and identical specs share one blob per partition. */
 export declare const specBlobKey: (specHash: string) => string;
+/** Blob key for a spec's compiled `#/$defs/*` schemas, keyed by the same
+ *  content hash as the spec. The serve path reads this instead of re-parsing
+ *  the (potentially multi-MB) spec to rebuild the shared `definitions`. */
+export declare const defsBlobKey: (specHash: string) => string;
 export interface OpenapiStore {
     /** Replace all stored operations for an integration. */
     readonly putOperations: (integration: string, operations: readonly StoredOperation[]) => Effect.Effect<void, StorageFailure>;
+    /** Append operations without clearing existing ones. The caller is
+     *  responsible for `removeOperations` first when doing a full rebuild. Used
+     *  by the streaming compile path, which persists operations chunk by chunk so
+     *  a huge spec's bindings are never all materialized at once. */
+    readonly appendOperations: (integration: string, operations: readonly StoredOperation[]) => Effect.Effect<void, StorageFailure>;
     /** Look up one operation by integration + tool name. */
     readonly getOperation: (integration: string, toolName: string) => Effect.Effect<StoredOperation | null, StorageFailure>;
     /** List every stored operation for an integration. */
@@ -26,5 +38,12 @@ export interface OpenapiStore {
     readonly putSpec: (specHash: string, specText: string) => Effect.Effect<void, StorageFailure>;
     /** Load spec text by content hash; null when no blob exists. */
     readonly getSpec: (specHash: string) => Effect.Effect<string | null, StorageFailure>;
+    /** Persist the compiled `#/$defs/*` JSON for a spec under its content hash.
+     *  Content-addressed like the spec blob; lets the serve path serve the shared
+     *  `definitions` without re-parsing the spec. */
+    readonly putDefs: (specHash: string, defsJson: string) => Effect.Effect<void, StorageFailure>;
+    /** Load the compiled `#/$defs/*` JSON by content hash; null when no blob
+     *  exists (legacy rows added before the defs blob). */
+    readonly getDefs: (specHash: string) => Effect.Effect<string | null, StorageFailure>;
 }
 export declare const makeDefaultOpenapiStore: ({ pluginStorage, blobs }: StorageDeps) => OpenapiStore;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@executor-js/plugin-openapi",
-  "version": "1.5.16",
+  "version": "1.5.18",
   "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/openapi",
   "bugs": {
     "url": "https://github.com/RhysSullivan/executor/issues"
@@ -53,8 +53,8 @@
   },
   "dependencies": {
     "@effect/platform-node": "4.0.0-beta.59",
-    "@executor-js/config": "1.5.16",
-    "@executor-js/sdk": "1.5.16",
+    "@executor-js/config": "1.5.18",
+    "@executor-js/sdk": "1.5.18",
     "js-yaml": "4.1.1",
     "lucide-react": "^1.7.0",
     "openapi-types": "^12.1.3"