@superatomai/sdk-node 0.0.93-dsp → 0.0.94-dsp

package/dist/index.js

@@ -45,6 +45,8 @@ __export(index_exports, {
   MainAgent: () => MainAgent,
   MissingRequestContextError: () => MissingRequestContextError,
   NUMERIC_KEYS: () => NUMERIC_KEYS,
+  PARCEL_THRESHOLD_BYTES: () => PARCEL_THRESHOLD_BYTES,
+  ParcelFile: () => ParcelFile,
   STORAGE_CONFIG: () => STORAGE_CONFIG,
   ScriptMatcher: () => ScriptMatcher,
   ScriptStore: () => ScriptStore,
@@ -124,7 +126,7 @@ __export(index_exports, {
   writeShims: () => writeShims
 });
 module.exports = __toCommonJS(index_exports);
-var import_crypto7 = require("crypto");
+var import_crypto8 = require("crypto");
 
 // src/websocket.ts
 var import_ws = __toESM(require("ws"));
@@ -3027,6 +3029,7 @@ async function handleDataRequestInner(data, collections, sendMessage, tools) {
   let requestId;
   let collection;
   let op;
+  const fromId = typeof data?.from?.id === "string" ? data.from.id : void 0;
   try {
     const dataRequest = DataRequestMessageSchema.parse(data);
     const { id, payload } = dataRequest;
@@ -3037,13 +3040,13 @@ async function handleDataRequestInner(data, collections, sendMessage, tools) {
     if (!collections[collection]) {
       sendDataResponse(id, collection, op, null, {
         error: `Collection '${collection}' not found`
-      }, sendMessage);
+      }, sendMessage, fromId);
       return;
     }
     if (!collections[collection][op]) {
       sendDataResponse(id, collection, op, null, {
         error: `Operation '${op}' not found for collection '${collection}'`
-      }, sendMessage);
+      }, sendMessage, fromId);
       return;
     }
     const startTime = performance.now();
@@ -3115,21 +3118,22 @@ async function handleDataRequestInner(data, collections, sendMessage, tools) {
         logger.warn(`UIBlock ${uiBlockId} not found in threads`);
       }
     }
-    sendDataResponse(id, collection, op, result, { executionMs }, sendMessage);
+    sendDataResponse(id, collection, op, result, { executionMs }, sendMessage, fromId);
   } catch (error) {
     const errorMsg = error instanceof Error ? error.message : String(error);
     logger.error("Failed to handle data request:", error);
     if (requestId) {
       sendDataResponse(requestId, collection || "unknown", op || "unknown", null, {
         error: errorMsg
-      }, sendMessage);
+      }, sendMessage, fromId);
     }
   }
 }
-function sendDataResponse(id, collection, op, data, meta, sendMessage) {
+function sendDataResponse(id, collection, op, data, meta, sendMessage, fromId) {
   const response = {
     id,
     from: { type: "data-agent" },
+    to: fromId ? { id: fromId } : void 0,
     type: "DATA_RES",
     payload: {
       collection,
@@ -28361,6 +28365,355 @@ var ReportManager = class {
   }
 };
 
+// src/parcels/index.ts
+var import_crypto7 = require("crypto");
+var PARCEL_THRESHOLD_BYTES = 256 * 1024;
+var MAX_INLINE_BYTES = 900 * 1024;
+var MAX_PARCEL_BYTES = 95 * 1024 * 1024;
+var NEVER_PARCEL = /* @__PURE__ */ new Set(["PING", "PONG", "BUNDLE_CHUNK", "BUNDLE_CSS_CHUNK"]);
+var RETRY_DELAYS_MS = [500, 1500];
+var REQUEST_TIMEOUT_MS = 6e4;
+var MAX_DEPTH = 32;
+var PARCEL_FIELD = "__parcel";
+var FILE_FIELD = "__parcelFile";
+var isRef = (ref) => !!ref && typeof ref === "object" && typeof ref.projectId === "string" && typeof ref.hash === "string" && typeof ref.ext === "string" && typeof ref.size === "number";
+function parcelRefOf(payload) {
+  const ref = payload?.[PARCEL_FIELD];
+  return isRef(ref) ? ref : null;
+}
+var fileRefOf = (value) => {
+  const ref = value?.[FILE_FIELD];
+  return typeof ref?.contentType === "string" && isRef(ref) ? ref : null;
+};
+var sha256Hex = (bytes) => (0, import_crypto7.createHash)("sha256").update(bytes).digest("hex");
+var sleep = (ms) => new Promise((resolve4) => setTimeout(resolve4, ms));
+var ParcelHttpError = class extends Error {
+  constructor(message, status) {
+    super(message);
+    this.status = status;
+  }
+};
+var retryable = (err) => !(err instanceof ParcelHttpError) || err.status >= 500 || err.status === 408 || err.status === 429;
+async function withRetries(what, run) {
+  let lastError;
+  for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
+    try {
+      return await run();
+    } catch (err) {
+      lastError = err;
+      if (!retryable(err) || attempt === RETRY_DELAYS_MS.length) break;
+      logger.warn(`[parcels] ${what} failed (attempt ${attempt + 1}), retrying: ${err?.message ?? err}`);
+      await sleep(RETRY_DELAYS_MS[attempt] ?? 0);
+    }
+  }
+  throw lastError;
+}
+var ParcelClient = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.base = opts.apiUrl.replace(/\/+$/, "");
+    this.fetchImpl = opts.fetchImpl ?? fetch;
+    this.timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS;
+  }
+  url(ref) {
+    return `${this.base}/parcels/${encodeURIComponent(ref.projectId)}/${ref.hash}.${ref.ext}`;
+  }
+  async call(url, init) {
+    const res = await this.fetchImpl(url, {
+      ...init,
+      headers: { ...init.headers, Authorization: `Bearer ${this.opts.apiKey}` },
+      signal: AbortSignal.timeout(this.timeoutMs)
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new ParcelHttpError(`HTTP ${res.status}${text ? ` ${text.slice(0, 200)}` : ""}`, res.status);
+    }
+    return res;
+  }
+  /** Stores the bytes and returns the receipt for them. */
+  async upload(bytes, ext = "json", contentType = "application/json") {
+    if (bytes.byteLength > MAX_PARCEL_BYTES) {
+      throw new Error(`Parcel too large (${bytes.byteLength} bytes, limit ${MAX_PARCEL_BYTES})`);
+    }
+    const ref = { projectId: this.opts.projectId, hash: sha256Hex(bytes), ext, size: bytes.byteLength };
+    await withRetries(
+      "upload",
+      () => this.call(this.url(ref), { method: "PUT", headers: { "Content-Type": contentType }, body: bytes })
+    );
+    return ref;
+  }
+  /** The bytes behind a receipt, checked against its hash. */
+  async download(ref) {
+    return withRetries("download", async () => {
+      const res = await this.call(this.url(ref), { method: "GET" });
+      const bytes = new Uint8Array(await res.arrayBuffer());
+      if (sha256Hex(bytes) !== ref.hash) {
+        throw new ParcelHttpError("Parcel content does not match its hash", 502);
+      }
+      return bytes;
+    });
+  }
+};
+var ParcelFile = class {
+  constructor(ref, client) {
+    this.ref = ref;
+    this.client = client;
+    this.name = ref.name;
+    this.size = ref.size;
+    this.contentType = ref.contentType;
+  }
+  /** The file's bytes, checked against their hash. Throws if they cannot be fetched. */
+  async download() {
+    if (!this.client) throw new Error("Cannot download files: SA_API_URL or the API key is not set");
+    return this.client.download(this.ref);
+  }
+  toJSON() {
+    return { [FILE_FIELD]: this.ref };
+  }
+};
+var isBinary = (v) => v instanceof Uint8Array || v instanceof ArrayBuffer;
+var isPlain = (v) => {
+  if (!v || typeof v !== "object") return false;
+  const proto = Object.getPrototypeOf(v);
+  return proto === Object.prototype || proto === null;
+};
+function hasBinary(value, depth = 0) {
+  if (isBinary(value)) return true;
+  if (depth >= MAX_DEPTH) return false;
+  if (Array.isArray(value)) return value.some((v) => hasBinary(v, depth + 1));
+  if (isPlain(value)) return Object.values(value).some((v) => hasBinary(v, depth + 1));
+  return false;
+}
+function hasFileRef(value, depth = 0) {
+  if (fileRefOf(value)) return true;
+  if (depth >= MAX_DEPTH) return false;
+  if (Array.isArray(value)) return value.some((v) => hasFileRef(v, depth + 1));
+  if (isPlain(value)) return Object.values(value).some((v) => hasFileRef(v, depth + 1));
+  return false;
+}
+var extOf = (name) => typeof name === "string" && /\.([a-z0-9]{1,10})$/i.exec(name)?.[1]?.toLowerCase() || "bin";
+async function uploadFiles(value, client, parent, depth = 0) {
+  if (isBinary(value)) {
+    const bytes = value instanceof Uint8Array ? value : new Uint8Array(value);
+    const name = parent?.name ?? parent?.fileName;
+    const contentType = String(parent?.contentType ?? parent?.mimeType ?? "application/octet-stream");
+    const ref = await client.upload(bytes, extOf(name), contentType);
+    const fileRef = { ...ref, contentType, ...typeof name === "string" ? { name } : {} };
+    return { [FILE_FIELD]: fileRef };
+  }
+  if (depth >= MAX_DEPTH) return value;
+  if (Array.isArray(value)) {
+    const out = [];
+    for (const v of value) out.push(await uploadFiles(v, client, void 0, depth + 1));
+    return out;
+  }
+  if (isPlain(value)) {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = await uploadFiles(v, client, value, depth + 1);
+    return out;
+  }
+  return value;
+}
+function toFileHandles(value, client, depth = 0) {
+  const ref = fileRefOf(value);
+  if (ref) return new ParcelFile(ref, client);
+  if (depth >= MAX_DEPTH) return value;
+  if (Array.isArray(value)) return value.map((v) => toFileHandles(v, client, depth + 1));
+  if (isPlain(value)) {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = toFileHandles(v, client, depth + 1);
+    return out;
+  }
+  return value;
+}
+var asError = (message, reason) => ({
+  ...message,
+  payload: { error: reason, parcelError: true }
+});
+var recipientOf = (message) => {
+  const id = message.to?.id;
+  return typeof id === "string" ? id : "";
+};
+var InOrder = class {
+  /** `onIdle` runs each time the last waiting task finishes. */
+  constructor(onIdle) {
+    this.onIdle = onIdle;
+    this.pending = 0;
+    this.chain = Promise.resolve();
+  }
+  get busy() {
+    return this.pending > 0;
+  }
+  run(task) {
+    this.pending++;
+    this.chain = this.chain.then(task).catch((err) => logger.error("[parcels] queue fault:", err)).finally(() => {
+      if (--this.pending === 0) this.onIdle?.();
+    });
+  }
+  idle() {
+    return this.chain;
+  }
+};
+var OutgoingParcels = class {
+  constructor(client, jsonParcels, deliver, threshold = PARCEL_THRESHOLD_BYTES) {
+    this.client = client;
+    this.jsonParcels = jsonParcels;
+    this.deliver = deliver;
+    this.threshold = threshold;
+    /** Only recipients with something waiting; a queue is dropped once it empties. */
+    this.queues = /* @__PURE__ */ new Map();
+    this.warnedNoClient = false;
+  }
+  /**
+   * Same contract as the SDK's `send`: never throws, returns whether the message
+   * went out (or, for a queued one, was accepted).
+   */
+  send(message) {
+    if (NEVER_PARCEL.has(message.type)) return this.now(message);
+    let files = false;
+    try {
+      files = hasBinary(message.payload);
+    } catch {
+    }
+    if (files && !this.client) {
+      if (!this.warnedNoClient) {
+        this.warnedNoClient = true;
+        logger.warn("[parcels] a message carries a file but SA_API_URL or the API key is missing \u2014 sent as-is");
+      }
+      files = false;
+    }
+    if (!files) {
+      if (!this.jsonParcels || !this.client) return this.now(message);
+      let wire;
+      try {
+        wire = JSON.stringify(message);
+      } catch {
+        return this.now(message);
+      }
+      if (Buffer.byteLength(wire, "utf8") < this.threshold) return this.now(message, wire);
+    }
+    this.queueFor(message).run(async () => {
+      const out = await this.prepare(message, files);
+      this.deliver(out.message, out.wire);
+    });
+    return true;
+  }
+  /** Delivers now, unless something for the same recipient is already waiting — then behind it. */
+  now(message, wire) {
+    if (!this.queues.get(recipientOf(message))?.busy) return this.deliver(message, wire);
+    this.queueFor(message).run(async () => {
+      this.deliver(message, wire);
+    });
+    return true;
+  }
+  queueFor(message) {
+    const key = recipientOf(message);
+    let queue = this.queues.get(key);
+    if (!queue) {
+      queue = new InOrder(() => this.queues.delete(key));
+      this.queues.set(key, queue);
+    }
+    return queue;
+  }
+  /** Resolves when everything queued so far has been handed to the socket. For tests and shutdown. */
+  async idle() {
+    await Promise.all(Array.from(this.queues.values(), (queue) => queue.idle()));
+  }
+  async prepare(message, files) {
+    let current = message;
+    if (files) {
+      const started2 = Date.now();
+      try {
+        current = { ...message, payload: await uploadFiles(message.payload, this.client) };
+        logger.info(`[parcels] ${message.type} ${message.id ?? ""}: file(s) uploaded (${Date.now() - started2}ms)`);
+      } catch (err) {
+        const reason = err?.message ?? String(err);
+        logger.error(`[parcels] file upload failed for ${message.type}, sending an error instead: ${reason}`);
+        return { message: asError(message, `Could not send a file: ${reason}`) };
+      }
+    }
+    let wire;
+    try {
+      wire = JSON.stringify(current);
+    } catch {
+      return { message: current };
+    }
+    const size = Buffer.byteLength(wire, "utf8");
+    if (!this.jsonParcels || size < this.threshold) return { message: current, wire };
+    const started = Date.now();
+    try {
+      const bytes = Buffer.from(JSON.stringify(current.payload ?? null), "utf8");
+      const ref = await this.client.upload(bytes);
+      logger.info(
+        `[parcels] ${message.type} ${message.id ?? ""} \u2192 parcel ${ref.hash.slice(0, 12)} (${ref.size} bytes, ${Date.now() - started}ms)`
+      );
+      return { message: { ...current, payload: { [PARCEL_FIELD]: ref } } };
+    } catch (err) {
+      const reason = err?.message ?? String(err);
+      if (size <= MAX_INLINE_BYTES) {
+        logger.warn(`[parcels] upload failed for ${message.type}, sending inline (${size} bytes): ${reason}`);
+        return { message: current, wire };
+      }
+      logger.error(`[parcels] upload failed for ${message.type} (${size} bytes), sending an error instead: ${reason}`);
+      return { message: asError(message, `The message was too large to deliver (${size} bytes): ${reason}`) };
+    }
+  }
+};
+var IncomingParcels = class {
+  constructor(client, dispatch) {
+    this.client = client;
+    this.dispatch = dispatch;
+    this.queue = new InOrder();
+  }
+  /**
+   * `raw` is the message's text when it is at hand: a message that never
+   * mentions a file placeholder is not searched for one.
+   */
+  receive(message, raw) {
+    const ref = parcelRefOf(message?.payload);
+    if (!ref) {
+      const withFiles = raw === void 0 || raw.includes(FILE_FIELD) ? this.withFiles(message) : message;
+      if (this.queue.busy) {
+        this.queue.run(async () => this.dispatch(withFiles));
+      } else {
+        this.dispatch(withFiles);
+      }
+      return;
+    }
+    this.queue.run(async () => this.dispatch(await this.restore(message, ref)));
+  }
+  idle() {
+    return this.queue.idle();
+  }
+  withFiles(message) {
+    try {
+      return hasFileRef(message.payload) ? { ...message, payload: toFileHandles(message.payload, this.client) } : message;
+    } catch {
+      return message;
+    }
+  }
+  async restore(message, ref) {
+    if (!this.client) {
+      logger.error(`[parcels] received a parcel for ${message.type} but SA_API_URL or the API key is not set`);
+      return asError(message, "This engine cannot download parcels: SA_API_URL or the API key is not set.");
+    }
+    const started = Date.now();
+    try {
+      const bytes = await this.client.download(ref);
+      const payload = JSON.parse(Buffer.from(bytes).toString("utf8"));
+      logger.info(
+        `[parcels] ${message.type} ${message.id ?? ""} \u2190 parcel ${ref.hash.slice(0, 12)} (${ref.size} bytes, ${Date.now() - started}ms)`
+      );
+      return this.withFiles({ ...message, payload });
+    } catch (err) {
+      const reason = err?.message ?? String(err);
+      logger.error(`[parcels] download failed for ${message.type} (${ref.hash.slice(0, 12)}): ${reason}`);
+      return asError(message, `Could not download the message content: ${reason}`);
+    }
+  }
+};
+var parcelsFlagFromEnv = () => !/^(off|false|0|no)$/i.test((process.env.SA_PARCELS ?? "").trim());
+
 // src/userResponse/scripts/fmt.ts
 var DASH = "\u2014";
 var DEFAULT_FMT_CONFIG = { locale: "en-IN", currency: "INR" };
@@ -29712,6 +30065,18 @@ var SuperatomSDK = class {
     this.type = config2.type || "data-agent";
     this.bundleDir = config2.bundleDir;
     this.url = config2.url || process.env.SA_WEBSOCKET_URL || DEFAULT_WS_URL;
+    const apiUrl = config2.apiUrl || process.env.SA_API_URL;
+    this.parcelClient = apiUrl && this.apiKey ? new ParcelClient({ apiUrl, projectId: this.projectId, apiKey: this.apiKey }) : null;
+    const jsonParcels = config2.parcels ?? parcelsFlagFromEnv();
+    if (jsonParcels && !this.parcelClient) {
+      logger.warn("[parcels] parcels are on but SA_API_URL or the API key is missing \u2014 large messages go inline");
+    }
+    this.outgoing = new OutgoingParcels(
+      this.parcelClient,
+      jsonParcels,
+      (message, wire) => this.deliver(message, wire)
+    );
+    this.incoming = new IncomingParcels(this.parcelClient, (parsed) => this.dispatchMessage(parsed));
     this.anthropicApiKey = config2.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || "";
     this.groqApiKey = config2.GROQ_API_KEY || process.env.GROQ_API_KEY || "";
     this.geminiApiKey = config2.GEMINI_API_KEY || process.env.GEMINI_API_KEY || "";
@@ -29863,11 +30228,24 @@ var SuperatomSDK = class {
     });
   }
   /**
-   * Handle incoming WebSocket messages
+   * Handle incoming WebSocket messages.
+   *
+   * Parsed here, then handed to the parcel layer, which restores a parcel's
+   * payload (and holds later messages back until it has) before routing.
    */
   handleMessage(data) {
+    let parsed;
+    try {
+      parsed = JSON.parse(data);
+    } catch (error) {
+      logger.error("Failed to parse incoming message:", error);
+      return;
+    }
+    this.incoming.receive(parsed, data);
+  }
+  /** Routes one parsed message — everything `handleMessage` did after parsing. */
+  dispatchMessage(parsed) {
     try {
-      const parsed = JSON.parse(data);
       const message = IncomingMessageSchema.parse(parsed);
       logger.debug("Received message:", message.type);
       switch (message.type) {
@@ -30057,8 +30435,19 @@ var SuperatomSDK = class {
    * Send a message to the Superatom service.
    * Returns true if the message was sent, false if the WebSocket is not connected.
    * Does NOT throw on closed connections — callers can check the return value if needed.
+   *
+   * Large payloads and files go as parcels (see ./parcels): a file is any
+   * Buffer / Uint8Array / ArrayBuffer inside the payload, with an optional
+   * `name` and `contentType` beside it — e.g. `payload: { file: { name, bytes } }`.
+   * Such a message goes out once its upload is done (true = accepted), and later
+   * messages keep their order. A received file arrives as a `ParcelFile` handle
+   * whose `download()` fetches the bytes.
    */
   send(message) {
+    return this.outgoing.send(message);
+  }
+  /** Puts one message on the socket, or in the reconnect outbox. `wire` is its JSON, when already made. */
+  deliver(message, wire) {
     if (!this.ws || !this.connected) {
       logger.warn("WebSocket is not connected, queueing message for reconnect:", message.type);
       this.queuePendingMessage(message);
@@ -30070,7 +30459,7 @@ var SuperatomSDK = class {
       return false;
     }
     try {
-      const payload = JSON.stringify(message);
+      const payload = wire ?? JSON.stringify(message);
       this.ws.send(payload);
       return true;
     } catch (error) {
@@ -30323,7 +30712,7 @@ var SuperatomSDK = class {
       waiters.push(finish);
       this.registryWaiters.set(resType, waiters);
       const sent = this.send({
-        id: `${reqType.toLowerCase()}_${(0, import_crypto7.randomUUID)()}`,
+        id: `${reqType.toLowerCase()}_${(0, import_crypto8.randomUUID)()}`,
         type: reqType,
         from: { type: this.type },
         to: { type: "runtime" },
@@ -30414,6 +30803,8 @@ var SuperatomSDK = class {
   MainAgent,
   MissingRequestContextError,
   NUMERIC_KEYS,
+  PARCEL_THRESHOLD_BYTES,
+  ParcelFile,
   STORAGE_CONFIG,
   ScriptMatcher,
   ScriptStore,