@requence/service 1.0.0-alpha.21 → 1.0.0-alpha.23

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @requence/service
 
+## 1.0.0-alpha.23
+
+### Patch Changes
+
+- 13c28db: backchannel
+
+## 1.0.0-alpha.22
+
+### Patch Changes
+
+- 7cbcfe5: continuous mode improvement
+
 ## 1.0.0-alpha.21
 
 ### Patch Changes

package/build/index.js

@@ -88,9 +88,15 @@ function mapData(data, visitors) {
       if (typeof data2 === "symbol") {
         return notUndefined(visitors.Symbol?.(data2, path), data2);
       }
+      if (typeof data2 === "function") {
+        return notUndefined(visitors.Function?.(data2, path), data2);
+      }
       if (isRecord(data2)) {
         return resolve(visitors.Object?.(structuredClone(data2), path), (resolvedData) => {
           const nextData = resolvedData ?? data2;
+          if (!isRecord(nextData)) {
+            return nextData;
+          }
           const entries = Object.entries(nextData).map(([key, value]) => [
             key,
             traverseData(value, [...path, key])
@@ -119,6 +125,34 @@ mapData.abort = () => {
   throw new AbortMapData;
 };
 
+// ../helpers/src/utils/requenceCallback.ts
+import { z as z2 } from "zod/v4";
+var callbackObjectSchema = z2.object({
+  sourceId: z2.string(),
+  outputName: z2.string().nullable(),
+  propertyPath: z2.array(z2.union([z2.string(), z2.int()])),
+  callbackId: z2.uuid(),
+  __REQUENCE_type: z2.literal("RequenceCallback")
+});
+function isCallbackObject(data) {
+  return callbackObjectSchema.safeParse(data).success;
+}
+function createCallback(callback) {
+  let resolvePromise;
+  const promise = new Promise((resolve2) => {
+    resolvePromise = resolve2;
+  });
+  const executor = (arg) => {
+    resolvePromise(arg);
+    callback?.(arg);
+  };
+  return Object.assign(executor, {
+    response() {
+      return promise;
+    }
+  });
+}
+
 // ../helpers/src/files/mapOutput.ts
 async function upload(uploadUrl, body, signal, contentType) {
   const uploadResponse = await fetch(uploadUrl, {
@@ -346,11 +380,11 @@ class RequenceStream {
 }
 
 // ../helpers/src/files/streamObject.ts
-import { z as z2 } from "zod/v4";
-var streamObjectSchema = z2.object({
-  mime: z2.string(),
-  url: z2.string().url(),
-  __REQUENCE_type: z2.literal("RequenceStream")
+import { z as z3 } from "zod/v4";
+var streamObjectSchema = z3.object({
+  mime: z3.string(),
+  url: z3.string().url(),
+  __REQUENCE_type: z3.literal("RequenceStream")
 });
 function isStreamObject(data) {
   return streamObjectSchema.safeParse(data).success;
@@ -363,12 +397,12 @@ function toStreamObject(streamInfo) {
 }
 
 // ../helpers/src/files/fileObject.ts
-import { z as z3 } from "zod/v4";
-var fileObjectSchema = z3.object({
-  size: z3.number().int().positive(),
-  mime: z3.string(),
-  url: z3.string().url(),
-  __REQUENCE_type: z3.literal("RequenceFile")
+import { z as z4 } from "zod/v4";
+var fileObjectSchema = z4.object({
+  size: z4.int().positive(),
+  mime: z4.string(),
+  url: z4.url(),
+  __REQUENCE_type: z4.literal("RequenceFile")
 });
 function toFileObject(fileInfo) {
   return {
@@ -381,7 +415,7 @@ function isFileObject(data) {
 }
 
 // ../helpers/src/utils/resolveRequenceTypes.ts
-function resolveRequenceTypes(data) {
+function resolveRequenceTypes(data, onCallback) {
   return mapData(data, {
     Object(obj) {
       if (isFileObject(obj)) {
@@ -390,18 +424,23 @@ function resolveRequenceTypes(data) {
       if (isStreamObject(obj)) {
         return RequenceStream.fromStreamObject(obj);
       }
+      if (isCallbackObject(obj)) {
+        return (arg) => {
+          onCallback({ ...obj, arg });
+        };
+      }
     }
   });
 }
 
 // src/index.ts
-import { z as z5 } from "zod/v4";
+import { z as z6 } from "zod/v4";
 
 // src/createAmqpConnection.ts
 import {
   connect
 } from "amqp-connection-manager";
-import { z as z4 } from "zod/v4";
+import { z as z5 } from "zod/v4";
 
 // src/helpers.ts
 class RetryError extends Error {
@@ -421,54 +460,131 @@ var OUTPUT_VALUE = Symbol("output_value");
 function isOutputValue(value) {
   return value !== null && typeof value === "object" && OUTPUT_VALUE in value;
 }
-async function unwrapResult(result, options, onDone) {
-  if (isOutputValue(result)) {
+function processData(params) {
+  return mapOutput({
+    value: mapData(params.data, {
+      Function(data, path) {
+        return {
+          ...params.onCallback(data),
+          outputName: params.outputName,
+          propertyPath: path,
+          __REQUENCE_type: "RequenceCallback"
+        };
+      }
+    }),
+    options: params.options,
+    onDone: params.onDone
+  });
+}
+async function unwrapResult(params) {
+  if (isOutputValue(params.result)) {
     return {
-      value: await mapOutput({ value: result.value, options, onDone }),
-      outputName: result[OUTPUT_VALUE]
+      value: await processData({
+        outputName: params.result[OUTPUT_VALUE],
+        data: params.result.value,
+        options: params.options,
+        onDone: params.onDone,
+        onCallback: params.onCallback
+      }),
+      outputName: params.result[OUTPUT_VALUE]
     };
   }
   return {
-    value: await mapOutput({ value: result, options, onDone }),
+    value: await processData({
+      outputName: null,
+      data: params.result,
+      options: params.options,
+      onDone: params.onDone,
+      onCallback: params.onCallback
+    }),
     outputName: null
   };
 }
+function isAsyncIterable(iterable) {
+  return typeof iterable[Symbol.asyncIterator] === "function";
+}
 function createContextHelper(options) {
-  return {
-    taskId: options.taskId,
-    input: resolveRequenceTypes(options.input),
-    configuration: resolveRequenceTypes(options.configuration),
-    toOutput(name, value) {
-      return {
-        [OUTPUT_VALUE]: name,
-        value
-      };
-    },
-    defer: options.onDefer,
-    debug: {
-      log(...data) {
-        options.onLog("log", data);
+  const terminatePromise = createRemotePromise();
+  return [
+    {
+      taskId: options.taskId,
+      input: resolveRequenceTypes(options.input, options.onCallback),
+      configuration: resolveRequenceTypes(options.configuration, () => {
+        throw new Error("configuration may not have callbacks");
+      }),
+      toOutput(name, value) {
+        return {
+          [OUTPUT_VALUE]: name,
+          value
+        };
+      },
+      defer: options.onDefer,
+      debug: {
+        log(...data) {
+          options.onLog("log", data);
+        },
+        info(...data) {
+          options.onLog("info", data);
+        },
+        warn(...data) {
+          options.onLog("warn", data);
+        },
+        error(...data) {
+          options.onLog("error", data);
+        }
       },
-      info(...data) {
-        options.onLog("info", data);
+      retry(delay) {
+        throw new RetryError(delay);
       },
-      warn(...data) {
-        options.onLog("warn", data);
+      abort(reason) {
+        throw new AbortError(reason);
       },
-      error(...data) {
-        options.onLog("error", data);
+      skip() {
+        throw new SkipError;
+      },
+      terminated: terminatePromise.promise,
+      wrapIterable: function(iterable) {
+        if (isAsyncIterable(iterable)) {
+          return {
+            [Symbol.asyncIterator]() {
+              const iterator = iterable[Symbol.asyncIterator]();
+              return {
+                async next() {
+                  const result = await Promise.race([
+                    terminatePromise.promise,
+                    iterator.next()
+                  ]);
+                  if (typeof result === "string") {
+                    return { done: true, value: undefined };
+                  }
+                  return result;
+                }
+              };
+            }
+          };
+        }
+        return {
+          [Symbol.iterator]() {
+            const iterator = iterable[Symbol.iterator]();
+            let terminated = false;
+            terminatePromise.then(() => {
+              terminated = true;
+            });
+            return {
+              next() {
+                if (terminated) {
+                  return { done: true, value: undefined };
+                }
+                const result = iterator.next();
+                return result;
+              }
+            };
+          }
+        };
       }
     },
-    retry(delay) {
-      throw new RetryError(delay);
-    },
-    abort(reason) {
-      throw new AbortError(reason);
-    },
-    skip() {
-      throw new SkipError;
-    }
-  };
+    terminatePromise
+  ];
 }
 
 // src/mapInput.ts
@@ -486,33 +602,50 @@ function mapInput(data) {
 }
 
 // src/createAmqpConnection.ts
-var messageKeySchema = z4.string().transform((str) => deobfuscate(str)).pipe(z4.tuple([z4.uuid(), z4.uuid()]).rest(z4.string())).transform((arr) => ({
+var messageKeySchema = z5.string().transform((str) => deobfuscate(str)).pipe(z5.tuple([z5.uuid(), z5.uuid()]).rest(z5.string())).transform((arr) => ({
   taskId: arr[0],
   messageId: arr[1],
   uploadUrl: arr[2] || undefined,
   streamUrl: arr[3] || undefined
 }));
-var headerSchema = z4.object({
-  properties: z4.object({
-    correlationId: z4.uuid(),
-    headers: z4.object({
-      version: z4.string(),
-      taskId: z4.uuid()
+var headerSchema = z5.object({
+  properties: z5.object({
+    correlationId: z5.uuid(),
+    headers: z5.object({
+      taskId: z5.uuid()
     })
   })
 }).transform((msg) => ({
-  version: msg.properties.headers.version,
   messageId: msg.properties.correlationId,
   taskId: msg.properties.headers.taskId
 }));
-var payloadSchema = z4.object({ content: z4.instanceof(Buffer) }).transform((msg) => JSON.parse(msg.content.toString())).pipe(z4.object({
-  input: z4.unknown(),
-  configuration: z4.unknown(),
-  options: z4.object({
-    uploadUrl: z4.url().optional(),
-    streamUrl: z4.url().optional()
+var payloadSchema = z5.object({ content: z5.instanceof(Buffer) }).transform((msg) => JSON.parse(msg.content.toString())).pipe(z5.object({
+  input: z5.unknown(),
+  configuration: z5.unknown(),
+  options: z5.object({
+    uploadUrl: z5.url().optional(),
+    streamUrl: z5.url().optional()
   })
 }));
+var backchannelMessageSchema = z5.object({
+  content: z5.instanceof(Buffer).transform((content) => JSON.parse(content.toString())).pipe(z5.discriminatedUnion("type", [
+    z5.object({
+      type: z5.literal("stop")
+    }),
+    z5.object({
+      type: z5.literal("callback"),
+      callbackId: z5.uuid(),
+      arg: z5.any()
+    })
+  ])),
+  properties: z5.object({
+    correlationId: z5.string()
+  })
+}).transform((message) => ({
+  payload: message.content,
+  messageId: message.properties.correlationId
+}));
+var abortSymbol = Symbol();
 function isMessageHandlerGenerator(handler) {
   return handler.constructor.name === "GeneratorFunction" || handler.constructor.name === "AsyncGeneratorFunction";
 }
@@ -548,11 +681,17 @@ function createConnection({
     connection.addListener("connect", handleConnect);
     connection.addListener("connectFailed", handleConnectFailed);
   });
+  const connectionId = crypto.randomUUID();
+  const backchannelQueue = `${queue}-backchannel-${connectionId}`;
   const channel = connection.createChannel({
     async setup(channel2) {
       await channel2.checkExchange(exchange);
       await channel2.checkExchange(`${exchange}-retry`);
       await channel2.checkQueue(queue);
+      await channel2.assertQueue(backchannelQueue, {
+        exclusive: true
+      });
+      await channel2.bindQueue(backchannelQueue, "service", backchannelQueue);
       await channel2.prefetch(prefetch);
     }
   });
@@ -570,13 +709,36 @@ function createConnection({
       priority,
       headers: {
         outputName: outputName ?? null,
+        connectionId,
         ...headers
       }
     });
   };
+  const generatorTerminators = new Map;
+  const callbacks = new Map;
+  channel.consume(backchannelQueue, async (message) => {
+    const result = backchannelMessageSchema.safeParse(message);
+    if (!result.success) {
+      return;
+    }
+    const backchannelMessage = result.data;
+    switch (backchannelMessage.payload.type) {
+      case "stop": {
+        generatorTerminators.get(backchannelMessage.messageId)?.("task stopped");
+        break;
+      }
+      case "callback": {
+        callbacks.get(backchannelMessage.payload.callbackId)?.(backchannelMessage.payload.arg);
+        break;
+      }
+    }
+  }, {
+    noAck: true
+  });
   channel.consume(queue, async (message) => {
     let publishIndex = 0;
     const donePromises = [];
+    const registeredCallbackIds = new Set;
     let removeActiveMessage = () => {};
     let acked = false;
     const ack = () => {
@@ -585,14 +747,14 @@ function createConnection({
         channel.ack(message);
       }
     };
+    const { messageId, taskId } = headerSchema.parse(message);
     try {
       let deferred = false;
       let deferredReason = null;
-      const { messageId, taskId } = headerSchema.parse(message);
       activeMessages.add(messageId);
       removeActiveMessage = () => activeMessages.delete(messageId);
       const payload = payloadSchema.parse(message);
-      const extendedContext = createContextHelper({
+      const [extendedContext, terminatePromise] = createContextHelper({
         taskId,
         input: mapInput(payload.input),
         configuration: mapInput(payload.configuration),
@@ -612,12 +774,45 @@ function createConnection({
           deferredReason = reason || null;
           ack();
           return obfuscate(taskId, messageId, payload.options.uploadUrl ?? "", payload.options.streamUrl ?? "");
+        },
+        onCallback(params) {
+          publish({
+            index: 0,
+            taskId,
+            messageId,
+            data: params.arg,
+            outputName: params.outputName,
+            connectionId: params.sourceId,
+            callbackId: params.callbackId,
+            propertyPath: params.propertyPath
+          });
         }
       });
+      publish({
+        index: publishIndex++,
+        messageId,
+        taskId,
+        priority: message.properties.priority,
+        begin: true
+      });
       if (isMessageHandlerGenerator(messageHandler)) {
         const generator = messageHandler(extendedContext);
+        const terminationPromise = createRemotePromise();
+        generatorTerminators.set(messageId, (reason) => {
+          generator.return();
+          terminationPromise.resolve(abortSymbol);
+          terminatePromise.resolve(reason);
+          ack();
+        });
         while (true) {
-          const { value: result, done } = await generator.next();
+          const next = await Promise.race([
+            terminationPromise,
+            generator.next()
+          ]);
+          if (next === abortSymbol) {
+            break;
+          }
+          const { value: result, done } = next;
           ack();
           if (deferred) {
             deferred = false;
@@ -625,7 +820,20 @@ function createConnection({
           }
           const donePromise = createRemotePromise();
           donePromises.push(donePromise);
-          const { value, outputName } = await unwrapResult(result, payload.options, donePromise.resolve);
+          const { value, outputName } = await unwrapResult({
+            result,
+            options: payload.options,
+            onDone: donePromise.resolve,
+            onCallback: (callback) => {
+              const callbackId = crypto.randomUUID();
+              registeredCallbackIds.add(callbackId);
+              callbacks.set(callbackId, callback);
+              return {
+                sourceId: connectionId,
+                callbackId
+              };
+            }
+          });
           if (!done || typeof value !== "undefined") {
             publish({
               index: publishIndex++,
@@ -645,7 +853,14 @@ function createConnection({
         ack();
         const donePromise = createRemotePromise();
         donePromises.push(donePromise);
-        const { value, outputName } = await unwrapResult(result, payload.options, donePromise.resolve);
+        const { value, outputName } = await unwrapResult({
+          result,
+          options: payload.options,
+          onDone: donePromise.resolve,
+          onCallback: () => {
+            throw new Error("callbacks can only be used in generator functions");
+          }
+        });
         if (deferred && value) {
           console.warn("returning a value in a deferred message resets the deferred status");
           deferred = false;
@@ -695,9 +910,10 @@ function createConnection({
             ...message.properties,
             expiration: error.delay || 1000
           };
-          channel.publish(`${exchange}-retry`, "retry", message.content, properties);
+          channel.publish(`${exchange}-retry`, version, message.content, properties);
           ack();
         }
+        return;
       }
       if (error instanceof AbortError && error.message.length > 0) {
         if (assertNack("data already send, cannot abort")) {
@@ -706,6 +922,7 @@ function createConnection({
             contentType: "text/plain",
             headers: {
               ...message.properties.headers,
+              index: publishIndex++,
               abort: true
             }
           });
@@ -723,7 +940,9 @@ function createConnection({
             ...message.properties,
             headers: {
               ...message.properties.headers,
+              version,
               exception: true,
+              index: publishIndex++,
               "error-message": error.message || "error in service handler"
             }
           });
@@ -735,6 +954,8 @@ function createConnection({
         channel.nack(message, false, false);
       }
     } finally {
+      generatorTerminators.delete(messageId);
+      registeredCallbackIds.forEach((id) => callbacks.delete(id));
       removeActiveMessage();
     }
   });
@@ -752,6 +973,7 @@ function createConnection({
           throw new Error("Cannot send deferred message while callback handler is still running");
         }
       };
+      const registeredCallbackIds = new Set;
       const donePromises = [];
       await actor({
         async sendToOutput(outputName, data) {
@@ -762,10 +984,20 @@ function createConnection({
             index: publishIndex++,
             messageId,
             taskId,
-            data: await mapOutput({
-              value: data,
+            data: await processData({
+              outputName,
+              data,
               options,
-              onDone: donePromise.resolve
+              onDone: donePromise.resolve,
+              onCallback: (callback) => {
+                const callbackId = crypto.randomUUID();
+                registeredCallbackIds.add(callbackId);
+                callbacks.set(callbackId, callback);
+                return {
+                  sourceId: connectionId,
+                  callbackId
+                };
+              }
             }),
             outputName
           });
@@ -778,10 +1010,20 @@ function createConnection({
             index: publishIndex++,
             messageId,
             taskId,
-            data: await mapOutput({
-              value: data,
+            data: await processData({
+              outputName: null,
+              data,
               options,
-              onDone: donePromise.resolve
+              onDone: donePromise.resolve,
+              onCallback: (callback) => {
+                const callbackId = crypto.randomUUID();
+                registeredCallbackIds.add(callbackId);
+                callbacks.set(callbackId, callback);
+                return {
+                  sourceId: connectionId,
+                  callbackId
+                };
+              }
             })
           });
         },
@@ -795,6 +1037,7 @@ function createConnection({
             contentType: "text/plain",
             headers: {
               taskId,
+              index: publishIndex++,
               abort: true
             }
           });
@@ -802,6 +1045,7 @@ function createConnection({
       });
       setImmediate(async () => {
         await Promise.allSettled(donePromises);
+        registeredCallbackIds.forEach((id) => callbacks.delete(id));
         publish({
           index: publishIndex++,
           messageId,
@@ -815,16 +1059,16 @@ function createConnection({
 }
 
 // src/index.ts
-var connectionOptionsSchema = z5.object({
-  url: z5.union([
-    z5.string().regex(/^amqps?:\/\/(.+):(.+)@(.+)$/).transform((str) => new URL(str)),
-    z5.instanceof(URL).refine((url) => url.protocol.match(/^amqps?:/), "Wrong protocol")
+var connectionOptionsSchema = z6.object({
+  url: z6.union([
+    z6.string().regex(/^amqps?:\/\/(.+):(.+)@(.+)$/).transform((str) => new URL(str)),
+    z6.instanceof(URL).refine((url) => url.protocol.match(/^amqps?:/), "Wrong protocol")
   ]),
-  version: z5.string(),
-  silent: z5.boolean().optional(),
-  prefetch: z5.number().int().optional(),
-  connectionTimeout: z5.number().positive().default(3000),
-  connectionOptions: z5.any()
+  version: z6.string(),
+  silent: z6.boolean().optional(),
+  prefetch: z6.number().int().optional(),
+  connectionTimeout: z6.number().positive().default(3000),
+  connectionOptions: z6.any()
 });
 function createService(optionsOrVersion, optionalMessageHandler) {
   let options;
@@ -858,9 +1102,10 @@ function createService(optionsOrVersion, optionalMessageHandler) {
 }
 export {
   createService,
+  createCallback,
   RequenceStream,
   RequenceFile
 };
 
-//# debugId=62B89EF7DDD1ECAA64756E2164756E21
+//# debugId=DB60EAB4653D0F1964756E2164756E21
 //# sourceMappingURL=index.js.map