@graphrefly/graphrefly 0.4.0 → 0.5.0

package/dist/extra/index.cjs

@@ -34,6 +34,8 @@ __export(extra_exports, {
   cached: () => cached,
   catchError: () => catchError,
   checkpointNodeValue: () => checkpointNodeValue,
+  checkpointToRedis: () => checkpointToRedis,
+  checkpointToS3: () => checkpointToS3,
   circuitBreaker: () => circuitBreaker,
   combine: () => combine,
   combineLatest: () => combineLatest,
@@ -75,10 +77,13 @@ __export(extra_exports, {
   fromIter: () => fromIter,
   fromKafka: () => fromKafka,
   fromMCP: () => fromMCP,
+  fromNATS: () => fromNATS,
   fromNDJSON: () => fromNDJSON,
   fromOTel: () => fromOTel,
   fromPrometheus: () => fromPrometheus,
   fromPromise: () => fromPromise,
+  fromPulsar: () => fromPulsar,
+  fromRabbitMQ: () => fromRabbitMQ,
   fromRedisStream: () => fromRedisStream,
   fromStatsD: () => fromStatsD,
   fromSyslog: () => fromSyslog,
@@ -142,11 +147,22 @@ __export(extra_exports, {
   throwError: () => throwError,
   timeout: () => timeout,
   toArray: () => toArray,
+  toCSV: () => toCSV,
+  toClickHouse: () => toClickHouse,
+  toFile: () => toFile,
   toKafka: () => toKafka,
+  toLoki: () => toLoki,
   toMessages$: () => toMessages$,
+  toMongo: () => toMongo,
+  toNATS: () => toNATS,
   toObservable: () => toObservable,
+  toPostgres: () => toPostgres,
+  toPulsar: () => toPulsar,
+  toRabbitMQ: () => toRabbitMQ,
   toRedisStream: () => toRedisStream,
+  toS3: () => toS3,
   toSSE: () => toSSE,
+  toTempo: () => toTempo,
   toWebSocket: () => toWebSocket,
   tokenBucket: () => tokenBucket,
   tokenTracker: () => tokenTracker,
@@ -3114,6 +3130,797 @@ function fromClickHouseWatch(client, query, opts) {
     };
   }, sourceOpts2(rest));
 }
+function fromPulsar(consumer, opts) {
+  const {
+    autoAck = true,
+    deserialize = (buf) => {
+      try {
+        return JSON.parse(buf.toString());
+      } catch {
+        return buf.toString();
+      }
+    },
+    ...rest
+  } = opts ?? {};
+  return producer((_d, a) => {
+    let active = true;
+    const loop = async () => {
+      while (active) {
+        try {
+          const msg = await consumer.receive();
+          if (!active) return;
+          a.emit({
+            topic: msg.getTopicName(),
+            messageId: msg.getMessageId().toString(),
+            key: msg.getPartitionKey(),
+            value: deserialize(msg.getData()),
+            properties: msg.getProperties(),
+            publishTime: msg.getPublishTimestamp(),
+            eventTime: msg.getEventTimestamp(),
+            timestampNs: wallClockNs()
+          });
+          if (autoAck) await consumer.acknowledge(msg);
+        } catch (err) {
+          if (active) a.down([[ERROR, err]]);
+          return;
+        }
+      }
+    };
+    void loop();
+    return () => {
+      active = false;
+    };
+  }, sourceOpts2(rest));
+}
+function toPulsar(source, pulsarProducer, opts) {
+  const {
+    serialize = (v) => Buffer.from(JSON.stringify(v)),
+    keyExtractor,
+    propertiesExtractor,
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let data;
+        try {
+          data = serialize(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        void pulsarProducer.send({
+          data,
+          partitionKey: keyExtractor?.(value),
+          properties: propertiesExtractor?.(value)
+        }).catch((err) => {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        });
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function fromNATS(client, subject, opts) {
+  const decoder = new TextDecoder();
+  const {
+    queue,
+    deserialize = (data) => {
+      const text = decoder.decode(data);
+      try {
+        return JSON.parse(text);
+      } catch {
+        return text;
+      }
+    },
+    ...rest
+  } = opts ?? {};
+  return producer((_d, a) => {
+    let active = true;
+    const sub = client.subscribe(subject, queue ? { queue } : void 0);
+    const loop = async () => {
+      try {
+        for await (const msg of sub) {
+          if (!active) return;
+          const headers = {};
+          if (msg.headers) {
+            for (const k of msg.headers.keys()) {
+              headers[k] = msg.headers.get(k);
+            }
+          }
+          a.emit({
+            subject: msg.subject,
+            data: deserialize(msg.data),
+            headers,
+            reply: msg.reply,
+            sid: msg.sid,
+            timestampNs: wallClockNs()
+          });
+        }
+        if (active) a.down([[COMPLETE]]);
+      } catch (err) {
+        if (active) a.down([[ERROR, err]]);
+      }
+    };
+    void loop();
+    return () => {
+      active = false;
+    };
+  }, sourceOpts2(rest));
+}
+function toNATS(source, client, subject, opts) {
+  const encoder = new TextEncoder();
+  const {
+    serialize = (v) => encoder.encode(JSON.stringify(v)),
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let data;
+        try {
+          data = serialize(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        try {
+          client.publish(subject, data);
+        } catch (err) {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        }
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function fromRabbitMQ(channel, queue, opts) {
+  const {
+    autoAck = true,
+    deserialize = (buf) => {
+      try {
+        return JSON.parse(buf.toString());
+      } catch {
+        return buf.toString();
+      }
+    },
+    ...rest
+  } = opts ?? {};
+  return producer((_d, a) => {
+    let active = true;
+    let consumerTag;
+    const start = async () => {
+      try {
+        const result = await channel.consume(
+          queue,
+          (msg) => {
+            if (!active) return;
+            if (msg === null) {
+              if (active) a.down([[ERROR, new Error("Consumer cancelled by broker")]]);
+              return;
+            }
+            a.emit({
+              queue,
+              routingKey: msg.fields.routingKey,
+              exchange: msg.fields.exchange,
+              content: deserialize(msg.content),
+              properties: msg.properties,
+              deliveryTag: msg.fields.deliveryTag,
+              redelivered: msg.fields.redelivered,
+              timestampNs: wallClockNs()
+            });
+            if (autoAck) channel.ack(msg);
+          },
+          { noAck: false }
+        );
+        consumerTag = result.consumerTag;
+      } catch (err) {
+        if (active) a.down([[ERROR, err]]);
+      }
+    };
+    void start();
+    return () => {
+      active = false;
+      if (consumerTag !== void 0) {
+        void channel.cancel(consumerTag);
+      }
+    };
+  }, sourceOpts2(rest));
+}
+function toRabbitMQ(source, channel, exchange, opts) {
+  const {
+    serialize = (v) => Buffer.from(JSON.stringify(v)),
+    routingKeyExtractor = () => "",
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let routingKey;
+        try {
+          routingKey = routingKeyExtractor(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "routing_key",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        let content;
+        try {
+          content = serialize(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        try {
+          channel.publish(exchange, routingKey, content);
+        } catch (err) {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        }
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function toFile(source, writer, opts) {
+  const {
+    serialize = (v) => `${JSON.stringify(v)}
+`,
+    flushIntervalMs = 0,
+    batchSize = Number.POSITIVE_INFINITY,
+    onTransportError,
+    mode: _mode,
+    ...rest
+  } = opts ?? {};
+  let buffer2 = [];
+  let timer;
+  const doFlush = () => {
+    if (buffer2.length === 0) return;
+    const chunk = buffer2.join("");
+    buffer2 = [];
+    try {
+      writer.write(chunk);
+    } catch (err) {
+      onTransportError?.({
+        stage: "send",
+        error: err instanceof Error ? err : new Error(String(err)),
+        value: chunk
+      });
+    }
+  };
+  const scheduleFlush = () => {
+    if (flushIntervalMs > 0 && timer === void 0) {
+      timer = setTimeout(() => {
+        timer = void 0;
+        doFlush();
+      }, flushIntervalMs);
+    }
+  };
+  const buffered = flushIntervalMs > 0 || batchSize < Number.POSITIVE_INFINITY;
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let line;
+        try {
+          line = serialize(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        if (buffered) {
+          buffer2.push(line);
+          if (buffer2.length >= batchSize) doFlush();
+          else scheduleFlush();
+        } else {
+          try {
+            writer.write(line);
+          } catch (err) {
+            onTransportError?.({
+              stage: "send",
+              error: err instanceof Error ? err : new Error(String(err)),
+              value
+            });
+          }
+        }
+        return true;
+      }
+      if (msg[0] === COMPLETE || msg[0] === TEARDOWN) {
+        doFlush();
+      }
+      return false;
+    }
+  });
+  const unsub = inner.subscribe(() => {
+  });
+  const dispose = () => {
+    if (timer !== void 0) {
+      clearTimeout(timer);
+      timer = void 0;
+    }
+    doFlush();
+    writer.end();
+    unsub();
+  };
+  return {
+    dispose,
+    flush: async () => {
+      doFlush();
+    }
+  };
+}
+function escapeCSVField(value, delimiter) {
+  if (value.includes(delimiter) || value.includes('"') || value.includes("\n")) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+  return value;
+}
+function toCSV(source, writer, opts) {
+  const {
+    columns,
+    delimiter = ",",
+    writeHeader = true,
+    cellExtractor = (row, col) => String(row[col] ?? ""),
+    flushIntervalMs = 0,
+    batchSize = Number.POSITIVE_INFINITY,
+    onTransportError,
+    ...rest
+  } = opts;
+  let headerWritten = false;
+  const serializeRow = (row) => {
+    if (!headerWritten && writeHeader) {
+      headerWritten = true;
+      const header = columns.map((c) => escapeCSVField(c, delimiter)).join(delimiter);
+      const data = columns.map((c) => escapeCSVField(cellExtractor(row, c), delimiter)).join(delimiter);
+      return `${header}
+${data}
+`;
+    }
+    return `${columns.map((c) => escapeCSVField(cellExtractor(row, c), delimiter)).join(delimiter)}
+`;
+  };
+  return toFile(source, writer, {
+    serialize: serializeRow,
+    flushIntervalMs,
+    batchSize,
+    onTransportError,
+    ...rest
+  });
+}
+function toClickHouse(source, client, table, opts) {
+  const {
+    batchSize = 1e3,
+    flushIntervalMs = 5e3,
+    format = "JSONEachRow",
+    transform = (v) => v,
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  let buffer2 = [];
+  let timer;
+  let lastFlush = Promise.resolve();
+  const doFlush = () => {
+    if (buffer2.length === 0) return Promise.resolve();
+    const batch2 = buffer2;
+    buffer2 = [];
+    try {
+      const p = client.insert({ table, values: batch2, format }).catch((err) => {
+        onTransportError?.({
+          stage: "send",
+          error: err instanceof Error ? err : new Error(String(err)),
+          value: batch2
+        });
+      });
+      lastFlush = p;
+      return p;
+    } catch (err) {
+      onTransportError?.({
+        stage: "send",
+        error: err instanceof Error ? err : new Error(String(err)),
+        value: batch2
+      });
+      return Promise.resolve();
+    }
+  };
+  const scheduleFlush = () => {
+    if (timer === void 0) {
+      timer = setTimeout(() => {
+        timer = void 0;
+        doFlush();
+      }, flushIntervalMs);
+    }
+  };
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        try {
+          buffer2.push(transform(value));
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        if (buffer2.length >= batchSize) doFlush();
+        else scheduleFlush();
+        return true;
+      }
+      if (msg[0] === COMPLETE || msg[0] === TEARDOWN) {
+        doFlush();
+      }
+      return false;
+    }
+  });
+  const unsub = inner.subscribe(() => {
+  });
+  const dispose = () => {
+    if (timer !== void 0) {
+      clearTimeout(timer);
+      timer = void 0;
+    }
+    doFlush();
+    unsub();
+  };
+  return {
+    dispose,
+    flush: () => doFlush().then(() => lastFlush)
+  };
+}
+function toS3(source, client, bucket, opts) {
+  const {
+    format = "ndjson",
+    keyGenerator = (seq2, timestampNs) => {
+      const ms = Math.floor(timestampNs / 1e6);
+      const ts = new Date(ms).toISOString().replace(/[:.]/g, "-");
+      return `data/${ts}-${seq2}.${format === "ndjson" ? "ndjson" : "json"}`;
+    },
+    batchSize = 1e3,
+    flushIntervalMs = 1e4,
+    transform = (v) => v,
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  let buffer2 = [];
+  let timer;
+  let seq = 0;
+  let lastFlush = Promise.resolve();
+  const doFlush = () => {
+    if (buffer2.length === 0) return Promise.resolve();
+    const batch2 = buffer2;
+    buffer2 = [];
+    seq += 1;
+    const body = format === "ndjson" ? `${batch2.map((v) => JSON.stringify(v)).join("\n")}
+` : JSON.stringify(batch2);
+    const contentType = format === "ndjson" ? "application/x-ndjson" : "application/json";
+    const key = keyGenerator(seq, wallClockNs());
+    try {
+      const p = client.putObject({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }).then(() => {
+      }).catch((err) => {
+        onTransportError?.({
+          stage: "send",
+          error: err instanceof Error ? err : new Error(String(err)),
+          value: batch2
+        });
+      });
+      lastFlush = p;
+      return p;
+    } catch (err) {
+      onTransportError?.({
+        stage: "send",
+        error: err instanceof Error ? err : new Error(String(err)),
+        value: batch2
+      });
+      return Promise.resolve();
+    }
+  };
+  const scheduleFlush = () => {
+    if (timer === void 0) {
+      timer = setTimeout(() => {
+        timer = void 0;
+        doFlush();
+      }, flushIntervalMs);
+    }
+  };
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        try {
+          buffer2.push(transform(value));
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        if (buffer2.length >= batchSize) doFlush();
+        else scheduleFlush();
+        return true;
+      }
+      if (msg[0] === COMPLETE || msg[0] === TEARDOWN) {
+        doFlush();
+      }
+      return false;
+    }
+  });
+  const unsub = inner.subscribe(() => {
+  });
+  const dispose = () => {
+    if (timer !== void 0) {
+      clearTimeout(timer);
+      timer = void 0;
+    }
+    doFlush();
+    unsub();
+  };
+  return {
+    dispose,
+    flush: () => doFlush().then(() => lastFlush)
+  };
+}
+function toPostgres(source, client, table, opts) {
+  const {
+    toSQL = (v, t) => ({
+      sql: `INSERT INTO "${t.replace(/"/g, '""')}" (data) VALUES ($1)`,
+      params: [JSON.stringify(v)]
+    }),
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let query;
+        try {
+          query = toSQL(value, table);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        void client.query(query.sql, query.params).catch((err) => {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        });
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function toMongo(source, collection, opts) {
+  const { toDocument = (v) => v, onTransportError, ...rest } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let doc;
+        try {
+          doc = toDocument(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        void collection.insertOne(doc).catch((err) => {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        });
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function toLoki(source, client, opts) {
+  const {
+    labels = {},
+    toLine = (v) => JSON.stringify(v),
+    toLabels,
+    onTransportError,
+    ...rest
+  } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let line;
+        try {
+          line = toLine(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        let streamLabels;
+        try {
+          streamLabels = toLabels ? { ...labels, ...toLabels(value) } : labels;
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        const ts = `${wallClockNs()}`;
+        void client.push({ streams: [{ stream: streamLabels, values: [[ts, line]] }] }).catch((err) => {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        });
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function toTempo(source, client, opts) {
+  const { toResourceSpans = (v) => [v], onTransportError, ...rest } = opts ?? {};
+  const inner = node([source], () => void 0, {
+    describeKind: "effect",
+    ...rest,
+    onMessage(msg) {
+      if (msg[0] === DATA) {
+        const value = msg[1];
+        let spans;
+        try {
+          spans = toResourceSpans(value);
+        } catch (err) {
+          onTransportError?.({
+            stage: "serialize",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+          return true;
+        }
+        void client.push({ resourceSpans: spans }).catch((err) => {
+          onTransportError?.({
+            stage: "send",
+            error: err instanceof Error ? err : new Error(String(err)),
+            value
+          });
+        });
+        return true;
+      }
+      return false;
+    }
+  });
+  return inner.subscribe(() => {
+  });
+}
+function checkpointToS3(graph, client, bucket, opts) {
+  const { prefix = "checkpoints/", debounceMs, compactEvery, onError } = opts ?? {};
+  const adapter = {
+    save(data) {
+      const ms = Math.floor(wallClockNs() / 1e6);
+      const key = `${prefix}${graph.name}/checkpoint-${ms}.json`;
+      let body;
+      try {
+        body = JSON.stringify(data);
+      } catch (err) {
+        onError?.(err);
+        return;
+      }
+      void client.putObject({
+        Bucket: bucket,
+        Key: key,
+        Body: body,
+        ContentType: "application/json"
+      }).catch((err) => onError?.(err));
+    }
+  };
+  return graph.autoCheckpoint(adapter, { debounceMs, compactEvery, onError });
+}
+function checkpointToRedis(graph, client, opts) {
+  const { prefix = "graphrefly:checkpoint:", debounceMs, compactEvery, onError } = opts ?? {};
+  const key = `${prefix}${graph.name}`;
+  const adapter = {
+    save(data) {
+      let body;
+      try {
+        body = JSON.stringify(data);
+      } catch (err) {
+        onError?.(err);
+        return;
+      }
+      void client.set(key, body).catch((err) => onError?.(err));
+    }
+  };
+  return graph.autoCheckpoint(adapter, { debounceMs, compactEvery, onError });
+}
 
 // src/extra/backpressure.ts
 var nextLockId = 0;
@@ -6754,6 +7561,8 @@ function workerSelf(target, opts) {
   cached,
   catchError,
   checkpointNodeValue,
+  checkpointToRedis,
+  checkpointToS3,
   circuitBreaker,
   combine,
   combineLatest,
@@ -6795,10 +7604,13 @@ function workerSelf(target, opts) {
   fromIter,
   fromKafka,
   fromMCP,
+  fromNATS,
   fromNDJSON,
   fromOTel,
   fromPrometheus,
   fromPromise,
+  fromPulsar,
+  fromRabbitMQ,
   fromRedisStream,
   fromStatsD,
   fromSyslog,
@@ -6862,11 +7674,22 @@ function workerSelf(target, opts) {
   throwError,
   timeout,
   toArray,
+  toCSV,
+  toClickHouse,
+  toFile,
   toKafka,
+  toLoki,
   toMessages$,
+  toMongo,
+  toNATS,
   toObservable,
+  toPostgres,
+  toPulsar,
+  toRabbitMQ,
   toRedisStream,
+  toS3,
   toSSE,
+  toTempo,
   toWebSocket,
   tokenBucket,
   tokenTracker,