@powerhousedao/reactor-workflow 6.2.3-dev.54 → 6.2.3-dev.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { AppConnectionType, DEDUPE_KEY_PROPERTY, PieceCategory, Property, ReactorJobPendingError, TriggerStrategy, createAction, createPiece, createTrigger } from "@powerhousedao/pieces-framework";
2
2
  import { execFile, fork } from "node:child_process";
3
- import { accessSync, constants, existsSync, lstatSync, mkdirSync } from "node:fs";
3
+ import { accessSync, constants, existsSync, lstatSync, mkdirSync, readFileSync } from "node:fs";
4
4
  import { copyFile, lstat, mkdir, readFile, readdir, realpath, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
5
5
  import path from "node:path";
6
6
  import { promisify } from "node:util";
@@ -10,13 +10,13 @@ import { fileURLToPath } from "node:url";
10
10
  import { BRANCH_PORTS, CORE_PIECE_NAME, CORE_PIECE_NAME as CORE_PIECE_NAME$1, NEXT_PORT, PIECE_ACTION_PORTS, SCHEDULE_MODES, SCHEDULE_UNIT_MS, TRIGGER_PORTS, checkTriggerStrategy, referenceDocuments, undeclaredPortEdges } from "@powerhousedao/pieces-framework/workflow";
11
11
  import { Cron } from "croner";
12
12
  import { createHash, randomUUID } from "node:crypto";
13
- import "node:stream";
14
13
  import { ssrfIpClassifier } from "@powerhousedao/pieces-framework/host";
15
14
  import dns from "node:dns";
16
15
  import net, { isIP } from "node:net";
17
16
  import { AsyncLocalStorage } from "node:async_hooks";
18
- import { tmpdir, userInfo } from "node:os";
17
+ import os, { tmpdir, userInfo } from "node:os";
19
18
  import { blockKey, isExactVersion } from "@powerhousedao/pieces-framework/block-type";
19
+ import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
20
20
  //#region src/pieces/activepieces/types.ts
21
21
  function getActions(piece) {
22
22
  return (typeof piece.actions === "function" ? piece.actions() : piece.actions) ?? {};
@@ -1119,12 +1119,14 @@ function workerCompileCache(entryPath) {
1119
1119
  }
1120
1120
  return compileCache ?? void 0;
1121
1121
  }
1122
- function createForkTransport(entryPath) {
1122
+ function createForkTransport(entryPath, options = {}) {
1123
1123
  const profileDir = workerProfileDir();
1124
1124
  const cacheDir = workerCompileCache(entryPath);
1125
+ const execArgv = profileDir ? ["--cpu-prof", `--cpu-prof-dir=${profileDir}`] : [];
1126
+ if (options.heapMb) execArgv.push(`--max-old-space-size=${Math.trunc(options.heapMb)}`);
1125
1127
  const child = fork(entryPath, [], {
1126
1128
  env: cacheDir ? { NODE_COMPILE_CACHE: cacheDir } : {},
1127
- execArgv: profileDir ? ["--cpu-prof", `--cpu-prof-dir=${profileDir}`] : [],
1129
+ execArgv,
1128
1130
  serialization: "advanced",
1129
1131
  stdio: [
1130
1132
  "ignore",
@@ -1178,9 +1180,22 @@ function createForkTransport(entryPath) {
1178
1180
  }
1179
1181
  //#endregion
1180
1182
  //#region src/pieces/activepieces/worker/timings.ts
1181
- function epochNow() {
1182
- return performance.timeOrigin + performance.now();
1183
+ const ANCHOR_MS = 1e3;
1184
+ const STEP_MS = 5;
1185
+ function wallClock() {
1186
+ let anchoredAt = performance.now();
1187
+ let anchor = Date.now() - anchoredAt;
1188
+ return () => {
1189
+ const now = performance.now();
1190
+ if (now - anchoredAt >= ANCHOR_MS) {
1191
+ anchoredAt = now;
1192
+ const next = Date.now() - now;
1193
+ if (Math.abs(next - anchor) > STEP_MS) anchor = next;
1194
+ }
1195
+ return anchor + now;
1196
+ };
1183
1197
  }
1198
+ const epochNow = wallClock();
1184
1199
  new AsyncLocalStorage();
1185
1200
  new AsyncLocalStorage();
1186
1201
  //#endregion
@@ -1229,9 +1244,12 @@ var PieceWorker = class {
1229
1244
  queue = Promise.resolve();
1230
1245
  nextId = 1;
1231
1246
  disposed = false;
1247
+ memory;
1248
+ tlsPoisoned = false;
1232
1249
  constructor(options = {}) {
1233
1250
  const entryPath = options.entryPath;
1234
- this.connect = options.transport ?? (() => createForkTransport(entryPath ?? defaultEntryPath()));
1251
+ const { heapMb } = options;
1252
+ this.connect = options.transport ?? (() => createForkTransport(entryPath ?? defaultEntryPath(), { heapMb }));
1235
1253
  this.defaultTimeoutMs = options.defaultTimeoutMs ?? 3e4;
1236
1254
  this.hostCallTimeoutMs = options.hostCallTimeoutMs;
1237
1255
  this.models = options.models;
@@ -1268,8 +1286,30 @@ var PieceWorker = class {
1268
1286
  this.worker?.kill();
1269
1287
  this.worker = void 0;
1270
1288
  }
1289
+ health() {
1290
+ return {
1291
+ alive: this.worker !== void 0,
1292
+ ...this.memory ? { memory: this.memory } : {},
1293
+ tlsPoisoned: this.tlsPoisoned
1294
+ };
1295
+ }
1296
+ forgetChild() {
1297
+ this.memory = void 0;
1298
+ this.tlsPoisoned = false;
1299
+ }
1300
+ noteReply(response) {
1301
+ if ("tlsPoisoned" in response && response.tlsPoisoned === true) this.tlsPoisoned = true;
1302
+ const memory = response.timings?.memory;
1303
+ if (memory && [
1304
+ memory.rss,
1305
+ memory.heapUsed,
1306
+ memory.heapTotal,
1307
+ memory.external
1308
+ ].every((value) => Number.isFinite(value))) this.memory = memory;
1309
+ }
1271
1310
  spawn() {
1272
1311
  if (this.worker) return this.worker;
1312
+ this.forgetChild();
1273
1313
  this.forkedAt = epochNow();
1274
1314
  const worker = this.connect();
1275
1315
  this.sendModels(worker);
@@ -1345,6 +1385,7 @@ var PieceWorker = class {
1345
1385
  if (response.id !== id) return;
1346
1386
  cleanup();
1347
1387
  const received = epochNow();
1388
+ this.noteReply(response);
1348
1389
  if (response.type === "result") resolve({
1349
1390
  output: response.output,
1350
1391
  touched: response.touched,
@@ -1423,7 +1464,7 @@ var PieceWorker = class {
1423
1464
  //#region src/pieces/activepieces/worker/pool.ts
1424
1465
  var PieceWorkerPoolBusyError = class extends Error {
1425
1466
  constructor(size, waiting) {
1426
- super(`All ${size} piece workers are busy and ${waiting} runs are already waiting`);
1467
+ super(`All ${size} piece workers are busy and ${waiting} requests are already waiting`);
1427
1468
  this.name = "PieceWorkerPoolBusyError";
1428
1469
  }
1429
1470
  };
@@ -1433,148 +1474,348 @@ var PieceWorkerSessionClosedError = class extends Error {
1433
1474
  this.name = "PieceWorkerSessionClosedError";
1434
1475
  }
1435
1476
  };
1436
- const DEFAULT_SIZE = 4;
1477
+ const DEFAULT_TRIGGER_SLOTS = 1;
1478
+ const DEFAULT_DESIGN_SLOTS = 1;
1479
+ const DEFAULT_MIN_WARM = 1;
1480
+ const DEFAULT_MAX_REQUESTS = 1e3;
1481
+ const DEFAULT_IDLE_MS = 6e4;
1482
+ const DEFAULT_MAX_CONCURRENT_FORKS = 4;
1483
+ const HEAP_RETIREMENT = .8;
1484
+ const RSS_PER_HEAP = 1.6;
1485
+ const FORK_HEADROOM = .9;
1486
+ const REACTOR_THREADS_PER_CORE = 3;
1487
+ const FRESH_CHILD_BYTES = 83886080;
1488
+ const MIB = 1048576;
1489
+ const BOOT_WINDOW_MS = 1e3;
1490
+ const LANES = [
1491
+ "run",
1492
+ "trigger",
1493
+ "design"
1494
+ ];
1495
+ function pieceKey(request) {
1496
+ return request.bundleDir ?? request.entryPath;
1497
+ }
1498
+ function defaultPoolSize(options = {}) {
1499
+ const reactorThreads = Math.max(0, options.reactorThreads ?? 0);
1500
+ const cores = 2 * os.availableParallelism() - 1 - Math.floor(reactorThreads / REACTOR_THREADS_PER_CORE);
1501
+ const constrained = process.constrainedMemory();
1502
+ const limit = constrained > 0 && constrained < os.totalmem() ? constrained : os.totalmem();
1503
+ const perChild = RSS_PER_HEAP * (options.heapMb ?? 256) * MIB;
1504
+ const memory = Math.floor((limit - process.memoryUsage.rss()) / perChild);
1505
+ return Math.max(1, Math.min(cores, memory, 32));
1506
+ }
1507
+ function cgroupMemory() {
1508
+ try {
1509
+ const max = readFileSync("/sys/fs/cgroup/memory.max", "utf8").trim();
1510
+ if (max === "max") return void 0;
1511
+ const stat = readFileSync("/sys/fs/cgroup/memory.stat", "utf8");
1512
+ const anon = /^anon (\d+)$/m.exec(stat)?.[1];
1513
+ if (!anon) return void 0;
1514
+ return {
1515
+ limit: Number(max),
1516
+ used: Number(anon)
1517
+ };
1518
+ } catch {
1519
+ return;
1520
+ }
1521
+ }
1522
+ function cachedCgroupMemory() {
1523
+ let readAt = -Infinity;
1524
+ let last;
1525
+ return () => {
1526
+ const now = performance.now();
1527
+ if (now - readAt >= 1e3) {
1528
+ last = cgroupMemory();
1529
+ readAt = now;
1530
+ }
1531
+ return last;
1532
+ };
1533
+ }
1437
1534
  var PieceWorkerSession = class {
1438
1535
  pool;
1439
- worker;
1440
- taking;
1441
- held = false;
1536
+ lane;
1442
1537
  closed = false;
1443
- constructor(pool) {
1538
+ constructor(pool, lane) {
1444
1539
  this.pool = pool;
1540
+ this.lane = lane;
1445
1541
  }
1446
1542
  runAction(request, options) {
1447
- return this.request((worker) => worker.runAction(request, options));
1543
+ return this.pool.request(this, pieceKey(request), (worker) => worker.runAction(request, options));
1448
1544
  }
1449
1545
  resolveOptions(request, options) {
1450
- return this.request((worker) => worker.resolveOptions(request, options));
1546
+ return this.pool.request(this, pieceKey(request), (worker) => worker.resolveOptions(request, options));
1451
1547
  }
1452
1548
  checkConnection(request, options) {
1453
- return this.request((worker) => worker.checkConnection(request, options));
1549
+ return this.pool.request(this, pieceKey(request), (worker) => worker.checkConnection(request, options));
1454
1550
  }
1455
1551
  describePiece(request, options) {
1456
- return this.request((worker) => worker.describePiece(request, options));
1552
+ return this.pool.request(this, pieceKey(request), (worker) => worker.describePiece(request, options));
1457
1553
  }
1458
1554
  runTriggerHook(request, options) {
1459
- return this.request((worker) => worker.runTriggerHook(request, options));
1555
+ return this.pool.request(this, pieceKey(request), (worker) => worker.runTriggerHook(request, options));
1460
1556
  }
1461
1557
  close() {
1462
1558
  if (this.closed) return;
1463
1559
  this.closed = true;
1464
- this.pool.cancel(this);
1465
- this.worker?.dispose();
1466
- this.worker = void 0;
1467
- this.pool.forget(this);
1468
- this.giveBack();
1560
+ this.pool.closeSession(this);
1561
+ }
1562
+ /** @internal */
1563
+ isClosed() {
1564
+ return this.closed;
1469
1565
  }
1470
1566
  dispose() {
1471
1567
  this.close();
1472
1568
  }
1473
- async request(fn) {
1474
- const worker = await this.take();
1475
- if (this.closed) throw new PieceWorkerSessionClosedError();
1476
- return fn(worker);
1477
- }
1478
- take() {
1479
- if (this.closed) return Promise.reject(new PieceWorkerSessionClosedError());
1480
- if (this.worker) return Promise.resolve(this.worker);
1481
- this.taking ??= this.acquire();
1482
- return this.taking;
1483
- }
1484
- async acquire() {
1485
- const { telemetry } = this.pool;
1486
- await (telemetry ? telemetry.phase("worker.acquire", {}, () => this.pool.acquire(this)) : this.pool.acquire(this));
1487
- this.held = true;
1488
- if (this.closed) {
1489
- this.giveBack();
1490
- throw new PieceWorkerSessionClosedError();
1491
- }
1492
- this.worker = this.pool.build();
1493
- return this.worker;
1494
- }
1495
- giveBack() {
1496
- if (!this.held) return;
1497
- this.held = false;
1498
- this.pool.release();
1499
- }
1500
1569
  };
1501
1570
  var PieceWorkerPool = class {
1502
1571
  size;
1503
1572
  /** @internal */
1504
1573
  telemetry;
1505
1574
  maxQueueDepth;
1575
+ triggerSlots;
1576
+ designSlots;
1577
+ minWarm;
1578
+ maxRequests;
1579
+ idleMs;
1580
+ maxConcurrentForks;
1581
+ heapMb;
1582
+ memory;
1506
1583
  createWorker;
1507
1584
  workerOptions;
1508
- sessions = /* @__PURE__ */ new Set();
1509
- waiters = [];
1510
- inUse = 0;
1585
+ children = /* @__PURE__ */ new Set();
1586
+ waiters = {
1587
+ run: [],
1588
+ trigger: [],
1589
+ design: []
1590
+ };
1591
+ largestChild = FRESH_CHILD_BYTES;
1592
+ idleTimer;
1593
+ bootTimer;
1511
1594
  disposed = false;
1512
1595
  constructor(options = {}) {
1513
- const { size, maxQueueDepth, createWorker, ...workerOptions } = options;
1514
- this.size = Math.max(1, Math.trunc(size ?? DEFAULT_SIZE));
1596
+ const { size, maxQueueDepth, triggerSlots, designSlots, minWarm, maxRequests, idleMs, maxConcurrentForks, reactorThreads, memory, createWorker, ...workerOptions } = options;
1597
+ this.heapMb = workerOptions.heapMb ?? 256;
1598
+ this.size = Math.max(1, Math.trunc(size ?? defaultPoolSize({
1599
+ reactorThreads,
1600
+ heapMb: this.heapMb
1601
+ })));
1515
1602
  this.maxQueueDepth = Math.max(0, Math.trunc(maxQueueDepth ?? 0));
1603
+ this.triggerSlots = Math.max(1, Math.trunc(triggerSlots ?? DEFAULT_TRIGGER_SLOTS));
1604
+ this.designSlots = Math.max(1, Math.trunc(designSlots ?? DEFAULT_DESIGN_SLOTS));
1605
+ this.minWarm = Math.max(0, Math.trunc(minWarm ?? DEFAULT_MIN_WARM));
1606
+ this.maxRequests = Math.max(1, maxRequests ?? DEFAULT_MAX_REQUESTS);
1607
+ this.idleMs = Math.max(0, idleMs ?? DEFAULT_IDLE_MS);
1608
+ this.maxConcurrentForks = Math.max(1, maxConcurrentForks ?? DEFAULT_MAX_CONCURRENT_FORKS);
1609
+ this.memory = memory ?? cachedCgroupMemory();
1516
1610
  this.createWorker = createWorker ?? ((o) => new PieceWorker(o));
1517
- this.workerOptions = workerOptions;
1611
+ this.workerOptions = {
1612
+ ...workerOptions,
1613
+ heapMb: this.heapMb
1614
+ };
1518
1615
  this.telemetry = workerOptions.telemetry;
1519
1616
  }
1520
- session() {
1521
- if (this.disposed) throw new Error("Piece worker pool is disposed");
1522
- const session = new PieceWorkerSession(this);
1523
- this.sessions.add(session);
1524
- return session;
1617
+ session(lane = "run") {
1618
+ return new PieceWorkerSession(this, lane);
1525
1619
  }
1526
1620
  stats() {
1621
+ let active = 0;
1622
+ let idle = 0;
1623
+ for (const child of this.children) {
1624
+ if (child.lane !== "run") continue;
1625
+ if (child.busy) active++;
1626
+ else idle++;
1627
+ }
1527
1628
  return {
1528
1629
  size: this.size,
1529
- active: this.inUse,
1530
- waiting: this.waiters.length
1630
+ active,
1631
+ idle,
1632
+ waiting: this.waiters.run.length
1531
1633
  };
1532
1634
  }
1533
1635
  dispose() {
1636
+ if (this.disposed) return;
1534
1637
  this.disposed = true;
1535
- for (const waiter of this.waiters.splice(0)) waiter.reject(/* @__PURE__ */ new Error("Piece worker pool is disposed"));
1536
- for (const session of [...this.sessions]) session.close();
1537
- this.sessions.clear();
1538
- }
1539
- /** @internal — a session taking its slot. */
1540
- acquire(session) {
1541
- if (this.disposed) return Promise.reject(/* @__PURE__ */ new Error("Piece worker pool is disposed"));
1542
- if (this.inUse < this.size) {
1543
- this.inUse++;
1544
- return Promise.resolve();
1638
+ clearTimeout(this.idleTimer);
1639
+ this.idleTimer = void 0;
1640
+ clearTimeout(this.bootTimer);
1641
+ this.bootTimer = void 0;
1642
+ for (const lane of LANES) for (const waiter of this.waiters[lane].splice(0)) waiter.reject(new PieceWorkerDisposedError());
1643
+ for (const child of this.children) child.worker.dispose();
1644
+ this.children.clear();
1645
+ }
1646
+ /** @internal — one request on a leased child. */
1647
+ async request(session, key, fn) {
1648
+ if (session.isClosed()) throw new PieceWorkerSessionClosedError();
1649
+ const child = await this.lease(session, key);
1650
+ if (this.disposed) {
1651
+ this.giveBack(child, key);
1652
+ throw new PieceWorkerDisposedError();
1545
1653
  }
1546
- if (this.maxQueueDepth > 0 && this.waiters.length >= this.maxQueueDepth) return Promise.reject(new PieceWorkerPoolBusyError(this.size, this.waiters.length));
1547
- return new Promise((resolve, reject) => {
1548
- this.waiters.push({
1654
+ if (session.isClosed()) {
1655
+ this.giveBack(child, key);
1656
+ throw new PieceWorkerSessionClosedError();
1657
+ }
1658
+ try {
1659
+ return await fn(child.worker);
1660
+ } finally {
1661
+ this.giveBack(child, key);
1662
+ }
1663
+ }
1664
+ /** @internal — a session that closed. */
1665
+ closeSession(session) {
1666
+ const queue = this.waiters[session.lane];
1667
+ for (let i = queue.length - 1; i >= 0; i--) {
1668
+ if (queue[i].session !== session) continue;
1669
+ const [waiter] = queue.splice(i, 1);
1670
+ waiter.reject(new PieceWorkerSessionClosedError());
1671
+ }
1672
+ for (const child of this.children) if (child.busy && child.session === session) {
1673
+ child.doomed = true;
1674
+ child.worker.dispose();
1675
+ }
1676
+ }
1677
+ lease(session, key) {
1678
+ if (this.disposed) return Promise.reject(new PieceWorkerDisposedError());
1679
+ const queue = this.waiters[session.lane];
1680
+ const child = queue.length === 0 ? this.take(session.lane, key) : void 0;
1681
+ if (child) {
1682
+ child.session = session;
1683
+ return Promise.resolve(child);
1684
+ }
1685
+ if (session.lane === "run" && this.maxQueueDepth > 0 && queue.length >= this.maxQueueDepth) return Promise.reject(new PieceWorkerPoolBusyError(this.size, queue.length));
1686
+ const wait = new Promise((resolve, reject) => {
1687
+ queue.push({
1549
1688
  session,
1689
+ key,
1550
1690
  resolve,
1551
1691
  reject
1552
1692
  });
1553
1693
  });
1554
- }
1555
- /** @internal — a session that closed while queueing. */
1556
- cancel(session) {
1557
- const index = this.waiters.findIndex((w) => w.session === session);
1558
- if (index < 0) return;
1559
- const [waiter] = this.waiters.splice(index, 1);
1560
- waiter.reject(new PieceWorkerSessionClosedError());
1561
- }
1562
- /** @internal — a session giving its slot back. */
1563
- release() {
1564
- const next = this.waiters.shift();
1565
- if (next) {
1566
- next.resolve();
1567
- return;
1694
+ return this.telemetry ? this.telemetry.phase("worker.acquire", {}, () => wait) : wait;
1695
+ }
1696
+ take(lane, key) {
1697
+ let warm;
1698
+ let recent;
1699
+ for (const child of this.children) {
1700
+ if (child.lane !== lane || child.busy) continue;
1701
+ if (key && child.loaded.has(key)) {
1702
+ warm = child;
1703
+ break;
1704
+ }
1705
+ if (!recent || child.idleSince > recent.idleSince) recent = child;
1706
+ }
1707
+ const child = warm ?? recent ?? this.fork(lane);
1708
+ if (child) child.busy = true;
1709
+ return child;
1710
+ }
1711
+ fork(lane) {
1712
+ let inLane = 0;
1713
+ let booting = 0;
1714
+ let firstBootEnds = Infinity;
1715
+ const now = performance.now();
1716
+ for (const child of this.children) {
1717
+ if (child.lane === lane) inLane++;
1718
+ if (!child.replied && now - child.forkedAt < BOOT_WINDOW_MS) {
1719
+ booting++;
1720
+ firstBootEnds = Math.min(firstBootEnds, child.forkedAt + BOOT_WINDOW_MS);
1721
+ }
1722
+ }
1723
+ const capacity = {
1724
+ run: this.size,
1725
+ trigger: this.triggerSlots,
1726
+ design: this.designSlots
1727
+ }[lane];
1728
+ if (inLane >= capacity) return void 0;
1729
+ if (inLane > 0) {
1730
+ if (booting >= this.maxConcurrentForks) {
1731
+ this.redispatchAt(firstBootEnds - now);
1732
+ return;
1733
+ }
1734
+ if (!this.roomForChild()) return void 0;
1735
+ }
1736
+ const child = {
1737
+ worker: this.createWorker(this.workerOptions),
1738
+ lane,
1739
+ loaded: /* @__PURE__ */ new Set(),
1740
+ requests: 0,
1741
+ busy: false,
1742
+ forkedAt: now,
1743
+ replied: false,
1744
+ doomed: false,
1745
+ idleSince: 0,
1746
+ session: void 0
1747
+ };
1748
+ this.children.add(child);
1749
+ return child;
1750
+ }
1751
+ roomForChild() {
1752
+ const memory = this.memory();
1753
+ if (!memory) return true;
1754
+ return memory.used + this.largestChild <= FORK_HEADROOM * memory.limit;
1755
+ }
1756
+ giveBack(child, key) {
1757
+ child.busy = false;
1758
+ child.replied = true;
1759
+ child.session = void 0;
1760
+ child.requests++;
1761
+ child.idleSince = performance.now();
1762
+ const health = child.worker.health?.();
1763
+ if (health?.memory) this.largestChild = Math.max(this.largestChild, health.memory.rss);
1764
+ if (this.spent(child, health)) this.retire(child);
1765
+ else if (key) child.loaded.add(key);
1766
+ this.dispatch();
1767
+ this.scheduleIdleSweep();
1768
+ }
1769
+ spent(child, health) {
1770
+ if (this.disposed || child.doomed) return true;
1771
+ if (child.requests >= this.maxRequests) return true;
1772
+ if (!health) return false;
1773
+ if (!health.alive || health.tlsPoisoned) return true;
1774
+ return health.memory !== void 0 && health.memory.heapUsed > HEAP_RETIREMENT * this.heapMb * MIB;
1775
+ }
1776
+ retire(child) {
1777
+ this.children.delete(child);
1778
+ child.worker.dispose();
1779
+ }
1780
+ dispatch() {
1781
+ if (this.disposed) return;
1782
+ for (const lane of LANES) {
1783
+ const queue = this.waiters[lane];
1784
+ while (queue.length > 0) {
1785
+ const child = this.take(lane, queue[0].key);
1786
+ if (!child) break;
1787
+ const waiter = queue.shift();
1788
+ child.session = waiter.session;
1789
+ waiter.resolve(child);
1790
+ }
1568
1791
  }
1569
- this.inUse = Math.max(0, this.inUse - 1);
1570
- }
1571
- /** @internal — one child for one session. */
1572
- build() {
1573
- return this.createWorker(this.workerOptions);
1574
1792
  }
1575
- /** @internal — a session that closed itself. */
1576
- forget(session) {
1577
- this.sessions.delete(session);
1793
+ redispatchAt(ms) {
1794
+ if (this.bootTimer || this.disposed) return;
1795
+ this.bootTimer = setTimeout(() => {
1796
+ this.bootTimer = void 0;
1797
+ this.dispatch();
1798
+ }, Math.max(1, ms));
1799
+ this.bootTimer.unref();
1800
+ }
1801
+ scheduleIdleSweep() {
1802
+ if (this.idleTimer || this.disposed || this.idleMs === 0) return;
1803
+ if (this.retirable().length === 0) return;
1804
+ this.idleTimer = setTimeout(() => {
1805
+ this.idleTimer = void 0;
1806
+ this.sweepIdle();
1807
+ }, this.idleMs);
1808
+ this.idleTimer.unref();
1809
+ }
1810
+ sweepIdle() {
1811
+ const now = performance.now();
1812
+ for (const child of this.retirable()) if (now - child.idleSince >= this.idleMs) this.retire(child);
1813
+ this.scheduleIdleSweep();
1814
+ }
1815
+ retirable() {
1816
+ const idle = (lane) => [...this.children].filter((child) => child.lane === lane && !child.busy).sort((a, b) => a.idleSince - b.idleSince);
1817
+ const run = idle("run");
1818
+ return [...run.slice(0, Math.max(0, run.length - this.minWarm)), ...idle("design")];
1578
1819
  }
1579
1820
  };
1580
1821
  //#endregion
@@ -2654,7 +2895,7 @@ const manualTrigger = createTrigger({
2654
2895
  });
2655
2896
  //#endregion
2656
2897
  //#region src/pieces/core/version.ts
2657
- const CORE_PIECE_VERSION = "6.2.3-dev.54";
2898
+ const CORE_PIECE_VERSION = "6.2.3-dev.56";
2658
2899
  //#endregion
2659
2900
  //#region src/pieces/core/index.ts
2660
2901
  const corePiece = createPiece({
@@ -3577,6 +3818,277 @@ function installedPieces() {
3577
3818
  return [...builtinLocalPieces(), ...packagePieces.entries().filter((piece) => !isBuiltinPiece(piece.name))];
3578
3819
  }
3579
3820
  //#endregion
3580
- export { isPrivateAddress as $, declaredConnectionIds as A, stepBlock as B, builtinPiece as C, AttachmentCache as D, CORE_PIECE_VERSION as E, InMemorySecretProvider as F, PieceWorkerError as G, PieceWorkerPool as H, SecretDeletedError as I, redact as J, PieceWorkerTimeoutError as K, SecretNotFoundError as L, shapeConnection as M, resolveStepInput as N, BoundConnectionResolver as O, expressionsIn as P, DEFAULT_EGRESS_POLICY as Q, parseSecretRef as R, blockPorts as S, CORE_PIECE_NAME$1 as T, PieceWorker as U, triggerBlock as V, PieceWorkerDisposedError as W, rememberSecrets as X, redactMessage as Y, secretsFor as Z, resolvedBlock as _, isIndeterminateError as _t, RUN_DEADLINE_ERROR_NAME as a, pieceModuleRef as at, resolutionOf as b, canonicalJson as c, UnsupportedPieceFeatureError as ct, DynamicPropertiesError as d, unsupportedTrigger as dt, parseAddress as et, checkDynamicProperties as f, ensurePieceBundle as ft, isHostBound as g, setPieceRegistryUrl as gt, CompositeBlockExecutor as h, pieceRegistrySource as ht, packagePieces as i, bundleResolver as it, declaredReactorConnectionIds as j, StaticConnectionResolver as k, hashOf as l, authMethodFor as lt, ActivepiecesBlockExecutor as m, npmRegistryBaseUrl as mt, installedPiece as n, FileTooLargeError as nt, errorNameOf as o, sourcedResolver as ot, effectiveRetryPolicy as p, fetchPieceBundle as pt, containsRedactedMarker as q, installedPieces as r, maxFileBytes as rt, runWorkflow as s, buildDescriptor as st, PieceRegistry as t, extractDedupeKey as tt, stepConfigHash as u, unsupportedAuth as ut, storeHandlers as v, describeBuiltinPiece as w, unpinnedNote as x, blockLabel as y, secretRefFromId as z };
3821
+ //#region src/telemetry.ts
3822
+ const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
3823
+ var WorkflowTelemetry = class {
3824
+ tracer;
3825
+ meter;
3826
+ runs;
3827
+ runDuration;
3828
+ stepDuration;
3829
+ phaseDuration;
3830
+ dispatcherGauges;
3831
+ poolGauges;
3832
+ constructor(options = {}) {
3833
+ this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
3834
+ this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
3835
+ this.runs = this.meter.createCounter("workflow.runs", {
3836
+ description: "Runs finished, by status and trigger kind",
3837
+ unit: "{run}"
3838
+ });
3839
+ this.runDuration = this.meter.createHistogram("workflow.run.duration", {
3840
+ description: "Run duration from fire to the journal closing it",
3841
+ unit: "ms"
3842
+ });
3843
+ this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
3844
+ description: "Step duration, by piece and status",
3845
+ unit: "ms"
3846
+ });
3847
+ this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
3848
+ description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
3849
+ unit: "ms"
3850
+ });
3851
+ }
3852
+ phase(name, attributes, fn, parent) {
3853
+ const started = performance.now();
3854
+ return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
3855
+ }
3856
+ workerTimings(request, timings, forkedAt, ipc) {
3857
+ const under = (span) => trace.setSpan(context.active(), span);
3858
+ const child = (name, start, end, parent = request, attributes) => {
3859
+ const span = this.tracer.startSpan(`workflow.worker.${name}`, {
3860
+ startTime: start,
3861
+ ...attributes ? { attributes } : {}
3862
+ }, under(parent));
3863
+ span.end(end);
3864
+ this.recordPhase(`worker.${name}`, end - start);
3865
+ return span;
3866
+ };
3867
+ const ready = finite(timings.ready);
3868
+ if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
3869
+ const received = finite(timings.received);
3870
+ const sent = finite(timings.sent);
3871
+ const delivered = Math.max(ipc.sent, ready ?? 0);
3872
+ if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
3873
+ const byName = /* @__PURE__ */ new Map();
3874
+ const phases = Array.isArray(timings.phases) ? timings.phases : [];
3875
+ for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
3876
+ const parent = phase.parent ? byName.get(phase.parent) : void 0;
3877
+ byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
3878
+ }
3879
+ if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
3880
+ request.setAttributes(workerMemory(timings.memory));
3881
+ }
3882
+ recordPhase(name, ms) {
3883
+ this.phaseDuration.record(ms, { phase: name });
3884
+ }
3885
+ span(name, attributes, fn, parent = context.active()) {
3886
+ return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
3887
+ try {
3888
+ return await fn(span);
3889
+ } catch (error) {
3890
+ failSpan(span, error);
3891
+ throw error;
3892
+ } finally {
3893
+ span.end();
3894
+ }
3895
+ });
3896
+ }
3897
+ recordRun(ms, status, triggerKind) {
3898
+ const attributes = {
3899
+ "run.status": status,
3900
+ "trigger.kind": metricTriggerKind(triggerKind)
3901
+ };
3902
+ this.runs.add(1, attributes);
3903
+ this.runDuration.record(ms, attributes);
3904
+ }
3905
+ recordStep(ms, pieceName, status) {
3906
+ this.stepDuration.record(ms, {
3907
+ "piece.name": pieceName,
3908
+ "step.status": status
3909
+ });
3910
+ }
3911
+ observeWorkerPool(stats) {
3912
+ this.poolGauges ??= this.createPoolGauges();
3913
+ const { size, active, idle, waiting } = this.poolGauges;
3914
+ const observables = [
3915
+ size,
3916
+ active,
3917
+ idle,
3918
+ waiting
3919
+ ];
3920
+ const callback = (result) => {
3921
+ const current = stats();
3922
+ result.observe(size, current.size);
3923
+ result.observe(active, current.active);
3924
+ result.observe(idle, current.idle);
3925
+ result.observe(waiting, current.waiting);
3926
+ };
3927
+ this.meter.addBatchObservableCallback(callback, observables);
3928
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
3929
+ }
3930
+ observeDispatcher(stats) {
3931
+ this.dispatcherGauges ??= {
3932
+ inFlight: this.meter.createObservableGauge("workflow.runs.in_flight", {
3933
+ description: "Runs admitted and not yet finished",
3934
+ unit: "{run}"
3935
+ }),
3936
+ waiting: this.meter.createObservableGauge("workflow.runs.waiting", {
3937
+ description: "Runs waiting for admission across workflows",
3938
+ unit: "{run}"
3939
+ }),
3940
+ loop: this.meter.createObservableGauge("workflow.host.event_loop.utilization", {
3941
+ description: "Main thread busy share over the last second",
3942
+ unit: "1"
3943
+ })
3944
+ };
3945
+ const { inFlight, waiting, loop } = this.dispatcherGauges;
3946
+ const observables = [
3947
+ inFlight,
3948
+ waiting,
3949
+ loop
3950
+ ];
3951
+ const callback = (result) => {
3952
+ const current = stats();
3953
+ result.observe(inFlight, current.inFlight);
3954
+ result.observe(waiting, current.waiting);
3955
+ result.observe(loop, current.utilization);
3956
+ };
3957
+ this.meter.addBatchObservableCallback(callback, observables);
3958
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
3959
+ }
3960
+ createPoolGauges() {
3961
+ const gauge = (name, description) => this.meter.createObservableGauge(name, {
3962
+ description,
3963
+ unit: "{worker}"
3964
+ });
3965
+ return {
3966
+ size: gauge("workflow.worker.pool.size", "Most run children"),
3967
+ active: gauge("workflow.worker.pool.active", "Children serving a request"),
3968
+ idle: gauge("workflow.worker.pool.idle", "Warm children waiting"),
3969
+ waiting: gauge("workflow.worker.pool.waiting", "Requests waiting for a child")
3970
+ };
3971
+ }
3972
+ };
3973
+ const WORKER_PHASES = /* @__PURE__ */ new Set([
3974
+ "piece.load",
3975
+ "reactor.open",
3976
+ "models",
3977
+ "action"
3978
+ ]);
3979
+ const WORKER_ATTRIBUTES = {
3980
+ "piece.cached": "boolean",
3981
+ "document.type": "string"
3982
+ };
3983
+ const MAX_ATTRIBUTE_LENGTH = 256;
3984
+ function finite(value) {
3985
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3986
+ }
3987
+ function knownPhase(phase) {
3988
+ if (typeof phase !== "object" || phase === null) return false;
3989
+ const { name, start, end, parent } = phase;
3990
+ return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && finite(start) !== void 0 && finite(end) !== void 0 && end >= start;
3991
+ }
3992
+ function phaseAttributes(attributes) {
3993
+ if (typeof attributes !== "object" || attributes === null) return void 0;
3994
+ const kept = {};
3995
+ for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
3996
+ const value = attributes[key];
3997
+ if (typeof value !== type) continue;
3998
+ kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
3999
+ }
4000
+ return Object.keys(kept).length > 0 ? kept : void 0;
4001
+ }
4002
+ const MB = 2 ** 20;
4003
+ function workerMemory(memory) {
4004
+ if (typeof memory !== "object" || memory === null) return {};
4005
+ const attributes = {};
4006
+ for (const [key, name] of [
4007
+ ["rss", "worker.memory.rss_mb"],
4008
+ ["heapUsed", "worker.memory.heap_used_mb"],
4009
+ ["heapTotal", "worker.memory.heap_total_mb"],
4010
+ ["external", "worker.memory.external_mb"]
4011
+ ]) {
4012
+ const bytes = finite(memory[key]);
4013
+ if (bytes !== void 0) attributes[name] = bytes / MB;
4014
+ }
4015
+ return attributes;
4016
+ }
4017
+ function metricTriggerKind(triggerKind) {
4018
+ return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
4019
+ }
4020
+ function metricPieceName(step, resolution) {
4021
+ return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
4022
+ }
4023
+ function tracedSteps(executor, telemetry) {
4024
+ return { execute(execution) {
4025
+ const { step } = execution;
4026
+ const started = performance.now();
4027
+ let status = "FAILED";
4028
+ let resolution;
4029
+ return telemetry.span("workflow.step", {
4030
+ "step.id": step.id,
4031
+ "step.key": step.key,
4032
+ "piece.name": step.pieceName,
4033
+ "action.name": step.actionName
4034
+ }, async (span) => {
4035
+ try {
4036
+ const result = await executor.execute(execution);
4037
+ resolution = result.resolution;
4038
+ status = "SUCCEEDED";
4039
+ const version = resolution?.resolved?.version;
4040
+ if (version) span.setAttribute("piece.version", version);
4041
+ return result;
4042
+ } catch (error) {
4043
+ resolution = resolutionOf(error);
4044
+ throw error;
4045
+ }
4046
+ }).finally(() => {
4047
+ telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
4048
+ });
4049
+ } };
4050
+ }
4051
+ function failSpan(span, error) {
4052
+ span.recordException(error instanceof Error ? error : String(error));
4053
+ span.setStatus({
4054
+ code: SpanStatusCode.ERROR,
4055
+ message: error instanceof Error ? error.message : String(error)
4056
+ });
4057
+ }
4058
+ function tracedMethods(target, telemetry, prefix, parent) {
4059
+ return new Proxy(target, { get(object, property, receiver) {
4060
+ const value = Reflect.get(object, property, receiver);
4061
+ if (typeof value !== "function" || typeof property !== "string") return value;
4062
+ const phase = `${prefix}.${property}`;
4063
+ return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
4064
+ const started = performance.now();
4065
+ const end = (error) => {
4066
+ if (error !== void 0) failSpan(span, error);
4067
+ span.end();
4068
+ telemetry.recordPhase(phase, performance.now() - started);
4069
+ };
4070
+ let result;
4071
+ try {
4072
+ result = value.apply(object, args);
4073
+ } catch (error) {
4074
+ end(error);
4075
+ throw error;
4076
+ }
4077
+ if (!(result instanceof Promise)) {
4078
+ end();
4079
+ return result;
4080
+ }
4081
+ return result.then((resolved) => {
4082
+ end();
4083
+ return resolved;
4084
+ }, (error) => {
4085
+ end(error);
4086
+ throw error;
4087
+ });
4088
+ });
4089
+ } });
4090
+ }
4091
+ //#endregion
4092
+ export { rememberSecrets as $, AttachmentCache as A, SecretNotFoundError as B, blockLabel as C, describeBuiltinPiece as D, builtinPiece as E, shapeConnection as F, PieceWorkerPool as G, secretRefFromId as H, resolveStepInput as I, PieceWorkerError as J, PieceWorker as K, expressionsIn as L, StaticConnectionResolver as M, declaredConnectionIds as N, CORE_PIECE_NAME$1 as O, declaredReactorConnectionIds as P, redactMessage as Q, InMemorySecretProvider as R, storeHandlers as S, blockPorts as T, stepBlock as U, parseSecretRef as V, triggerBlock as W, containsRedactedMarker as X, PieceWorkerTimeoutError as Y, redact as Z, effectiveRetryPolicy as _, npmRegistryBaseUrl as _t, PieceRegistry as a, FileTooLargeError as at, isHostBound as b, isIndeterminateError as bt, packagePieces as c, pieceModuleRef as ct, runWorkflow as d, UnsupportedPieceFeatureError as dt, secretsFor as et, canonicalJson as f, authMethodFor as ft, checkDynamicProperties as g, fetchPieceBundle as gt, DynamicPropertiesError as h, ensurePieceBundle as ht, tracedSteps as i, extractDedupeKey as it, BoundConnectionResolver as j, CORE_PIECE_VERSION as k, RUN_DEADLINE_ERROR_NAME as l, sourcedResolver as lt, stepConfigHash as m, unsupportedTrigger as mt, WorkflowTelemetry as n, isPrivateAddress as nt, installedPiece as o, maxFileBytes as ot, hashOf as p, unsupportedAuth as pt, PieceWorkerDisposedError as q, tracedMethods as r, parseAddress as rt, installedPieces as s, bundleResolver as st, TELEMETRY_SCOPE as t, DEFAULT_EGRESS_POLICY as tt, errorNameOf as u, buildDescriptor as ut, ActivepiecesBlockExecutor as v, pieceRegistrySource as vt, unpinnedNote as w, resolvedBlock as x, CompositeBlockExecutor as y, setPieceRegistryUrl as yt, SecretDeletedError as z };
3581
4093
 
3582
- //# sourceMappingURL=piece-registry-0pdOQQdk.js.map
4094
+ //# sourceMappingURL=telemetry-CM82x7pC.js.map