@powerhousedao/reactor-workflow 6.2.3-dev.55 → 6.2.3-dev.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { AppConnectionType, DEDUPE_KEY_PROPERTY, PieceCategory, Property, ReactorJobPendingError, TriggerStrategy, createAction, createPiece, createTrigger } from "@powerhousedao/pieces-framework";
2
2
  import { execFile, fork } from "node:child_process";
3
- import { accessSync, constants, existsSync, lstatSync, mkdirSync } from "node:fs";
3
+ import { accessSync, constants, existsSync, lstatSync, mkdirSync, readFileSync } from "node:fs";
4
4
  import { copyFile, lstat, mkdir, readFile, readdir, realpath, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
5
5
  import path from "node:path";
6
6
  import { promisify } from "node:util";
@@ -14,8 +14,9 @@ import { ssrfIpClassifier } from "@powerhousedao/pieces-framework/host";
14
14
  import dns from "node:dns";
15
15
  import net, { isIP } from "node:net";
16
16
  import { AsyncLocalStorage } from "node:async_hooks";
17
- import { tmpdir, userInfo } from "node:os";
17
+ import os, { tmpdir, userInfo } from "node:os";
18
18
  import { blockKey, isExactVersion } from "@powerhousedao/pieces-framework/block-type";
19
+ import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
19
20
  //#region src/pieces/activepieces/types.ts
20
21
  function getActions(piece) {
21
22
  return (typeof piece.actions === "function" ? piece.actions() : piece.actions) ?? {};
@@ -1118,12 +1119,14 @@ function workerCompileCache(entryPath) {
1118
1119
  }
1119
1120
  return compileCache ?? void 0;
1120
1121
  }
1121
- function createForkTransport(entryPath) {
1122
+ function createForkTransport(entryPath, options = {}) {
1122
1123
  const profileDir = workerProfileDir();
1123
1124
  const cacheDir = workerCompileCache(entryPath);
1125
+ const execArgv = profileDir ? ["--cpu-prof", `--cpu-prof-dir=${profileDir}`] : [];
1126
+ if (options.heapMb) execArgv.push(`--max-old-space-size=${Math.trunc(options.heapMb)}`);
1124
1127
  const child = fork(entryPath, [], {
1125
1128
  env: cacheDir ? { NODE_COMPILE_CACHE: cacheDir } : {},
1126
- execArgv: profileDir ? ["--cpu-prof", `--cpu-prof-dir=${profileDir}`] : [],
1129
+ execArgv,
1127
1130
  serialization: "advanced",
1128
1131
  stdio: [
1129
1132
  "ignore",
@@ -1177,9 +1180,22 @@ function createForkTransport(entryPath) {
1177
1180
  }
1178
1181
  //#endregion
1179
1182
  //#region src/pieces/activepieces/worker/timings.ts
1180
- function epochNow() {
1181
- return performance.timeOrigin + performance.now();
1183
+ const ANCHOR_MS = 1e3;
1184
+ const STEP_MS = 5;
1185
+ function wallClock() {
1186
+ let anchoredAt = performance.now();
1187
+ let anchor = Date.now() - anchoredAt;
1188
+ return () => {
1189
+ const now = performance.now();
1190
+ if (now - anchoredAt >= ANCHOR_MS) {
1191
+ anchoredAt = now;
1192
+ const next = Date.now() - now;
1193
+ if (Math.abs(next - anchor) > STEP_MS) anchor = next;
1194
+ }
1195
+ return anchor + now;
1196
+ };
1182
1197
  }
1198
+ const epochNow = wallClock();
1183
1199
  new AsyncLocalStorage();
1184
1200
  new AsyncLocalStorage();
1185
1201
  //#endregion
@@ -1228,9 +1244,12 @@ var PieceWorker = class {
1228
1244
  queue = Promise.resolve();
1229
1245
  nextId = 1;
1230
1246
  disposed = false;
1247
+ memory;
1248
+ tlsPoisoned = false;
1231
1249
  constructor(options = {}) {
1232
1250
  const entryPath = options.entryPath;
1233
- this.connect = options.transport ?? (() => createForkTransport(entryPath ?? defaultEntryPath()));
1251
+ const { heapMb } = options;
1252
+ this.connect = options.transport ?? (() => createForkTransport(entryPath ?? defaultEntryPath(), { heapMb }));
1234
1253
  this.defaultTimeoutMs = options.defaultTimeoutMs ?? 3e4;
1235
1254
  this.hostCallTimeoutMs = options.hostCallTimeoutMs;
1236
1255
  this.models = options.models;
@@ -1267,8 +1286,30 @@ var PieceWorker = class {
1267
1286
  this.worker?.kill();
1268
1287
  this.worker = void 0;
1269
1288
  }
1289
+ health() {
1290
+ return {
1291
+ alive: this.worker !== void 0,
1292
+ ...this.memory ? { memory: this.memory } : {},
1293
+ tlsPoisoned: this.tlsPoisoned
1294
+ };
1295
+ }
1296
+ forgetChild() {
1297
+ this.memory = void 0;
1298
+ this.tlsPoisoned = false;
1299
+ }
1300
+ noteReply(response) {
1301
+ if ("tlsPoisoned" in response && response.tlsPoisoned === true) this.tlsPoisoned = true;
1302
+ const memory = response.timings?.memory;
1303
+ if (memory && [
1304
+ memory.rss,
1305
+ memory.heapUsed,
1306
+ memory.heapTotal,
1307
+ memory.external
1308
+ ].every((value) => Number.isFinite(value))) this.memory = memory;
1309
+ }
1270
1310
  spawn() {
1271
1311
  if (this.worker) return this.worker;
1312
+ this.forgetChild();
1272
1313
  this.forkedAt = epochNow();
1273
1314
  const worker = this.connect();
1274
1315
  this.sendModels(worker);
@@ -1344,6 +1385,7 @@ var PieceWorker = class {
1344
1385
  if (response.id !== id) return;
1345
1386
  cleanup();
1346
1387
  const received = epochNow();
1388
+ this.noteReply(response);
1347
1389
  if (response.type === "result") resolve({
1348
1390
  output: response.output,
1349
1391
  touched: response.touched,
@@ -1422,7 +1464,7 @@ var PieceWorker = class {
1422
1464
  //#region src/pieces/activepieces/worker/pool.ts
1423
1465
  var PieceWorkerPoolBusyError = class extends Error {
1424
1466
  constructor(size, waiting) {
1425
- super(`All ${size} piece workers are busy and ${waiting} runs are already waiting`);
1467
+ super(`All ${size} piece workers are busy and ${waiting} requests are already waiting`);
1426
1468
  this.name = "PieceWorkerPoolBusyError";
1427
1469
  }
1428
1470
  };
@@ -1432,148 +1474,348 @@ var PieceWorkerSessionClosedError = class extends Error {
1432
1474
  this.name = "PieceWorkerSessionClosedError";
1433
1475
  }
1434
1476
  };
1435
- const DEFAULT_SIZE = 4;
1477
+ const DEFAULT_TRIGGER_SLOTS = 1;
1478
+ const DEFAULT_DESIGN_SLOTS = 1;
1479
+ const DEFAULT_MIN_WARM = 1;
1480
+ const DEFAULT_MAX_REQUESTS = 1e3;
1481
+ const DEFAULT_IDLE_MS = 6e4;
1482
+ const DEFAULT_MAX_CONCURRENT_FORKS = 4;
1483
+ const HEAP_RETIREMENT = .8;
1484
+ const RSS_PER_HEAP = 1.6;
1485
+ const FORK_HEADROOM = .9;
1486
+ const REACTOR_THREADS_PER_CORE = 3;
1487
+ const FRESH_CHILD_BYTES = 83886080;
1488
+ const MIB = 1048576;
1489
+ const BOOT_WINDOW_MS = 1e3;
1490
+ const LANES = [
1491
+ "run",
1492
+ "trigger",
1493
+ "design"
1494
+ ];
1495
+ function pieceKey(request) {
1496
+ return request.bundleDir ?? request.entryPath;
1497
+ }
1498
+ function defaultPoolSize(options = {}) {
1499
+ const reactorThreads = Math.max(0, options.reactorThreads ?? 0);
1500
+ const cores = 2 * os.availableParallelism() - 1 - Math.floor(reactorThreads / REACTOR_THREADS_PER_CORE);
1501
+ const constrained = process.constrainedMemory();
1502
+ const limit = constrained > 0 && constrained < os.totalmem() ? constrained : os.totalmem();
1503
+ const perChild = RSS_PER_HEAP * (options.heapMb ?? 256) * MIB;
1504
+ const memory = Math.floor((limit - process.memoryUsage.rss()) / perChild);
1505
+ return Math.max(1, Math.min(cores, memory, 32));
1506
+ }
1507
+ function cgroupMemory() {
1508
+ try {
1509
+ const max = readFileSync("/sys/fs/cgroup/memory.max", "utf8").trim();
1510
+ if (max === "max") return void 0;
1511
+ const stat = readFileSync("/sys/fs/cgroup/memory.stat", "utf8");
1512
+ const anon = /^anon (\d+)$/m.exec(stat)?.[1];
1513
+ if (!anon) return void 0;
1514
+ return {
1515
+ limit: Number(max),
1516
+ used: Number(anon)
1517
+ };
1518
+ } catch {
1519
+ return;
1520
+ }
1521
+ }
1522
+ function cachedCgroupMemory() {
1523
+ let readAt = -Infinity;
1524
+ let last;
1525
+ return () => {
1526
+ const now = performance.now();
1527
+ if (now - readAt >= 1e3) {
1528
+ last = cgroupMemory();
1529
+ readAt = now;
1530
+ }
1531
+ return last;
1532
+ };
1533
+ }
1436
1534
  var PieceWorkerSession = class {
1437
1535
  pool;
1438
- worker;
1439
- taking;
1440
- held = false;
1536
+ lane;
1441
1537
  closed = false;
1442
- constructor(pool) {
1538
+ constructor(pool, lane) {
1443
1539
  this.pool = pool;
1540
+ this.lane = lane;
1444
1541
  }
1445
1542
  runAction(request, options) {
1446
- return this.request((worker) => worker.runAction(request, options));
1543
+ return this.pool.request(this, pieceKey(request), (worker) => worker.runAction(request, options));
1447
1544
  }
1448
1545
  resolveOptions(request, options) {
1449
- return this.request((worker) => worker.resolveOptions(request, options));
1546
+ return this.pool.request(this, pieceKey(request), (worker) => worker.resolveOptions(request, options));
1450
1547
  }
1451
1548
  checkConnection(request, options) {
1452
- return this.request((worker) => worker.checkConnection(request, options));
1549
+ return this.pool.request(this, pieceKey(request), (worker) => worker.checkConnection(request, options));
1453
1550
  }
1454
1551
  describePiece(request, options) {
1455
- return this.request((worker) => worker.describePiece(request, options));
1552
+ return this.pool.request(this, pieceKey(request), (worker) => worker.describePiece(request, options));
1456
1553
  }
1457
1554
  runTriggerHook(request, options) {
1458
- return this.request((worker) => worker.runTriggerHook(request, options));
1555
+ return this.pool.request(this, pieceKey(request), (worker) => worker.runTriggerHook(request, options));
1459
1556
  }
1460
1557
  close() {
1461
1558
  if (this.closed) return;
1462
1559
  this.closed = true;
1463
- this.pool.cancel(this);
1464
- this.worker?.dispose();
1465
- this.worker = void 0;
1466
- this.pool.forget(this);
1467
- this.giveBack();
1560
+ this.pool.closeSession(this);
1561
+ }
1562
+ /** @internal */
1563
+ isClosed() {
1564
+ return this.closed;
1468
1565
  }
1469
1566
  dispose() {
1470
1567
  this.close();
1471
1568
  }
1472
- async request(fn) {
1473
- const worker = await this.take();
1474
- if (this.closed) throw new PieceWorkerSessionClosedError();
1475
- return fn(worker);
1476
- }
1477
- take() {
1478
- if (this.closed) return Promise.reject(new PieceWorkerSessionClosedError());
1479
- if (this.worker) return Promise.resolve(this.worker);
1480
- this.taking ??= this.acquire();
1481
- return this.taking;
1482
- }
1483
- async acquire() {
1484
- const { telemetry } = this.pool;
1485
- await (telemetry ? telemetry.phase("worker.acquire", {}, () => this.pool.acquire(this)) : this.pool.acquire(this));
1486
- this.held = true;
1487
- if (this.closed) {
1488
- this.giveBack();
1489
- throw new PieceWorkerSessionClosedError();
1490
- }
1491
- this.worker = this.pool.build();
1492
- return this.worker;
1493
- }
1494
- giveBack() {
1495
- if (!this.held) return;
1496
- this.held = false;
1497
- this.pool.release();
1498
- }
1499
1569
  };
1500
1570
  var PieceWorkerPool = class {
1501
1571
  size;
1502
1572
  /** @internal */
1503
1573
  telemetry;
1504
1574
  maxQueueDepth;
1575
+ triggerSlots;
1576
+ designSlots;
1577
+ minWarm;
1578
+ maxRequests;
1579
+ idleMs;
1580
+ maxConcurrentForks;
1581
+ heapMb;
1582
+ memory;
1505
1583
  createWorker;
1506
1584
  workerOptions;
1507
- sessions = /* @__PURE__ */ new Set();
1508
- waiters = [];
1509
- inUse = 0;
1585
+ children = /* @__PURE__ */ new Set();
1586
+ waiters = {
1587
+ run: [],
1588
+ trigger: [],
1589
+ design: []
1590
+ };
1591
+ largestChild = FRESH_CHILD_BYTES;
1592
+ idleTimer;
1593
+ bootTimer;
1510
1594
  disposed = false;
1511
1595
  constructor(options = {}) {
1512
- const { size, maxQueueDepth, createWorker, ...workerOptions } = options;
1513
- this.size = Math.max(1, Math.trunc(size ?? DEFAULT_SIZE));
1596
+ const { size, maxQueueDepth, triggerSlots, designSlots, minWarm, maxRequests, idleMs, maxConcurrentForks, reactorThreads, memory, createWorker, ...workerOptions } = options;
1597
+ this.heapMb = workerOptions.heapMb ?? 256;
1598
+ this.size = Math.max(1, Math.trunc(size ?? defaultPoolSize({
1599
+ reactorThreads,
1600
+ heapMb: this.heapMb
1601
+ })));
1514
1602
  this.maxQueueDepth = Math.max(0, Math.trunc(maxQueueDepth ?? 0));
1603
+ this.triggerSlots = Math.max(1, Math.trunc(triggerSlots ?? DEFAULT_TRIGGER_SLOTS));
1604
+ this.designSlots = Math.max(1, Math.trunc(designSlots ?? DEFAULT_DESIGN_SLOTS));
1605
+ this.minWarm = Math.max(0, Math.trunc(minWarm ?? DEFAULT_MIN_WARM));
1606
+ this.maxRequests = Math.max(1, maxRequests ?? DEFAULT_MAX_REQUESTS);
1607
+ this.idleMs = Math.max(0, idleMs ?? DEFAULT_IDLE_MS);
1608
+ this.maxConcurrentForks = Math.max(1, maxConcurrentForks ?? DEFAULT_MAX_CONCURRENT_FORKS);
1609
+ this.memory = memory ?? cachedCgroupMemory();
1515
1610
  this.createWorker = createWorker ?? ((o) => new PieceWorker(o));
1516
- this.workerOptions = workerOptions;
1611
+ this.workerOptions = {
1612
+ ...workerOptions,
1613
+ heapMb: this.heapMb
1614
+ };
1517
1615
  this.telemetry = workerOptions.telemetry;
1518
1616
  }
1519
- session() {
1520
- if (this.disposed) throw new Error("Piece worker pool is disposed");
1521
- const session = new PieceWorkerSession(this);
1522
- this.sessions.add(session);
1523
- return session;
1617
+ session(lane = "run") {
1618
+ return new PieceWorkerSession(this, lane);
1524
1619
  }
1525
1620
  stats() {
1621
+ let active = 0;
1622
+ let idle = 0;
1623
+ for (const child of this.children) {
1624
+ if (child.lane !== "run") continue;
1625
+ if (child.busy) active++;
1626
+ else idle++;
1627
+ }
1526
1628
  return {
1527
1629
  size: this.size,
1528
- active: this.inUse,
1529
- waiting: this.waiters.length
1630
+ active,
1631
+ idle,
1632
+ waiting: this.waiters.run.length
1530
1633
  };
1531
1634
  }
1532
1635
  dispose() {
1636
+ if (this.disposed) return;
1533
1637
  this.disposed = true;
1534
- for (const waiter of this.waiters.splice(0)) waiter.reject(/* @__PURE__ */ new Error("Piece worker pool is disposed"));
1535
- for (const session of [...this.sessions]) session.close();
1536
- this.sessions.clear();
1537
- }
1538
- /** @internal — a session taking its slot. */
1539
- acquire(session) {
1540
- if (this.disposed) return Promise.reject(/* @__PURE__ */ new Error("Piece worker pool is disposed"));
1541
- if (this.inUse < this.size) {
1542
- this.inUse++;
1543
- return Promise.resolve();
1638
+ clearTimeout(this.idleTimer);
1639
+ this.idleTimer = void 0;
1640
+ clearTimeout(this.bootTimer);
1641
+ this.bootTimer = void 0;
1642
+ for (const lane of LANES) for (const waiter of this.waiters[lane].splice(0)) waiter.reject(new PieceWorkerDisposedError());
1643
+ for (const child of this.children) child.worker.dispose();
1644
+ this.children.clear();
1645
+ }
1646
+ /** @internal — one request on a leased child. */
1647
+ async request(session, key, fn) {
1648
+ if (session.isClosed()) throw new PieceWorkerSessionClosedError();
1649
+ const child = await this.lease(session, key);
1650
+ if (this.disposed) {
1651
+ this.giveBack(child, key);
1652
+ throw new PieceWorkerDisposedError();
1544
1653
  }
1545
- if (this.maxQueueDepth > 0 && this.waiters.length >= this.maxQueueDepth) return Promise.reject(new PieceWorkerPoolBusyError(this.size, this.waiters.length));
1546
- return new Promise((resolve, reject) => {
1547
- this.waiters.push({
1654
+ if (session.isClosed()) {
1655
+ this.giveBack(child, key);
1656
+ throw new PieceWorkerSessionClosedError();
1657
+ }
1658
+ try {
1659
+ return await fn(child.worker);
1660
+ } finally {
1661
+ this.giveBack(child, key);
1662
+ }
1663
+ }
1664
+ /** @internal — a session that closed. */
1665
+ closeSession(session) {
1666
+ const queue = this.waiters[session.lane];
1667
+ for (let i = queue.length - 1; i >= 0; i--) {
1668
+ if (queue[i].session !== session) continue;
1669
+ const [waiter] = queue.splice(i, 1);
1670
+ waiter.reject(new PieceWorkerSessionClosedError());
1671
+ }
1672
+ for (const child of this.children) if (child.busy && child.session === session) {
1673
+ child.doomed = true;
1674
+ child.worker.dispose();
1675
+ }
1676
+ }
1677
+ lease(session, key) {
1678
+ if (this.disposed) return Promise.reject(new PieceWorkerDisposedError());
1679
+ const queue = this.waiters[session.lane];
1680
+ const child = queue.length === 0 ? this.take(session.lane, key) : void 0;
1681
+ if (child) {
1682
+ child.session = session;
1683
+ return Promise.resolve(child);
1684
+ }
1685
+ if (session.lane === "run" && this.maxQueueDepth > 0 && queue.length >= this.maxQueueDepth) return Promise.reject(new PieceWorkerPoolBusyError(this.size, queue.length));
1686
+ const wait = new Promise((resolve, reject) => {
1687
+ queue.push({
1548
1688
  session,
1689
+ key,
1549
1690
  resolve,
1550
1691
  reject
1551
1692
  });
1552
1693
  });
1553
- }
1554
- /** @internal — a session that closed while queueing. */
1555
- cancel(session) {
1556
- const index = this.waiters.findIndex((w) => w.session === session);
1557
- if (index < 0) return;
1558
- const [waiter] = this.waiters.splice(index, 1);
1559
- waiter.reject(new PieceWorkerSessionClosedError());
1560
- }
1561
- /** @internal — a session giving its slot back. */
1562
- release() {
1563
- const next = this.waiters.shift();
1564
- if (next) {
1565
- next.resolve();
1566
- return;
1694
+ return this.telemetry ? this.telemetry.phase("worker.acquire", {}, () => wait) : wait;
1695
+ }
1696
+ take(lane, key) {
1697
+ let warm;
1698
+ let recent;
1699
+ for (const child of this.children) {
1700
+ if (child.lane !== lane || child.busy) continue;
1701
+ if (key && child.loaded.has(key)) {
1702
+ warm = child;
1703
+ break;
1704
+ }
1705
+ if (!recent || child.idleSince > recent.idleSince) recent = child;
1706
+ }
1707
+ const child = warm ?? recent ?? this.fork(lane);
1708
+ if (child) child.busy = true;
1709
+ return child;
1710
+ }
1711
+ fork(lane) {
1712
+ let inLane = 0;
1713
+ let booting = 0;
1714
+ let firstBootEnds = Infinity;
1715
+ const now = performance.now();
1716
+ for (const child of this.children) {
1717
+ if (child.lane === lane) inLane++;
1718
+ if (!child.replied && now - child.forkedAt < BOOT_WINDOW_MS) {
1719
+ booting++;
1720
+ firstBootEnds = Math.min(firstBootEnds, child.forkedAt + BOOT_WINDOW_MS);
1721
+ }
1722
+ }
1723
+ const capacity = {
1724
+ run: this.size,
1725
+ trigger: this.triggerSlots,
1726
+ design: this.designSlots
1727
+ }[lane];
1728
+ if (inLane >= capacity) return void 0;
1729
+ if (inLane > 0) {
1730
+ if (booting >= this.maxConcurrentForks) {
1731
+ this.redispatchAt(firstBootEnds - now);
1732
+ return;
1733
+ }
1734
+ if (!this.roomForChild()) return void 0;
1735
+ }
1736
+ const child = {
1737
+ worker: this.createWorker(this.workerOptions),
1738
+ lane,
1739
+ loaded: /* @__PURE__ */ new Set(),
1740
+ requests: 0,
1741
+ busy: false,
1742
+ forkedAt: now,
1743
+ replied: false,
1744
+ doomed: false,
1745
+ idleSince: 0,
1746
+ session: void 0
1747
+ };
1748
+ this.children.add(child);
1749
+ return child;
1750
+ }
1751
+ roomForChild() {
1752
+ const memory = this.memory();
1753
+ if (!memory) return true;
1754
+ return memory.used + this.largestChild <= FORK_HEADROOM * memory.limit;
1755
+ }
1756
+ giveBack(child, key) {
1757
+ child.busy = false;
1758
+ child.replied = true;
1759
+ child.session = void 0;
1760
+ child.requests++;
1761
+ child.idleSince = performance.now();
1762
+ const health = child.worker.health?.();
1763
+ if (health?.memory) this.largestChild = Math.max(this.largestChild, health.memory.rss);
1764
+ if (this.spent(child, health)) this.retire(child);
1765
+ else if (key) child.loaded.add(key);
1766
+ this.dispatch();
1767
+ this.scheduleIdleSweep();
1768
+ }
1769
+ spent(child, health) {
1770
+ if (this.disposed || child.doomed) return true;
1771
+ if (child.requests >= this.maxRequests) return true;
1772
+ if (!health) return false;
1773
+ if (!health.alive || health.tlsPoisoned) return true;
1774
+ return health.memory !== void 0 && health.memory.heapUsed > HEAP_RETIREMENT * this.heapMb * MIB;
1775
+ }
1776
+ retire(child) {
1777
+ this.children.delete(child);
1778
+ child.worker.dispose();
1779
+ }
1780
+ dispatch() {
1781
+ if (this.disposed) return;
1782
+ for (const lane of LANES) {
1783
+ const queue = this.waiters[lane];
1784
+ while (queue.length > 0) {
1785
+ const child = this.take(lane, queue[0].key);
1786
+ if (!child) break;
1787
+ const waiter = queue.shift();
1788
+ child.session = waiter.session;
1789
+ waiter.resolve(child);
1790
+ }
1567
1791
  }
1568
- this.inUse = Math.max(0, this.inUse - 1);
1569
- }
1570
- /** @internal — one child for one session. */
1571
- build() {
1572
- return this.createWorker(this.workerOptions);
1573
1792
  }
1574
- /** @internal — a session that closed itself. */
1575
- forget(session) {
1576
- this.sessions.delete(session);
1793
+ redispatchAt(ms) {
1794
+ if (this.bootTimer || this.disposed) return;
1795
+ this.bootTimer = setTimeout(() => {
1796
+ this.bootTimer = void 0;
1797
+ this.dispatch();
1798
+ }, Math.max(1, ms));
1799
+ this.bootTimer.unref();
1800
+ }
1801
+ scheduleIdleSweep() {
1802
+ if (this.idleTimer || this.disposed || this.idleMs === 0) return;
1803
+ if (this.retirable().length === 0) return;
1804
+ this.idleTimer = setTimeout(() => {
1805
+ this.idleTimer = void 0;
1806
+ this.sweepIdle();
1807
+ }, this.idleMs);
1808
+ this.idleTimer.unref();
1809
+ }
1810
+ sweepIdle() {
1811
+ const now = performance.now();
1812
+ for (const child of this.retirable()) if (now - child.idleSince >= this.idleMs) this.retire(child);
1813
+ this.scheduleIdleSweep();
1814
+ }
1815
+ retirable() {
1816
+ const idle = (lane) => [...this.children].filter((child) => child.lane === lane && !child.busy).sort((a, b) => a.idleSince - b.idleSince);
1817
+ const run = idle("run");
1818
+ return [...run.slice(0, Math.max(0, run.length - this.minWarm)), ...idle("design")];
1577
1819
  }
1578
1820
  };
1579
1821
  //#endregion
@@ -2653,7 +2895,7 @@ const manualTrigger = createTrigger({
2653
2895
  });
2654
2896
  //#endregion
2655
2897
  //#region src/pieces/core/version.ts
2656
- const CORE_PIECE_VERSION = "6.2.3-dev.55";
2898
+ const CORE_PIECE_VERSION = "6.2.3-dev.57";
2657
2899
  //#endregion
2658
2900
  //#region src/pieces/core/index.ts
2659
2901
  const corePiece = createPiece({
@@ -3576,6 +3818,277 @@ function installedPieces() {
3576
3818
  return [...builtinLocalPieces(), ...packagePieces.entries().filter((piece) => !isBuiltinPiece(piece.name))];
3577
3819
  }
3578
3820
  //#endregion
3579
- export { isPrivateAddress as $, declaredConnectionIds as A, stepBlock as B, builtinPiece as C, AttachmentCache as D, CORE_PIECE_VERSION as E, InMemorySecretProvider as F, PieceWorkerError as G, PieceWorkerPool as H, SecretDeletedError as I, redact as J, PieceWorkerTimeoutError as K, SecretNotFoundError as L, shapeConnection as M, resolveStepInput as N, BoundConnectionResolver as O, expressionsIn as P, DEFAULT_EGRESS_POLICY as Q, parseSecretRef as R, blockPorts as S, CORE_PIECE_NAME$1 as T, PieceWorker as U, triggerBlock as V, PieceWorkerDisposedError as W, rememberSecrets as X, redactMessage as Y, secretsFor as Z, resolvedBlock as _, isIndeterminateError as _t, RUN_DEADLINE_ERROR_NAME as a, pieceModuleRef as at, resolutionOf as b, canonicalJson as c, UnsupportedPieceFeatureError as ct, DynamicPropertiesError as d, unsupportedTrigger as dt, parseAddress as et, checkDynamicProperties as f, ensurePieceBundle as ft, isHostBound as g, setPieceRegistryUrl as gt, CompositeBlockExecutor as h, pieceRegistrySource as ht, packagePieces as i, bundleResolver as it, declaredReactorConnectionIds as j, StaticConnectionResolver as k, hashOf as l, authMethodFor as lt, ActivepiecesBlockExecutor as m, npmRegistryBaseUrl as mt, installedPiece as n, FileTooLargeError as nt, errorNameOf as o, sourcedResolver as ot, effectiveRetryPolicy as p, fetchPieceBundle as pt, containsRedactedMarker as q, installedPieces as r, maxFileBytes as rt, runWorkflow as s, buildDescriptor as st, PieceRegistry as t, extractDedupeKey as tt, stepConfigHash as u, unsupportedAuth as ut, storeHandlers as v, describeBuiltinPiece as w, unpinnedNote as x, blockLabel as y, secretRefFromId as z };
3821
+ //#region src/telemetry.ts
3822
+ const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
3823
+ var WorkflowTelemetry = class {
3824
+ tracer;
3825
+ meter;
3826
+ runs;
3827
+ runDuration;
3828
+ stepDuration;
3829
+ phaseDuration;
3830
+ dispatcherGauges;
3831
+ poolGauges;
3832
+ constructor(options = {}) {
3833
+ this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
3834
+ this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
3835
+ this.runs = this.meter.createCounter("workflow.runs", {
3836
+ description: "Runs finished, by status and trigger kind",
3837
+ unit: "{run}"
3838
+ });
3839
+ this.runDuration = this.meter.createHistogram("workflow.run.duration", {
3840
+ description: "Run duration from fire to the journal closing it",
3841
+ unit: "ms"
3842
+ });
3843
+ this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
3844
+ description: "Step duration, by piece and status",
3845
+ unit: "ms"
3846
+ });
3847
+ this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
3848
+ description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
3849
+ unit: "ms"
3850
+ });
3851
+ }
3852
+ phase(name, attributes, fn, parent) {
3853
+ const started = performance.now();
3854
+ return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
3855
+ }
3856
+ workerTimings(request, timings, forkedAt, ipc) {
3857
+ const under = (span) => trace.setSpan(context.active(), span);
3858
+ const child = (name, start, end, parent = request, attributes) => {
3859
+ const span = this.tracer.startSpan(`workflow.worker.${name}`, {
3860
+ startTime: start,
3861
+ ...attributes ? { attributes } : {}
3862
+ }, under(parent));
3863
+ span.end(end);
3864
+ this.recordPhase(`worker.${name}`, end - start);
3865
+ return span;
3866
+ };
3867
+ const ready = finite(timings.ready);
3868
+ if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
3869
+ const received = finite(timings.received);
3870
+ const sent = finite(timings.sent);
3871
+ const delivered = Math.max(ipc.sent, ready ?? 0);
3872
+ if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
3873
+ const byName = /* @__PURE__ */ new Map();
3874
+ const phases = Array.isArray(timings.phases) ? timings.phases : [];
3875
+ for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
3876
+ const parent = phase.parent ? byName.get(phase.parent) : void 0;
3877
+ byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
3878
+ }
3879
+ if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
3880
+ request.setAttributes(workerMemory(timings.memory));
3881
+ }
3882
+ recordPhase(name, ms) {
3883
+ this.phaseDuration.record(ms, { phase: name });
3884
+ }
3885
+ span(name, attributes, fn, parent = context.active()) {
3886
+ return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
3887
+ try {
3888
+ return await fn(span);
3889
+ } catch (error) {
3890
+ failSpan(span, error);
3891
+ throw error;
3892
+ } finally {
3893
+ span.end();
3894
+ }
3895
+ });
3896
+ }
3897
+ recordRun(ms, status, triggerKind) {
3898
+ const attributes = {
3899
+ "run.status": status,
3900
+ "trigger.kind": metricTriggerKind(triggerKind)
3901
+ };
3902
+ this.runs.add(1, attributes);
3903
+ this.runDuration.record(ms, attributes);
3904
+ }
3905
+ recordStep(ms, pieceName, status) {
3906
+ this.stepDuration.record(ms, {
3907
+ "piece.name": pieceName,
3908
+ "step.status": status
3909
+ });
3910
+ }
3911
+ observeWorkerPool(stats) {
3912
+ this.poolGauges ??= this.createPoolGauges();
3913
+ const { size, active, idle, waiting } = this.poolGauges;
3914
+ const observables = [
3915
+ size,
3916
+ active,
3917
+ idle,
3918
+ waiting
3919
+ ];
3920
+ const callback = (result) => {
3921
+ const current = stats();
3922
+ result.observe(size, current.size);
3923
+ result.observe(active, current.active);
3924
+ result.observe(idle, current.idle);
3925
+ result.observe(waiting, current.waiting);
3926
+ };
3927
+ this.meter.addBatchObservableCallback(callback, observables);
3928
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
3929
+ }
3930
+ observeDispatcher(stats) {
3931
+ this.dispatcherGauges ??= {
3932
+ inFlight: this.meter.createObservableGauge("workflow.runs.in_flight", {
3933
+ description: "Runs admitted and not yet finished",
3934
+ unit: "{run}"
3935
+ }),
3936
+ waiting: this.meter.createObservableGauge("workflow.runs.waiting", {
3937
+ description: "Runs waiting for admission across workflows",
3938
+ unit: "{run}"
3939
+ }),
3940
+ loop: this.meter.createObservableGauge("workflow.host.event_loop.utilization", {
3941
+ description: "Main thread busy share over the last second",
3942
+ unit: "1"
3943
+ })
3944
+ };
3945
+ const { inFlight, waiting, loop } = this.dispatcherGauges;
3946
+ const observables = [
3947
+ inFlight,
3948
+ waiting,
3949
+ loop
3950
+ ];
3951
+ const callback = (result) => {
3952
+ const current = stats();
3953
+ result.observe(inFlight, current.inFlight);
3954
+ result.observe(waiting, current.waiting);
3955
+ result.observe(loop, current.utilization);
3956
+ };
3957
+ this.meter.addBatchObservableCallback(callback, observables);
3958
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
3959
+ }
3960
+ createPoolGauges() {
3961
+ const gauge = (name, description) => this.meter.createObservableGauge(name, {
3962
+ description,
3963
+ unit: "{worker}"
3964
+ });
3965
+ return {
3966
+ size: gauge("workflow.worker.pool.size", "Most run children"),
3967
+ active: gauge("workflow.worker.pool.active", "Children serving a request"),
3968
+ idle: gauge("workflow.worker.pool.idle", "Warm children waiting"),
3969
+ waiting: gauge("workflow.worker.pool.waiting", "Requests waiting for a child")
3970
+ };
3971
+ }
3972
+ };
3973
+ const WORKER_PHASES = /* @__PURE__ */ new Set([
3974
+ "piece.load",
3975
+ "reactor.open",
3976
+ "models",
3977
+ "action"
3978
+ ]);
3979
+ const WORKER_ATTRIBUTES = {
3980
+ "piece.cached": "boolean",
3981
+ "document.type": "string"
3982
+ };
3983
+ const MAX_ATTRIBUTE_LENGTH = 256;
3984
+ function finite(value) {
3985
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3986
+ }
3987
+ function knownPhase(phase) {
3988
+ if (typeof phase !== "object" || phase === null) return false;
3989
+ const { name, start, end, parent } = phase;
3990
+ return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && finite(start) !== void 0 && finite(end) !== void 0 && end >= start;
3991
+ }
3992
+ function phaseAttributes(attributes) {
3993
+ if (typeof attributes !== "object" || attributes === null) return void 0;
3994
+ const kept = {};
3995
+ for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
3996
+ const value = attributes[key];
3997
+ if (typeof value !== type) continue;
3998
+ kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
3999
+ }
4000
+ return Object.keys(kept).length > 0 ? kept : void 0;
4001
+ }
4002
+ const MB = 2 ** 20;
4003
+ function workerMemory(memory) {
4004
+ if (typeof memory !== "object" || memory === null) return {};
4005
+ const attributes = {};
4006
+ for (const [key, name] of [
4007
+ ["rss", "worker.memory.rss_mb"],
4008
+ ["heapUsed", "worker.memory.heap_used_mb"],
4009
+ ["heapTotal", "worker.memory.heap_total_mb"],
4010
+ ["external", "worker.memory.external_mb"]
4011
+ ]) {
4012
+ const bytes = finite(memory[key]);
4013
+ if (bytes !== void 0) attributes[name] = bytes / MB;
4014
+ }
4015
+ return attributes;
4016
+ }
4017
+ function metricTriggerKind(triggerKind) {
4018
+ return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
4019
+ }
4020
+ function metricPieceName(step, resolution) {
4021
+ return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
4022
+ }
4023
+ function tracedSteps(executor, telemetry) {
4024
+ return { execute(execution) {
4025
+ const { step } = execution;
4026
+ const started = performance.now();
4027
+ let status = "FAILED";
4028
+ let resolution;
4029
+ return telemetry.span("workflow.step", {
4030
+ "step.id": step.id,
4031
+ "step.key": step.key,
4032
+ "piece.name": step.pieceName,
4033
+ "action.name": step.actionName
4034
+ }, async (span) => {
4035
+ try {
4036
+ const result = await executor.execute(execution);
4037
+ resolution = result.resolution;
4038
+ status = "SUCCEEDED";
4039
+ const version = resolution?.resolved?.version;
4040
+ if (version) span.setAttribute("piece.version", version);
4041
+ return result;
4042
+ } catch (error) {
4043
+ resolution = resolutionOf(error);
4044
+ throw error;
4045
+ }
4046
+ }).finally(() => {
4047
+ telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
4048
+ });
4049
+ } };
4050
+ }
4051
+ function failSpan(span, error) {
4052
+ span.recordException(error instanceof Error ? error : String(error));
4053
+ span.setStatus({
4054
+ code: SpanStatusCode.ERROR,
4055
+ message: error instanceof Error ? error.message : String(error)
4056
+ });
4057
+ }
4058
+ function tracedMethods(target, telemetry, prefix, parent) {
4059
+ return new Proxy(target, { get(object, property, receiver) {
4060
+ const value = Reflect.get(object, property, receiver);
4061
+ if (typeof value !== "function" || typeof property !== "string") return value;
4062
+ const phase = `${prefix}.${property}`;
4063
+ return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
4064
+ const started = performance.now();
4065
+ const end = (error) => {
4066
+ if (error !== void 0) failSpan(span, error);
4067
+ span.end();
4068
+ telemetry.recordPhase(phase, performance.now() - started);
4069
+ };
4070
+ let result;
4071
+ try {
4072
+ result = value.apply(object, args);
4073
+ } catch (error) {
4074
+ end(error);
4075
+ throw error;
4076
+ }
4077
+ if (!(result instanceof Promise)) {
4078
+ end();
4079
+ return result;
4080
+ }
4081
+ return result.then((resolved) => {
4082
+ end();
4083
+ return resolved;
4084
+ }, (error) => {
4085
+ end(error);
4086
+ throw error;
4087
+ });
4088
+ });
4089
+ } });
4090
+ }
4091
+ //#endregion
4092
+ export { rememberSecrets as $, AttachmentCache as A, SecretNotFoundError as B, blockLabel as C, describeBuiltinPiece as D, builtinPiece as E, shapeConnection as F, PieceWorkerPool as G, secretRefFromId as H, resolveStepInput as I, PieceWorkerError as J, PieceWorker as K, expressionsIn as L, StaticConnectionResolver as M, declaredConnectionIds as N, CORE_PIECE_NAME$1 as O, declaredReactorConnectionIds as P, redactMessage as Q, InMemorySecretProvider as R, storeHandlers as S, blockPorts as T, stepBlock as U, parseSecretRef as V, triggerBlock as W, containsRedactedMarker as X, PieceWorkerTimeoutError as Y, redact as Z, effectiveRetryPolicy as _, npmRegistryBaseUrl as _t, PieceRegistry as a, FileTooLargeError as at, isHostBound as b, isIndeterminateError as bt, packagePieces as c, pieceModuleRef as ct, runWorkflow as d, UnsupportedPieceFeatureError as dt, secretsFor as et, canonicalJson as f, authMethodFor as ft, checkDynamicProperties as g, fetchPieceBundle as gt, DynamicPropertiesError as h, ensurePieceBundle as ht, tracedSteps as i, extractDedupeKey as it, BoundConnectionResolver as j, CORE_PIECE_VERSION as k, RUN_DEADLINE_ERROR_NAME as l, sourcedResolver as lt, stepConfigHash as m, unsupportedTrigger as mt, WorkflowTelemetry as n, isPrivateAddress as nt, installedPiece as o, maxFileBytes as ot, hashOf as p, unsupportedAuth as pt, PieceWorkerDisposedError as q, tracedMethods as r, parseAddress as rt, installedPieces as s, bundleResolver as st, TELEMETRY_SCOPE as t, DEFAULT_EGRESS_POLICY as tt, errorNameOf as u, buildDescriptor as ut, ActivepiecesBlockExecutor as v, pieceRegistrySource as vt, unpinnedNote as w, resolvedBlock as x, CompositeBlockExecutor as y, setPieceRegistryUrl as yt, SecretDeletedError as z };
3580
4093
 
3581
- //# sourceMappingURL=piece-registry-CsBrnpGl.js.map
4094
+ //# sourceMappingURL=telemetry-DZYSsZSP.js.map