@axiom-lattice/core 6.2.0 → 6.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1660,10 +1660,12 @@ __export(index_exports, {
1660
1660
  InMemoryChunkBuffer: () => InMemoryChunkBuffer,
1661
1661
  InMemoryCollectionStore: () => InMemoryCollectionStore,
1662
1662
  InMemoryDatabaseConfigStore: () => InMemoryDatabaseConfigStore,
1663
+ InMemoryLeaseStore: () => InMemoryLeaseStore,
1663
1664
  InMemoryMailboxStore: () => InMemoryMailboxStore,
1664
1665
  InMemoryMenuStore: () => InMemoryMenuStore,
1665
1666
  InMemoryModelProviderStore: () => InMemoryModelProviderStore,
1666
1667
  InMemoryOpenAuditStore: () => InMemoryOpenAuditStore,
1668
+ InMemoryOutboxStore: () => InMemoryOutboxStore,
1667
1669
  InMemoryProjectBotMembershipStore: () => InMemoryProjectBotMembershipStore,
1668
1670
  InMemoryProjectMembershipStore: () => InMemoryProjectMembershipStore,
1669
1671
  InMemoryProjectRoomCoordinator: () => InMemoryProjectRoomCoordinator,
@@ -1683,6 +1685,7 @@ __export(index_exports, {
1683
1685
  LatticeEval: () => LatticeEval,
1684
1686
  LatticeEvalProject: () => LatticeEvalProject,
1685
1687
  LatticeEvalSuite: () => LatticeEvalSuite,
1688
+ LeaseAcquisitionTimeoutError: () => LeaseAcquisitionTimeoutError,
1686
1689
  LocalSandboxInstance: () => LocalSandboxInstance,
1687
1690
  LocalSandboxProvider: () => LocalSandboxProvider,
1688
1691
  LoggerLatticeManager: () => LoggerLatticeManager,
@@ -1705,7 +1708,10 @@ __export(index_exports, {
1705
1708
  MicrosandboxServiceClient: () => MicrosandboxServiceClient,
1706
1709
  ModelLatticeManager: () => ModelLatticeManager,
1707
1710
  MysqlDatabase: () => MysqlDatabase,
1711
+ OutboxPoller: () => OutboxPoller,
1708
1712
  PACK_DIRECTORY_TIMEOUT_SECONDS: () => PACK_DIRECTORY_TIMEOUT_SECONDS,
1713
+ PLATFORM_SECRET_ENV_VARS: () => PLATFORM_SECRET_ENV_VARS,
1714
+ PLUGIN_CATALOG_CHANGED_TOPIC: () => PLUGIN_CATALOG_CHANGED_TOPIC,
1709
1715
  PersonalAssistantConfig: () => PersonalAssistantConfig,
1710
1716
  PinoLoggerClient: () => PinoLoggerClient,
1711
1717
  PluginRegistry: () => PluginRegistry,
@@ -1726,6 +1732,8 @@ __export(index_exports, {
1726
1732
  RepeatedToolCallError: () => RepeatedToolCallError,
1727
1733
  RoomAgentMessageService: () => RoomAgentMessageService,
1728
1734
  SAFE_PATH_COMPONENT: () => SAFE_PATH_COMPONENT,
1735
+ SANDBOX_OPEN_GATEWAY_URL_ENV: () => SANDBOX_OPEN_GATEWAY_URL_ENV,
1736
+ SANDBOX_OPEN_TOKEN_ENV: () => SANDBOX_OPEN_TOKEN_ENV,
1729
1737
  STTModelLattice: () => STTModelLattice,
1730
1738
  STTModelLatticeManager: () => STTModelLatticeManager,
1731
1739
  SandboxFilesystem: () => SandboxFilesystem,
@@ -1766,11 +1774,13 @@ __export(index_exports, {
1766
1774
  appendReconciliationWarning: () => appendReconciliationWarning,
1767
1775
  appendUiFence: () => appendUiFence,
1768
1776
  assertSafeUpstreamUrl: () => assertSafeUpstreamUrl,
1777
+ buildBundleFromDist: () => buildBundleFromDist,
1769
1778
  buildGrepResultsDict: () => buildGrepResultsDict,
1770
1779
  buildInput: () => buildInput,
1771
1780
  buildNamedVolumeName: () => buildNamedVolumeName,
1772
1781
  buildRemotePackDirectoryCommand: () => buildRemotePackDirectoryCommand,
1773
1782
  buildSandboxMetadataEnv: () => buildSandboxMetadataEnv,
1783
+ buildSandboxOpenEnv: () => buildSandboxOpenEnv,
1774
1784
  buildSkillFile: () => buildSkillFile,
1775
1785
  buildStateAnnotation: () => buildStateAnnotation,
1776
1786
  buildTableName: () => buildTableName,
@@ -1779,11 +1789,14 @@ __export(index_exports, {
1779
1789
  checkEmptyContent: () => checkEmptyContent,
1780
1790
  clearEncryptionKeyCache: () => clearEncryptionKeyCache,
1781
1791
  clearEvalRunService: () => clearEvalRunService,
1792
+ clearSandboxOpenCredentialProviderForTests: () => clearSandboxOpenCredentialProviderForTests,
1782
1793
  collectionLatticeManager: () => collectionLatticeManager,
1783
1794
  compileInternal: () => compileInternal,
1784
1795
  compileSandboxPlugin: () => compileSandboxPlugin,
1785
1796
  compileWorkflow: () => compileWorkflow,
1786
1797
  computeSandboxName: () => computeSandboxName,
1798
+ configureCoordinationLeaseStore: () => configureCoordinationLeaseStore,
1799
+ configurePluginCatalogSync: () => configurePluginCatalogSync,
1787
1800
  configureStores: () => configureStores,
1788
1801
  connectAllChannels: () => connectAllChannels,
1789
1802
  convertToConnection: () => convertToConnection,
@@ -1815,6 +1828,7 @@ __export(index_exports, {
1815
1828
  createSandboxResolver: () => createSandboxResolver,
1816
1829
  createSchedulerMiddleware: () => createSchedulerMiddleware,
1817
1830
  createSharePayload: () => createSharePayload,
1831
+ createShellExecTool: () => createShellExecTool,
1818
1832
  createTaskExecutionService: () => createTaskExecutionService,
1819
1833
  createTaskLifecycleService: () => createTaskLifecycleService,
1820
1834
  createTaskMiddleware: () => createTaskMiddleware,
@@ -1861,6 +1875,7 @@ __export(index_exports, {
1861
1875
  getCheckpointSaver: () => getCheckpointSaver,
1862
1876
  getChunkBuffer: () => getChunkBuffer,
1863
1877
  getCollectionEntryCount: () => getCollectionEntryCount,
1878
+ getCoordinationLeaseStore: () => getCoordinationLeaseStore,
1864
1879
  getEmbeddingsClient: () => getEmbeddingsClient,
1865
1880
  getEmbeddingsLattice: () => getEmbeddingsLattice,
1866
1881
  getEncryptionKey: () => getEncryptionKey,
@@ -1872,15 +1887,18 @@ __export(index_exports, {
1872
1887
  getOrCreateCollectionVectorStore: () => getOrCreateCollectionVectorStore,
1873
1888
  getPluginSkillManifestVersion: () => getPluginSkillManifestVersion,
1874
1889
  getQueueLattice: () => getQueueLattice,
1890
+ getReplicaId: () => getReplicaId,
1875
1891
  getRuntimeCapabilityConfig: () => getRuntimeCapabilityConfig,
1876
1892
  getRuntimeSkillSelection: () => getRuntimeSkillSelection,
1877
1893
  getSTTClient: () => getSTTClient,
1878
1894
  getSTTClientWithTenant: () => getSTTClientWithTenant,
1879
1895
  getSTTModelLattice: () => getSTTModelLattice,
1880
1896
  getSandBoxManager: () => getSandBoxManager,
1897
+ getSandboxOpenCredential: () => getSandboxOpenCredential,
1881
1898
  getSandboxPluginCatalog: () => getSandboxPluginCatalog,
1882
1899
  getScheduleLattice: () => getScheduleLattice,
1883
1900
  getStoreLattice: () => getStoreLattice,
1901
+ getThreadRunOwner: () => getThreadRunOwner,
1884
1902
  getToolClient: () => getToolClient,
1885
1903
  getToolClientWithTenant: () => getToolClientWithTenant,
1886
1904
  getToolDefinition: () => getToolDefinition,
@@ -1934,6 +1952,8 @@ __export(index_exports, {
1934
1952
  projectMutationMutex: () => projectMutationMutex,
1935
1953
  projectRoomPlugin: () => projectRoomPlugin,
1936
1954
  projectTaskAdmissionNotifier: () => projectTaskAdmissionNotifier,
1955
+ provisionSandboxOpenClient: () => provisionSandboxOpenClient,
1956
+ publishPluginCatalogChanged: () => publishPluginCatalogChanged,
1937
1957
  queueLatticeManager: () => queueLatticeManager,
1938
1958
  readLoopBreakerConfig: () => readLoopBreakerConfig,
1939
1959
  reconcileAllTenantTaskRuns: () => reconcileAllTenantTaskRuns,
@@ -1952,6 +1972,7 @@ __export(index_exports, {
1952
1972
  registerModelLatticeForTenant: () => registerModelLatticeForTenant,
1953
1973
  registerQueueLattice: () => registerQueueLattice,
1954
1974
  registerSTTModelLattice: () => registerSTTModelLattice,
1975
+ registerSandboxOpenCredentialProvider: () => registerSandboxOpenCredentialProvider,
1955
1976
  registerSandboxProviderType: () => registerSandboxProviderType,
1956
1977
  registerScheduleLattice: () => registerScheduleLattice,
1957
1978
  registerStoreLattice: () => registerStoreLattice,
@@ -1962,6 +1983,7 @@ __export(index_exports, {
1962
1983
  registerWorkflowRun: () => registerWorkflowRun,
1963
1984
  removeCollectionVectorStore: () => removeCollectionVectorStore,
1964
1985
  renderTemplate: () => renderTemplate,
1986
+ resetReplicaIdForTests: () => resetReplicaIdForTests,
1965
1987
  resolveEnabledPluginSkill: () => resolveEnabledPluginSkill,
1966
1988
  resolveJudgeModelKey: () => resolveJudgeModelKey,
1967
1989
  resolvePath: () => resolvePath,
@@ -1969,6 +1991,7 @@ __export(index_exports, {
1969
1991
  resolveSandboxPluginConnections: () => resolveSandboxPluginConnections,
1970
1992
  resolveTenantPluginSkill: () => resolveTenantPluginSkill,
1971
1993
  roomCoordinatorConfig: () => roomCoordinatorConfig,
1994
+ runAsLeader: () => runAsLeader,
1972
1995
  runRemotePackDirectory: () => runRemotePackDirectory,
1973
1996
  safeTaskResponseId: () => safeTaskResponseId,
1974
1997
  sandboxLatticeManager: () => sandboxLatticeManager,
@@ -1987,11 +2010,15 @@ __export(index_exports, {
1987
2010
  storeLatticeManager: () => storeLatticeManager,
1988
2011
  sttModelLatticeManager: () => sttModelLatticeManager,
1989
2012
  taskLifecycleNotifier: () => taskLifecycleNotifier,
2013
+ taskRunLeaseKey: () => taskRunLeaseKey,
1990
2014
  taskRunRegistry: () => taskRunRegistry,
2015
+ threadRunLeaseKey: () => threadRunLeaseKey,
1991
2016
  toJsonSchema: () => toJsonSchema,
1992
2017
  toSafeStateExpr: () => toSafeStateExpr,
1993
2018
  toolLatticeManager: () => toolLatticeManager,
1994
2019
  truncateIfTooLong: () => truncateIfTooLong,
2020
+ tryAcquireTaskRun: () => tryAcquireTaskRun,
2021
+ tryAcquireThreadRun: () => tryAcquireThreadRun,
1995
2022
  unregisterTeammateAgent: () => unregisterTeammateAgent,
1996
2023
  unregisterWorkflowRun: () => unregisterWorkflowRun,
1997
2024
  updateFileData: () => updateFileData,
@@ -2007,6 +2034,7 @@ __export(index_exports, {
2007
2034
  validateToolInput: () => validateToolInput,
2008
2035
  vectorStoreLatticeManager: () => vectorStoreLatticeManager,
2009
2036
  vectorStoreProviderManager: () => vectorStoreProviderManager,
2037
+ withLease: () => withLease,
2010
2038
  withTimeout: () => withTimeout2,
2011
2039
  wrapMcpToolWithUi: () => wrapMcpToolWithUi
2012
2040
  });
@@ -4100,7 +4128,7 @@ var InMemoryModelProviderStore = class {
4100
4128
  */
4101
4129
  async listProviders(tenantId2) {
4102
4130
  return Array.from(this.providers.values()).filter(
4103
- (provider) => provider.tenantId === tenantId2
4131
+ (provider2) => provider2.tenantId === tenantId2
4104
4132
  );
4105
4133
  }
4106
4134
  /**
@@ -4114,7 +4142,7 @@ var InMemoryModelProviderStore = class {
4114
4142
  */
4115
4143
  async getProviderByName(tenantId2, name) {
4116
4144
  const providers = await this.listProviders(tenantId2);
4117
- return providers.find((provider) => provider.name === name) || null;
4145
+ return providers.find((provider2) => provider2.name === name) || null;
4118
4146
  }
4119
4147
  /**
4120
4148
  * Create a new provider configuration
@@ -4298,8 +4326,8 @@ var InMemoryModelProviderStore = class {
4298
4326
  * Clear providers and models for a specific tenant
4299
4327
  */
4300
4328
  clearByTenant(tenantId2) {
4301
- for (const [key5, provider] of Array.from(this.providers.entries())) {
4302
- if (provider.tenantId === tenantId2) {
4329
+ for (const [key5, provider2] of Array.from(this.providers.entries())) {
4330
+ if (provider2.tenantId === tenantId2) {
4303
4331
  this.providers.delete(key5);
4304
4332
  }
4305
4333
  }
@@ -5175,6 +5203,123 @@ function entryKey(tenantId2, userId, endpoint) {
5175
5203
  return JSON.stringify([tenantId2, userId, endpoint]);
5176
5204
  }
5177
5205
 
5206
+ // src/store_lattice/InMemoryLeaseStore.ts
5207
+ var InMemoryLeaseStore = class {
5208
+ /**
5209
+ * @param now Injectable clock in epoch milliseconds (tests use a fake clock).
5210
+ */
5211
+ constructor(now = Date.now) {
5212
+ this.leases = /* @__PURE__ */ new Map();
5213
+ this.now = now;
5214
+ }
5215
+ /** Atomically acquires a free/expired key (or the caller's own key). */
5216
+ async tryAcquire(key5, holder, ttlMs) {
5217
+ const existing = this.leases.get(key5);
5218
+ if (existing && Date.parse(existing.expiresAt) > this.now()) {
5219
+ if (existing.holder !== holder) return false;
5220
+ this.leases.set(key5, { ...existing, expiresAt: new Date(this.now() + ttlMs).toISOString() });
5221
+ return true;
5222
+ }
5223
+ const acquiredAt = existing && existing.holder === holder ? existing.acquiredAt : new Date(this.now()).toISOString();
5224
+ this.leases.set(key5, {
5225
+ key: key5,
5226
+ holder,
5227
+ acquiredAt,
5228
+ expiresAt: new Date(this.now() + ttlMs).toISOString()
5229
+ });
5230
+ return true;
5231
+ }
5232
+ /** Extends a lease only while `holder` still owns it. */
5233
+ async renew(key5, holder, ttlMs) {
5234
+ const existing = this.leases.get(key5);
5235
+ if (!existing || existing.holder !== holder) return false;
5236
+ this.leases.set(key5, { ...existing, expiresAt: new Date(this.now() + ttlMs).toISOString() });
5237
+ return true;
5238
+ }
5239
+ /** Deletes a lease only when `holder` owns it. */
5240
+ async release(key5, holder) {
5241
+ const existing = this.leases.get(key5);
5242
+ if (!existing || existing.holder !== holder) return false;
5243
+ this.leases.delete(key5);
5244
+ return true;
5245
+ }
5246
+ /** Returns the live lease, pruning an expired one. */
5247
+ async get(key5) {
5248
+ const existing = this.leases.get(key5);
5249
+ if (!existing) return null;
5250
+ if (Date.parse(existing.expiresAt) <= this.now()) {
5251
+ this.leases.delete(key5);
5252
+ return null;
5253
+ }
5254
+ return { ...existing };
5255
+ }
5256
+ };
5257
+
5258
+ // src/store_lattice/InMemoryOutboxStore.ts
5259
+ var InMemoryOutboxStore = class {
5260
+ /**
5261
+ * Creates an in-memory outbox store.
5262
+ *
5263
+ * @param now Optional clock override (defaults to the system clock); it stamps
5264
+ * `createdAt` and is used to evaluate age-based retention.
5265
+ */
5266
+ constructor(now = () => /* @__PURE__ */ new Date()) {
5267
+ this.events = [];
5268
+ this.counter = 0;
5269
+ this.now = now;
5270
+ }
5271
+ /**
5272
+ * Appends one event, assigning the next monotonic id.
5273
+ * @param input Topic, JSON-safe payload, and authoring replica id.
5274
+ * @returns A defensive copy of the stored event.
5275
+ */
5276
+ async append(input) {
5277
+ const event = {
5278
+ id: String(++this.counter),
5279
+ topic: input.topic,
5280
+ payload: input.payload,
5281
+ originReplicaId: input.originReplicaId,
5282
+ createdAt: this.now().toISOString()
5283
+ };
5284
+ this.events.push(event);
5285
+ return { ...event };
5286
+ }
5287
+ /**
5288
+ * Lists events strictly newer than `cursor`, ascending, bounded by `limit`.
5289
+ * @param cursor Last-seen id, or `null` to read from the start.
5290
+ * @param limit Maximum number of events to return.
5291
+ * @returns Defensive copies of the matching events.
5292
+ */
5293
+ async listAfter(cursor, limit) {
5294
+ const floor = cursor === null ? 0 : Number(cursor);
5295
+ return this.events.filter((event) => Number(event.id) > floor).slice(0, limit).map((event) => ({ ...event }));
5296
+ }
5297
+ /**
5298
+ * Returns the newest event id.
5299
+ * @returns The last id, or `null` when the log is empty.
5300
+ */
5301
+ async latestId() {
5302
+ return this.events.length === 0 ? null : this.events[this.events.length - 1].id;
5303
+ }
5304
+ /**
5305
+ * Deletes events older than `olderThanMs`, evaluated against this store's clock.
5306
+ *
5307
+ * @param olderThanMs Age threshold in milliseconds.
5308
+ * @returns The number of events removed.
5309
+ */
5310
+ async pruneOlderThan(olderThanMs) {
5311
+ const cutoff = this.now().getTime() - olderThanMs;
5312
+ let removed = 0;
5313
+ for (let i = this.events.length - 1; i >= 0; i -= 1) {
5314
+ if (new Date(this.events[i].createdAt).getTime() < cutoff) {
5315
+ this.events.splice(i, 1);
5316
+ removed += 1;
5317
+ }
5318
+ }
5319
+ return removed;
5320
+ }
5321
+ };
5322
+
5178
5323
  // src/store_lattice/InMemoryUserStore.ts
5179
5324
  var InMemoryUserStore = class {
5180
5325
  constructor() {
@@ -5378,7 +5523,8 @@ function isMessageContentValue(value) {
5378
5523
 
5379
5524
  // src/store_lattice/InMemoryThreadMessageQueueStore.ts
5380
5525
  var InMemoryThreadMessageQueueStore = class {
5381
- constructor() {
5526
+ constructor(now = Date.now) {
5527
+ this.now = now;
5382
5528
  this.messages = /* @__PURE__ */ new Map();
5383
5529
  this.messageIdCounter = 0;
5384
5530
  this.capacityTail = Promise.resolve();
@@ -5485,6 +5631,36 @@ var InMemoryThreadMessageQueueStore = class {
5485
5631
  const threadMessages = this.getMessagesForThread(threadId);
5486
5632
  return threadMessages.filter((msg) => msg.status === "processing" && this.inScope(msg, scope)).sort((a, b) => (b.priority || 0) - (a.priority || 0) || a.sequence - b.sequence).map(clonePendingMessage);
5487
5633
  }
5634
+ async claimPendingMessages(threadId, scope, options) {
5635
+ const threadMessages = this.getMessagesForThread(threadId);
5636
+ const candidates = threadMessages.filter((msg) => (msg.status === "pending" || !msg.status) && this.inScope(msg, scope)).sort((a, b) => (b.priority || 0) - (a.priority || 0) || a.sequence - b.sequence);
5637
+ const limit = options.limit === void 0 ? void 0 : Math.max(0, Math.floor(options.limit));
5638
+ const selected = limit === void 0 ? candidates : candidates.slice(0, limit);
5639
+ const now = this.now();
5640
+ for (const message of selected) {
5641
+ message.status = "processing";
5642
+ message.lockedBy = options.owner;
5643
+ message.lockedAt = new Date(now);
5644
+ message.leaseExpiresAt = new Date(now + options.ttlMs);
5645
+ }
5646
+ return selected.map(clonePendingMessage);
5647
+ }
5648
+ async reclaimExpiredProcessing(threadId, scope) {
5649
+ const messages = this.messages.get(threadId);
5650
+ if (!messages) return 0;
5651
+ const now = this.now();
5652
+ let count = 0;
5653
+ for (const msg of messages) {
5654
+ if (msg.status === "processing" && this.inScope(msg, scope) && msg.leaseExpiresAt !== void 0 && msg.leaseExpiresAt.getTime() <= now) {
5655
+ msg.status = "pending";
5656
+ msg.lockedBy = void 0;
5657
+ msg.lockedAt = void 0;
5658
+ msg.leaseExpiresAt = void 0;
5659
+ count += 1;
5660
+ }
5661
+ }
5662
+ return count;
5663
+ }
5488
5664
  async getQueueSize(threadId, scope) {
5489
5665
  const pending = await this.getPendingMessages(threadId, scope);
5490
5666
  return pending.length;
@@ -5548,6 +5724,9 @@ var InMemoryThreadMessageQueueStore = class {
5548
5724
  for (const msg of messages) {
5549
5725
  if (msg.status === "processing" && this.inScope(msg, scope)) {
5550
5726
  msg.status = "pending";
5727
+ msg.lockedBy = void 0;
5728
+ msg.lockedAt = void 0;
5729
+ msg.leaseExpiresAt = void 0;
5551
5730
  count++;
5552
5731
  }
5553
5732
  }
@@ -6801,6 +6980,8 @@ var defaultProjectBotMembershipStore = new InMemoryProjectBotMembershipStore(def
6801
6980
  var defaultProjectRoomMessageStore = new InMemoryProjectRoomMessageStore(defaultProjectRoomCoordinator);
6802
6981
  var defaultProjectRoomReadStateStore = new InMemoryProjectRoomReadStateStore();
6803
6982
  var defaultUserPushSubscriptionStore = new InMemoryUserPushSubscriptionStore();
6983
+ var defaultLeaseStore = new InMemoryLeaseStore();
6984
+ var defaultOutboxStore = new InMemoryOutboxStore();
6804
6985
  storeLatticeManager.registerLattice("default", "thread", defaultThreadStore);
6805
6986
  storeLatticeManager.registerLattice(
6806
6987
  "default",
@@ -6822,6 +7003,8 @@ storeLatticeManager.registerLattice("default", "projectBotMembership", defaultPr
6822
7003
  storeLatticeManager.registerLattice("default", "projectRoomMessage", defaultProjectRoomMessageStore);
6823
7004
  storeLatticeManager.registerLattice("default", "projectRoomReadState", defaultProjectRoomReadStateStore);
6824
7005
  storeLatticeManager.registerLattice("default", "userPushSubscription", defaultUserPushSubscriptionStore);
7006
+ storeLatticeManager.registerLattice("default", "lease", defaultLeaseStore);
7007
+ storeLatticeManager.registerLattice("default", "outbox", defaultOutboxStore);
6825
7008
  var defaultUserStore = new InMemoryUserStore();
6826
7009
  var defaultTenantStore = new InMemoryTenantStore();
6827
7010
  var defaultUserTenantLinkStore = new InMemoryUserTenantLinkStore();
@@ -8690,8 +8873,8 @@ var MetricsServerManager = class _MetricsServerManager {
8690
8873
  } catch {
8691
8874
  }
8692
8875
  const tenantConfigs = this.configs.get(tenantId2);
8693
- const cached = tenantConfigs?.get(key5);
8694
- if (cached) return cached;
8876
+ const cached2 = tenantConfigs?.get(key5);
8877
+ if (cached2) return cached2;
8695
8878
  throw new Error(`Metrics server '${key5}' not found for tenant '${tenantId2}'`);
8696
8879
  }
8697
8880
  /**
@@ -10004,8 +10187,8 @@ var SandboxLatticeManager = class _SandboxLatticeManager extends BaseLatticeMana
10004
10187
  getLatticeType() {
10005
10188
  return "sandbox_manager";
10006
10189
  }
10007
- registerLattice(key5, provider) {
10008
- this.register(key5, provider);
10190
+ registerLattice(key5, provider2) {
10191
+ this.register(key5, provider2);
10009
10192
  }
10010
10193
  getSandboxLattice(key5) {
10011
10194
  const lattice = this.get(key5);
@@ -10056,8 +10239,8 @@ var SandboxLatticeManager = class _SandboxLatticeManager extends BaseLatticeMana
10056
10239
  return this.getVolumeBackendForPath(config, "/project");
10057
10240
  }
10058
10241
  async getVolumeBackendForPath(config, filePath) {
10059
- const provider = this._requireProvider();
10060
- if (!provider.createVolumeFsClient) {
10242
+ const provider2 = this._requireProvider();
10243
+ if (!provider2.createVolumeFsClient) {
10061
10244
  console.log(`[getVolumeBackendForPath] provider has no createVolumeFsClient, returning null`);
10062
10245
  return null;
10063
10246
  }
@@ -10068,7 +10251,7 @@ var SandboxLatticeManager = class _SandboxLatticeManager extends BaseLatticeMana
10068
10251
  return null;
10069
10252
  }
10070
10253
  console.log(`[getVolumeBackendForPath] path=${filePath} volume=${mapping.volumeName} prefix=${mapping.pathPrefix}`);
10071
- const client = provider.createVolumeFsClient(mapping.volumeName, mapping.pathPrefix);
10254
+ const client = provider2.createVolumeFsClient(mapping.volumeName, mapping.pathPrefix);
10072
10255
  return new VolumeFilesystem(stripPrefixClient(client, mapping.pathPrefix), mapping.pathPrefix);
10073
10256
  }
10074
10257
  _resolveVolumeForPath(config, tenantId2, filePath) {
@@ -10103,35 +10286,35 @@ var SandboxLatticeManager = class _SandboxLatticeManager extends BaseLatticeMana
10103
10286
  };
10104
10287
  }
10105
10288
  async createSandbox(name, config) {
10106
- const provider = this._requireProvider();
10107
- return provider.createSandbox(name, config);
10289
+ const provider2 = this._requireProvider();
10290
+ return provider2.createSandbox(name, config);
10108
10291
  }
10109
10292
  async getSandbox(name) {
10110
- const provider = this._requireProvider();
10111
- return provider.getSandbox(name);
10293
+ const provider2 = this._requireProvider();
10294
+ return provider2.getSandbox(name);
10112
10295
  }
10113
10296
  async stopSandbox(name) {
10114
- const provider = this._requireProvider();
10115
- return provider.stopSandbox(name);
10297
+ const provider2 = this._requireProvider();
10298
+ return provider2.stopSandbox(name);
10116
10299
  }
10117
10300
  async deleteSandbox(name) {
10118
- const provider = this._requireProvider();
10119
- return provider.deleteSandbox(name);
10301
+ const provider2 = this._requireProvider();
10302
+ return provider2.deleteSandbox(name);
10120
10303
  }
10121
10304
  async listSandboxes() {
10122
- const provider = this._requireProvider();
10123
- return provider.listSandboxes();
10305
+ const provider2 = this._requireProvider();
10306
+ return provider2.listSandboxes();
10124
10307
  }
10125
10308
  _requireProvider() {
10126
10309
  const keys = this.keysByTenant("default");
10127
10310
  if (keys.length === 0) {
10128
10311
  throw new Error("Sandbox provider not configured.");
10129
10312
  }
10130
- const provider = this.get(keys[0]);
10131
- if (!provider) {
10313
+ const provider2 = this.get(keys[0]);
10314
+ if (!provider2) {
10132
10315
  throw new Error("Sandbox provider not configured.");
10133
10316
  }
10134
- return provider;
10317
+ return provider2;
10135
10318
  }
10136
10319
  /**
10137
10320
  * Returns the first registered (default) sandbox provider.
@@ -10161,6 +10344,46 @@ var import_langchain13 = require("langchain");
10161
10344
  // src/tool_lattice/shell_exec/index.ts
10162
10345
  var import_zod17 = __toESM(require("zod"));
10163
10346
  var import_langchain14 = require("langchain");
10347
+
10348
+ // src/sandbox_lattice/sandboxOpenCredential.ts
10349
+ var SANDBOX_OPEN_GATEWAY_URL_ENV = "API_GATEWAY_URL";
10350
+ var SANDBOX_OPEN_TOKEN_ENV = "OPEN_API_TOKEN";
10351
+ var provider;
10352
+ var providerErrorHandler;
10353
+ function registerSandboxOpenCredentialProvider(next, options = {}) {
10354
+ provider = next;
10355
+ providerErrorHandler = options.onError;
10356
+ return () => {
10357
+ if (provider === next) {
10358
+ provider = void 0;
10359
+ providerErrorHandler = void 0;
10360
+ }
10361
+ };
10362
+ }
10363
+ async function getSandboxOpenCredential(context) {
10364
+ if (!provider) return void 0;
10365
+ try {
10366
+ return await provider(context);
10367
+ } catch (error) {
10368
+ try {
10369
+ providerErrorHandler?.(error);
10370
+ } catch {
10371
+ }
10372
+ return void 0;
10373
+ }
10374
+ }
10375
+ function buildSandboxOpenEnv(credential) {
10376
+ return `export ${SANDBOX_OPEN_GATEWAY_URL_ENV}=${shellSingleQuote(credential.gatewayUrl)} && export ${SANDBOX_OPEN_TOKEN_ENV}=${shellSingleQuote(credential.token)}`;
10377
+ }
10378
+ function shellSingleQuote(value) {
10379
+ return `'${value.replace(/'/g, "'\\''")}'`;
10380
+ }
10381
+ function clearSandboxOpenCredentialProviderForTests() {
10382
+ provider = void 0;
10383
+ providerErrorHandler = void 0;
10384
+ }
10385
+
10386
+ // src/tool_lattice/shell_exec/index.ts
10164
10387
  var SHELL_EXEC_DESCRIPTION = `Execute a shell command in the sandbox environment.
10165
10388
 
10166
10389
  Use this to run CLI tools, install packages, inspect files, or perform any operation that requires a shell.
@@ -10193,8 +10416,19 @@ var createShellExecTool = ({
10193
10416
  projectId: runConfig.projectId,
10194
10417
  vmIsolation
10195
10418
  });
10419
+ const tenantId2 = runConfig.tenantId;
10420
+ const credential = tenantId2 ? await getSandboxOpenCredential({
10421
+ tenantId: tenantId2,
10422
+ workspaceId: runConfig.workspaceId,
10423
+ projectId: runConfig.projectId,
10424
+ sandboxName: sandbox.name,
10425
+ assistantId: runConfig.assistant_id || void 0,
10426
+ threadId: runConfig.thread_id || void 0,
10427
+ depth: runConfig.depth
10428
+ }) : void 0;
10429
+ const command = credential ? `${buildSandboxOpenEnv(credential)} && ${input.command}` : input.command;
10196
10430
  const executeResult = await sandbox.shell.execCommand({
10197
- command: input.command,
10431
+ command,
10198
10432
  exec_dir: input.exec_dir,
10199
10433
  timeout: input.timeout ?? 300
10200
10434
  });
@@ -14926,13 +15160,23 @@ var SandboxPluginExecutor = class {
14926
15160
  validateRelativePath(args.handler, "handler");
14927
15161
  const pluginDir = `${PLUGINS_ROOT}/${args.manifest.type}`;
14928
15162
  const handlerAbs = `${pluginDir}/${args.handler}`;
15163
+ const tenantId2 = args.runConfig?.tenantId ?? args.context.tenantId;
14929
15164
  const sandbox = await this.deps.resolveSandbox({
14930
- tenantId: args.runConfig?.tenantId ?? args.context.tenantId,
15165
+ tenantId: tenantId2,
14931
15166
  assistantId: args.runConfig?.assistantId,
14932
15167
  threadId: args.runConfig?.threadId,
14933
15168
  workspaceId: args.runConfig?.workspaceId,
14934
15169
  projectId: args.runConfig?.projectId
14935
15170
  });
15171
+ const openCredential = tenantId2 ? await getSandboxOpenCredential({
15172
+ tenantId: tenantId2,
15173
+ workspaceId: args.runConfig?.workspaceId,
15174
+ projectId: args.runConfig?.projectId,
15175
+ sandboxName: sandbox.name,
15176
+ assistantId: args.runConfig?.assistantId,
15177
+ threadId: args.runConfig?.threadId,
15178
+ depth: args.runConfig?.depth
15179
+ }) : void 0;
14936
15180
  const timeoutMs = clamp(args.timeoutMs ?? DEFAULT_TIMEOUT_MS, MIN_TIMEOUT_MS, MAX_TIMEOUT_MS);
14937
15181
  const cap = clamp(
14938
15182
  args.maxResultBytes ?? DEFAULT_MAX_RESULT_BYTES,
@@ -14963,6 +15207,12 @@ var SandboxPluginExecutor = class {
14963
15207
  envParts.push(`PLUGIN_CONNECTION_FILE=${quotePosixShellArg(connectionFile)}`);
14964
15208
  }
14965
15209
  envParts.push(`PLUGIN_CONTEXT=${quotePosixShellArg(encode(args.context))}`);
15210
+ if (openCredential) {
15211
+ envParts.push(
15212
+ `${SANDBOX_OPEN_GATEWAY_URL_ENV}=${quotePosixShellArg(openCredential.gatewayUrl)}`
15213
+ );
15214
+ envParts.push(`${SANDBOX_OPEN_TOKEN_ENV}=${quotePosixShellArg(openCredential.token)}`);
15215
+ }
14966
15216
  const command = `${envParts.join(" ")} node ${quotePosixShellArg(handlerAbs)}`;
14967
15217
  const execResult = await sandbox.shell.execCommand({
14968
15218
  command,
@@ -15914,10 +16164,10 @@ var TenantPluginCatalog = class {
15914
16164
  candidates.push({ type, dirPath: dir.path, raw, manifestJson, files: dirFiles });
15915
16165
  }
15916
16166
  const hash = (0, import_node_crypto3.createHash)("sha256").update(hashParts.sort().join("\n")).digest("hex").slice(0, 16);
15917
- const cached = this.cache.get(tenantId2)?.value;
15918
- if (cached && cached.hash === hash) {
15919
- this.cache.set(tenantId2, { value: cached, checkedAt: this.now() });
15920
- return cached;
16167
+ const cached2 = this.cache.get(tenantId2)?.value;
16168
+ if (cached2 && cached2.hash === hash) {
16169
+ this.cache.set(tenantId2, { value: cached2, checkedAt: this.now() });
16170
+ return cached2;
15921
16171
  }
15922
16172
  const plugins = [];
15923
16173
  const manifests = {};
@@ -17262,7 +17512,7 @@ var AgentWorkerState = import_langgraph5.Annotation.Root({
17262
17512
  default: () => ({})
17263
17513
  }),
17264
17514
  command2: (0, import_langgraph5.Annotation)({
17265
- reducer: (x, y) => y ?? x,
17515
+ reducer: (_x, y) => y,
17266
17516
  default: () => void 0
17267
17517
  }),
17268
17518
  // RunConfig for passing configuration through the graph
@@ -17279,12 +17529,12 @@ var AgentWorkerState = import_langgraph5.Annotation.Root({
17279
17529
  }),
17280
17530
  // Execution result
17281
17531
  result: (0, import_langgraph5.Annotation)({
17282
- reducer: (x, y) => y ?? x,
17532
+ reducer: (_x, y) => y,
17283
17533
  default: () => void 0
17284
17534
  }),
17285
17535
  // Interrupt data
17286
17536
  interruptValue: (0, import_langgraph5.Annotation)({
17287
- reducer: (x, y) => y ?? x,
17537
+ reducer: (_x, y) => y,
17288
17538
  default: () => void 0
17289
17539
  }),
17290
17540
  // Final state to return
@@ -18974,6 +19224,71 @@ var CapabilityRuntimeResolver = class {
18974
19224
  // src/services/TrustedAgentDispatchCapability.ts
18975
19225
  var TRUSTED_AGENT_DISPATCH = Symbol("axiom.trustedAgentDispatch");
18976
19226
 
19227
+ // src/services/threadRunLease.ts
19228
+ var import_node_crypto5 = require("crypto");
19229
+ function threadRunLeaseKey(tenantId2, assistantId, threadId) {
19230
+ return `thread-run:${tenantId2}:${assistantId}:${threadId}`;
19231
+ }
19232
+ async function tryAcquireThreadRun(input) {
19233
+ const key5 = threadRunLeaseKey(input.tenantId, input.assistantId, input.threadId);
19234
+ const holder = `${input.owner}#${(0, import_node_crypto5.randomUUID)()}`;
19235
+ if (!await input.store.tryAcquire(key5, holder, input.ttlMs)) return null;
19236
+ const heartbeatMs = input.heartbeatMs ?? Math.max(1e3, Math.floor(input.ttlMs / 3));
19237
+ let released = false;
19238
+ let lost = false;
19239
+ const heartbeat = setInterval(() => {
19240
+ void input.store.renew(key5, holder, input.ttlMs).then((renewed) => {
19241
+ if (released || renewed || lost) return;
19242
+ lost = true;
19243
+ try {
19244
+ input.onLost?.();
19245
+ } catch {
19246
+ }
19247
+ }).catch(() => void 0);
19248
+ }, heartbeatMs);
19249
+ heartbeat.unref?.();
19250
+ return {
19251
+ get lost() {
19252
+ return lost;
19253
+ },
19254
+ release: async () => {
19255
+ if (released) return;
19256
+ released = true;
19257
+ clearInterval(heartbeat);
19258
+ await input.store.release(key5, holder).catch(() => void 0);
19259
+ }
19260
+ };
19261
+ }
19262
+ async function getThreadRunOwner(input) {
19263
+ let store = input.store;
19264
+ if (!store) {
19265
+ try {
19266
+ store = storeLatticeManager.getStoreLattice("default", "lease").store;
19267
+ } catch {
19268
+ return null;
19269
+ }
19270
+ }
19271
+ try {
19272
+ const record = await store.get(threadRunLeaseKey(input.tenantId, input.assistantId, input.threadId));
19273
+ if (!record) return null;
19274
+ const hash = record.holder.lastIndexOf("#");
19275
+ return hash > 0 ? record.holder.slice(0, hash) : record.holder;
19276
+ } catch {
19277
+ return null;
19278
+ }
19279
+ }
19280
+
19281
+ // src/services/replicaIdentity.ts
19282
+ var import_node_os = __toESM(require("os"));
19283
+ var cached;
19284
+ function getReplicaId() {
19285
+ cached ?? (cached = process.env.AXIOM_REPLICA_ID?.trim() || `${import_node_os.default.hostname()}:${process.pid}`);
19286
+ return cached;
19287
+ }
19288
+ function resetReplicaIdForTests() {
19289
+ cached = void 0;
19290
+ }
19291
+
18977
19292
  // src/services/Agent.ts
18978
19293
  var ThreadStatus2 = /* @__PURE__ */ ((ThreadStatus3) => {
18979
19294
  ThreadStatus3["IDLE"] = "idle";
@@ -19291,6 +19606,10 @@ function checkpointCapabilityRevision(state) {
19291
19606
  var QUEUE_PROCESSOR_RETRY_BASE_MS = 100;
19292
19607
  var QUEUE_PROCESSOR_MAX_RETRIES = 3;
19293
19608
  var AGENT_RECURSION_LIMIT = 1e3;
19609
+ var THREAD_RUN_LEASE_TTL_MS = 6e4;
19610
+ var THREAD_RUN_LEASE_RETRY_BASE_MS = 150;
19611
+ var THREAD_RUN_LEASE_RETRY_CAP_MS = 1500;
19612
+ var THREAD_RUN_LEASE_MAX_RETRIES = 12;
19294
19613
  function hasExactKeys(record, required, optional = []) {
19295
19614
  const allowed = /* @__PURE__ */ new Set([...required, ...optional]);
19296
19615
  return required.every((key5) => Object.prototype.hasOwnProperty.call(record, key5)) && Reflect.ownKeys(record).every((key5) => typeof key5 === "string" && allowed.has(key5));
@@ -19320,6 +19639,13 @@ var Agent = class {
19320
19639
  this.queueProcessorRetryAttempt = 0;
19321
19640
  this.queueProcessorGeneration = 0;
19322
19641
  this.queueWakeRequested = false;
19642
+ /**
19643
+ * Thread-run ownership lease held while this replica is actively processing
19644
+ * the thread. Acquired non-blockingly in {@link startQueueProcessor}; released
19645
+ * exactly once when the processor definitively stops.
19646
+ */
19647
+ this.threadRunLease = null;
19648
+ this.threadRunLeaseRetryAttempt = 0;
19323
19649
  this.asyncTasks = [];
19324
19650
  this.agentExecutor = async ({ input, command, custom_run_config, expected_capability_revision, trusted_run_context }, signal) => {
19325
19651
  const { runnable_agent, runConfig } = await this.getLatticeClientAndRuntimeConfig(
@@ -19436,7 +19762,7 @@ var Agent = class {
19436
19762
  }
19437
19763
  };
19438
19764
  this.waitingForQueueEnd = async (signal) => {
19439
- while (!signal?.aborted) {
19765
+ while (!signal?.aborted && !this.threadRunLease?.lost) {
19440
19766
  const pendings = await this.getPendingMessages();
19441
19767
  if (pendings.length === 0) {
19442
19768
  const state = await this.getCurrentState();
@@ -20267,6 +20593,29 @@ var Agent = class {
20267
20593
  }
20268
20594
  return this.queueStore;
20269
20595
  }
20596
+ /**
20597
+ * Resolves the shared lease store used to serialize thread runs across
20598
+ * Gateway replicas. Returns `undefined` when no lease store is registered so
20599
+ * single-process deployments keep running unconditionally.
20600
+ */
20601
+ getLeaseStore() {
20602
+ try {
20603
+ return storeLatticeManager.getStoreLattice("default", "lease").store;
20604
+ } catch {
20605
+ return void 0;
20606
+ }
20607
+ }
20608
+ /**
20609
+ * Releases this replica's thread-run ownership lease, if held.
20610
+ *
20611
+ * @remarks Safe to call from any definitive-stop path; a second call is a
20612
+ * no-op. The handle's unique holder makes a stale release harmless.
20613
+ */
20614
+ async releaseThreadRunLease() {
20615
+ const lease = this.threadRunLease;
20616
+ this.threadRunLease = null;
20617
+ if (lease) await lease.release();
20618
+ }
20270
20619
  getDefaultQueueConfig() {
20271
20620
  return {
20272
20621
  mode: "collect" /* COLLECT */,
@@ -20292,6 +20641,8 @@ var Agent = class {
20292
20641
  */
20293
20642
  stopQueueProcessor() {
20294
20643
  this.cancelQueueProcessorRetry();
20644
+ this.cancelThreadRunLeaseRetry();
20645
+ void this.releaseThreadRunLease();
20295
20646
  this.clearRecoveryBlock();
20296
20647
  this.queueProcessorGeneration += 1;
20297
20648
  if (this.abortController) {
@@ -20439,16 +20790,50 @@ var Agent = class {
20439
20790
  startQueueProcessorIfNeeded() {
20440
20791
  return this.startQueueProcessor(true);
20441
20792
  }
20442
- startQueueProcessor(freshWakeup) {
20793
+ async startQueueProcessor(freshWakeup) {
20443
20794
  this.queueWakeRequested = true;
20444
20795
  if (this.queueProcessorPromise) return this.queueProcessorPromise;
20445
- if (this.queueProcessorRetryTimer) return Promise.resolve();
20796
+ if (this.queueProcessorRetryTimer) return;
20446
20797
  if (freshWakeup && this.queueProcessorRetryAttempt >= QUEUE_PROCESSOR_MAX_RETRIES) {
20447
20798
  this.queueProcessorRetryAttempt = 0;
20448
20799
  }
20800
+ if (freshWakeup && this.threadRunLeaseRetryAttempt >= THREAD_RUN_LEASE_MAX_RETRIES) {
20801
+ this.threadRunLeaseRetryAttempt = 0;
20802
+ }
20449
20803
  this.queueWakeRequested = false;
20450
- let processorFailed = false;
20451
20804
  const processorGeneration = this.queueProcessorGeneration;
20805
+ if (this.threadRunLease?.lost) await this.releaseThreadRunLease();
20806
+ let acquiredLeaseHere = false;
20807
+ if (this.threadRunLease === null) {
20808
+ const leaseStore = this.getLeaseStore();
20809
+ if (leaseStore) {
20810
+ const lease = await tryAcquireThreadRun({
20811
+ store: leaseStore,
20812
+ tenantId: this.tenant_id,
20813
+ assistantId: this.assistant_id,
20814
+ threadId: this.thread_id,
20815
+ owner: getReplicaId(),
20816
+ ttlMs: THREAD_RUN_LEASE_TTL_MS,
20817
+ onLost: () => {
20818
+ console.warn(`[Agent] Thread-run lease lost for thread ${this.thread_id}; aborting the held run`);
20819
+ this.abortController?.abort();
20820
+ }
20821
+ });
20822
+ if (!lease) {
20823
+ this.scheduleThreadRunLeaseRetry();
20824
+ return;
20825
+ }
20826
+ this.threadRunLease = lease;
20827
+ acquiredLeaseHere = true;
20828
+ this.cancelThreadRunLeaseRetry();
20829
+ }
20830
+ }
20831
+ if (this.queueProcessorPromise) return this.queueProcessorPromise;
20832
+ if (this.queueProcessorGeneration !== processorGeneration) {
20833
+ if (acquiredLeaseHere) await this.releaseThreadRunLease();
20834
+ return;
20835
+ }
20836
+ let processorFailed = false;
20452
20837
  const processor = this.runQueueProcessor().catch(async (error) => {
20453
20838
  processorFailed = true;
20454
20839
  console.error(`Queue processing error for thread ${this.thread_id}:`, error);
@@ -20466,7 +20851,11 @@ var Agent = class {
20466
20851
  if (this.queueWakeRequested) void this.startQueueProcessor(false);
20467
20852
  return;
20468
20853
  }
20469
- if (failed) return;
20854
+ if (failed) {
20855
+ this.cancelThreadRunLeaseRetry();
20856
+ await this.releaseThreadRunLease();
20857
+ return;
20858
+ }
20470
20859
  this.queueProcessorRetryAttempt = 0;
20471
20860
  if (this.queueWakeRequested) {
20472
20861
  void this.startQueueProcessor(false);
@@ -20479,7 +20868,11 @@ var Agent = class {
20479
20868
  void this.startQueueProcessor(false);
20480
20869
  return;
20481
20870
  }
20482
- if (pendingMessages.length === 0) return;
20871
+ if (pendingMessages.length === 0) {
20872
+ this.cancelThreadRunLeaseRetry();
20873
+ await this.releaseThreadRunLease();
20874
+ return;
20875
+ }
20483
20876
  const processingMessages = await this.getQueueStore().getProcessingMessages(
20484
20877
  this.thread_id,
20485
20878
  this.queueScope()
@@ -20491,12 +20884,20 @@ var Agent = class {
20491
20884
  }
20492
20885
  if (processingMessages.length > 0) {
20493
20886
  const runStatus = await this.getRunStatus();
20494
- if (runStatus === "busy" /* BUSY */ || runStatus === "interrupted" /* INTERRUPTED */) return;
20887
+ if (runStatus === "busy" /* BUSY */ || runStatus === "interrupted" /* INTERRUPTED */) {
20888
+ this.cancelThreadRunLeaseRetry();
20889
+ await this.releaseThreadRunLease();
20890
+ return;
20891
+ }
20495
20892
  }
20496
20893
  void this.startQueueProcessor(false);
20497
20894
  } catch (error) {
20498
20895
  console.error(`Queue processor finalization failed for thread ${this.thread_id}:`, error);
20499
20896
  await this.scheduleQueueProcessorRetry(error, generation);
20897
+ if (!this.queueProcessorPromise) {
20898
+ await this.releaseThreadRunLease();
20899
+ this.cancelThreadRunLeaseRetry();
20900
+ }
20500
20901
  }
20501
20902
  }
20502
20903
  async scheduleQueueProcessorRetry(error, generation) {
@@ -20534,6 +20935,36 @@ var Agent = class {
20534
20935
  this.queueProcessorRetryAttempt = 0;
20535
20936
  this.queueWakeRequested = false;
20536
20937
  }
20938
+ /**
20939
+ * Bounded re-attempt after losing the thread-run lease to a peer. Covers the
20940
+ * narrow window where the peer releases the lease without having observed a
20941
+ * just-enqueued message. Bounded so a genuinely long-held thread does not make
20942
+ * this replica poll forever — the owning replica's loop drains new messages.
20943
+ */
20944
+ scheduleThreadRunLeaseRetry() {
20945
+ if (this.threadRunLeaseRetryTimer || this.threadRunLeaseRetryAttempt >= THREAD_RUN_LEASE_MAX_RETRIES) return;
20946
+ const delay = Math.min(
20947
+ THREAD_RUN_LEASE_RETRY_BASE_MS * 2 ** this.threadRunLeaseRetryAttempt,
20948
+ THREAD_RUN_LEASE_RETRY_CAP_MS
20949
+ );
20950
+ this.threadRunLeaseRetryAttempt += 1;
20951
+ this.threadRunLeaseRetryTimer = setTimeout(() => {
20952
+ this.threadRunLeaseRetryTimer = void 0;
20953
+ void this.startQueueProcessor(false);
20954
+ }, delay);
20955
+ }
20956
+ /**
20957
+ * Clears any pending thread-run lease retry and resets its backoff budget.
20958
+ *
20959
+ * @remarks Called on lease acquisition and on every definitive processor
20960
+ * stop so the next enqueue starts from the base delay. Never called on
20961
+ * continue/re-arm paths, which must preserve the bounded budget.
20962
+ */
20963
+ cancelThreadRunLeaseRetry() {
20964
+ if (this.threadRunLeaseRetryTimer) clearTimeout(this.threadRunLeaseRetryTimer);
20965
+ this.threadRunLeaseRetryTimer = void 0;
20966
+ this.threadRunLeaseRetryAttempt = 0;
20967
+ }
20537
20968
  async runQueueProcessor() {
20538
20969
  const store = this.getQueueStore();
20539
20970
  const processingMessages = await store.getProcessingMessages(this.thread_id, this.queueScope());
@@ -20678,6 +21109,38 @@ var Agent = class {
20678
21109
  console.error(`[Agent] Failed to initialize agent lattice for ${this.assistant_id} (tenant: ${this.tenant_id}):`, error);
20679
21110
  throw error;
20680
21111
  }
21112
+ let acquiredLeaseHere = false;
21113
+ if (this.threadRunLease === null) {
21114
+ const leaseStore = this.getLeaseStore();
21115
+ if (leaseStore) {
21116
+ const lease = await tryAcquireThreadRun({
21117
+ store: leaseStore,
21118
+ tenantId: this.tenant_id,
21119
+ assistantId: this.assistant_id,
21120
+ threadId: this.thread_id,
21121
+ owner: getReplicaId(),
21122
+ ttlMs: THREAD_RUN_LEASE_TTL_MS,
21123
+ onLost: () => {
21124
+ console.warn(`[Agent] Thread-run lease lost for thread ${this.thread_id}; aborting the held run`);
21125
+ this.abortController?.abort();
21126
+ }
21127
+ });
21128
+ if (!lease) {
21129
+ return { status: await this.getRunStatus(), settled: false, leaseContended: true };
21130
+ }
21131
+ this.threadRunLease = lease;
21132
+ acquiredLeaseHere = true;
21133
+ this.cancelThreadRunLeaseRetry();
21134
+ }
21135
+ }
21136
+ try {
21137
+ return await this.resumeTaskLocked();
21138
+ } finally {
21139
+ if (acquiredLeaseHere && !this.queueProcessorPromise) await this.releaseThreadRunLease();
21140
+ }
21141
+ }
21142
+ /** Recovery body of {@link resumeTask}, executed while thread ownership is settled. */
21143
+ async resumeTaskLocked() {
20681
21144
  const store = this.getQueueStore();
20682
21145
  const processingMessages = await store.getProcessingMessages(this.thread_id, this.queueScope());
20683
21146
  const preRunState = await this.getCurrentState();
@@ -20793,6 +21256,8 @@ var Agent = class {
20793
21256
  */
20794
21257
  async abort() {
20795
21258
  this.cancelQueueProcessorRetry();
21259
+ this.cancelThreadRunLeaseRetry();
21260
+ await this.releaseThreadRunLease();
20796
21261
  this.clearRecoveryBlock();
20797
21262
  this.queueProcessorGeneration += 1;
20798
21263
  if (this.abortController) {
@@ -21452,27 +21917,27 @@ function createCheckAsyncTaskTool() {
21452
21917
  if (!mainAgent) {
21453
21918
  return "Cannot access main agent state.";
21454
21919
  }
21455
- const cached = mainAgent.getAsyncTask(task_id);
21456
- if (!cached) {
21920
+ const cached2 = mainAgent.getAsyncTask(task_id);
21921
+ if (!cached2) {
21457
21922
  return `Task ${task_id} not found. Use list_async_tasks to see all tasks.`;
21458
21923
  }
21459
21924
  try {
21460
21925
  if (!agentInstanceManager.hasAgent({
21461
- assistant_id: cached.assistantId,
21926
+ assistant_id: cached2.assistantId,
21462
21927
  thread_id: task_id,
21463
21928
  tenant_id: mainAgent.tenant_id
21464
21929
  })) {
21465
- return `Task ${task_id}: not found (may have been cleaned up). Status was: ${cached.status}`;
21930
+ return `Task ${task_id}: not found (may have been cleaned up). Status was: ${cached2.status}`;
21466
21931
  }
21467
21932
  const subAgent = agentInstanceManager.getAgent({
21468
- assistant_id: cached.assistantId,
21933
+ assistant_id: cached2.assistantId,
21469
21934
  thread_id: task_id,
21470
21935
  tenant_id: mainAgent.tenant_id
21471
21936
  });
21472
21937
  const status = await subAgent.getRunStatus();
21473
21938
  if (status === "busy") {
21474
21939
  return `Task ${task_id}: still running.
21475
- Description: ${cached.description}`;
21940
+ Description: ${cached2.description}`;
21476
21941
  }
21477
21942
  if (status === "idle") {
21478
21943
  const state = await subAgent.getCurrentState();
@@ -21483,21 +21948,21 @@ Description: ${cached.description}`;
21483
21948
  mainAgent.updateAsyncTaskStatus(task_id, "completed");
21484
21949
  const result = lastAI?.content ?? "(completed, no output)";
21485
21950
  return `Task ${task_id}: COMPLETED.
21486
- Description: ${cached.description}
21951
+ Description: ${cached2.description}
21487
21952
  Result:
21488
21953
  ${result}`;
21489
21954
  }
21490
21955
  if (status === "error") {
21491
21956
  mainAgent.updateAsyncTaskStatus(task_id, "failed");
21492
21957
  return `Task ${task_id}: failed.
21493
- Description: ${cached.description}`;
21958
+ Description: ${cached2.description}`;
21494
21959
  }
21495
21960
  if (status === "interrupted") {
21496
21961
  return `Task ${task_id}: interrupted (waiting for input).
21497
- Description: ${cached.description}`;
21962
+ Description: ${cached2.description}`;
21498
21963
  }
21499
21964
  return `Task ${task_id}: ${status}.
21500
- Description: ${cached.description}`;
21965
+ Description: ${cached2.description}`;
21501
21966
  } catch (err) {
21502
21967
  return `Task ${task_id}: error checking status: ${err instanceof Error ? err.message : String(err)}`;
21503
21968
  }
@@ -21570,25 +22035,25 @@ function createCancelAsyncTaskTool() {
21570
22035
  if (!mainAgent) {
21571
22036
  return "Cannot access main agent state.";
21572
22037
  }
21573
- const cached = mainAgent.getAsyncTask(task_id);
21574
- if (!cached) {
22038
+ const cached2 = mainAgent.getAsyncTask(task_id);
22039
+ if (!cached2) {
21575
22040
  return `Task ${task_id} not found.`;
21576
22041
  }
21577
22042
  try {
21578
22043
  const subAgentExists = agentInstanceManager.hasAgent({
21579
- assistant_id: cached.assistantId,
22044
+ assistant_id: cached2.assistantId,
21580
22045
  thread_id: task_id,
21581
22046
  tenant_id: mainAgent.tenant_id
21582
22047
  });
21583
22048
  if (subAgentExists) {
21584
22049
  const subAgent = agentInstanceManager.getAgent({
21585
- assistant_id: cached.assistantId,
22050
+ assistant_id: cached2.assistantId,
21586
22051
  thread_id: task_id,
21587
22052
  tenant_id: mainAgent.tenant_id
21588
22053
  });
21589
22054
  await subAgent.abort();
21590
22055
  agentInstanceManager.removeAgent({
21591
- assistant_id: cached.assistantId,
22056
+ assistant_id: cached2.assistantId,
21592
22057
  thread_id: task_id,
21593
22058
  tenant_id: mainAgent.tenant_id
21594
22059
  });
@@ -21831,6 +22296,48 @@ var MemoryScheduleStorage = class {
21831
22296
  });
21832
22297
  }
21833
22298
  }
22299
+ /**
22300
+ * Atomically claims one due fire by transitioning the task `pending -> running`.
22301
+ *
22302
+ * @param taskId Task to claim.
22303
+ * @returns True when this caller won the claim (task was `pending`); false if
22304
+ * another replica already claimed it or it is not pending.
22305
+ */
22306
+ async claimScheduledRun(taskId) {
22307
+ const task = this.tasks.get(taskId);
22308
+ if (!task || task.status !== import_protocols11.ScheduledTaskStatus.PENDING) {
22309
+ return false;
22310
+ }
22311
+ this.tasks.set(taskId, {
22312
+ ...task,
22313
+ status: import_protocols11.ScheduledTaskStatus.RUNNING,
22314
+ updatedAt: Date.now()
22315
+ });
22316
+ return true;
22317
+ }
22318
+ /**
22319
+ * Requeues `running` tasks whose `updatedAt` is older than `olderThanMs` back
22320
+ * to `pending` (recovery after a crash that left a claim unsettled).
22321
+ *
22322
+ * @param olderThanMs Age threshold in milliseconds.
22323
+ * @returns The number of tasks requeued.
22324
+ */
22325
+ async requeueStaleRunning(olderThanMs) {
22326
+ const now = Date.now();
22327
+ const cutoff = now - olderThanMs;
22328
+ let requeued = 0;
22329
+ for (const [taskId, task] of this.tasks.entries()) {
22330
+ if (task.status === import_protocols11.ScheduledTaskStatus.RUNNING && task.updatedAt < cutoff) {
22331
+ this.tasks.set(taskId, {
22332
+ ...task,
22333
+ status: import_protocols11.ScheduledTaskStatus.PENDING,
22334
+ updatedAt: now
22335
+ });
22336
+ requeued++;
22337
+ }
22338
+ }
22339
+ return requeued;
22340
+ }
21834
22341
  /**
21835
22342
  * Delete task
21836
22343
  */
@@ -22195,6 +22702,7 @@ function describeCronExpression(expression) {
22195
22702
  }
22196
22703
 
22197
22704
  // src/schedule_lattice/DefaultScheduleClient.ts
22705
+ var STALE_RUNNING_MS = 30 * 6e4;
22198
22706
  var _DefaultScheduleClient = class _DefaultScheduleClient {
22199
22707
  constructor(storage) {
22200
22708
  this.handlers = /* @__PURE__ */ new Map();
@@ -22435,6 +22943,19 @@ var _DefaultScheduleClient = class _DefaultScheduleClient {
22435
22943
  }
22436
22944
  // ===== Recovery =====
22437
22945
  async restore() {
22946
+ try {
22947
+ const requeued = await this.storage.requeueStaleRunning(STALE_RUNNING_MS);
22948
+ if (requeued > 0) {
22949
+ console.log(
22950
+ `[Scheduler] Requeued ${requeued} stale running task(s)`
22951
+ );
22952
+ }
22953
+ } catch (error) {
22954
+ console.error(
22955
+ "[Scheduler] Failed to requeue stale running tasks:",
22956
+ error
22957
+ );
22958
+ }
22438
22959
  const activeTasks = await this.storage.getActiveTasks();
22439
22960
  let restored = 0;
22440
22961
  for (const task of activeTasks) {
@@ -22495,6 +23016,16 @@ var _DefaultScheduleClient = class _DefaultScheduleClient {
22495
23016
  });
22496
23017
  }
22497
23018
  async executeTask(task) {
23019
+ let claimed = false;
23020
+ try {
23021
+ claimed = await this.storage.claimScheduledRun(task.taskId);
23022
+ } catch (error) {
23023
+ console.error(`[Scheduler] Failed to claim task ${task.taskId}:`, error);
23024
+ return;
23025
+ }
23026
+ if (!claimed) {
23027
+ return;
23028
+ }
22498
23029
  const handler = this.handlers.get(task.taskType);
22499
23030
  if (!handler) {
22500
23031
  console.error(
@@ -22506,9 +23037,6 @@ var _DefaultScheduleClient = class _DefaultScheduleClient {
22506
23037
  });
22507
23038
  return;
22508
23039
  }
22509
- await this.storage.update(task.taskId, {
22510
- status: import_protocols12.ScheduledTaskStatus.RUNNING
22511
- });
22512
23040
  console.log(`[Scheduler] Executing task: ${task.taskId}`);
22513
23041
  try {
22514
23042
  const latestTask = await this.storage.get(task.taskId);
@@ -27232,8 +27760,8 @@ var VectorStoreProviderManager = class _VectorStoreProviderManager extends BaseL
27232
27760
  * @param key Provider key name
27233
27761
  * @param provider VectorStoreProvider implementation
27234
27762
  */
27235
- registerProvider(key5, provider) {
27236
- const lattice = { key: key5, provider };
27763
+ registerProvider(key5, provider2) {
27764
+ const lattice = { key: key5, provider: provider2 };
27237
27765
  this.registerWithTenant("default", key5, lattice);
27238
27766
  }
27239
27767
  /**
@@ -27269,7 +27797,7 @@ var VectorStoreProviderManager = class _VectorStoreProviderManager extends BaseL
27269
27797
  };
27270
27798
  var vectorStoreProviderManager = VectorStoreProviderManager.getInstance();
27271
27799
  vectorStoreProviderManager.registerProvider("default", new MemoryVectorStoreProvider());
27272
- var registerVectorStoreProvider = (key5, provider) => vectorStoreProviderManager.registerProvider(key5, provider);
27800
+ var registerVectorStoreProvider = (key5, provider2) => vectorStoreProviderManager.registerProvider(key5, provider2);
27273
27801
  var getVectorStoreProvider = (key5) => vectorStoreProviderManager.getProvider(key5);
27274
27802
 
27275
27803
  // src/store_lattice/configureStores.ts
@@ -29971,7 +30499,110 @@ independently before the workflow's integration eval.
29971
30499
 
29972
30500
  An unsupported architecture conversion interrupts the flow and requests a new
29973
30501
  user decision. Do not route backward to the Charter or to [[agent-build]], and do
29974
- not create a replacement.`
30502
+ not create a replacement.`,
30503
+ "ptc": `---
30504
+ name: ptc
30505
+ description: Programmatic Tool Calling (PTC) \u2014 write sandbox code that discovers and calls platform Open capabilities (MCP tools) through the gateway. Use when orchestration is multi-step, data-heavy, or needs loops/branches across platform tools instead of one-at-a-time model tool calls.
30506
+ metadata:
30507
+ role: workflow
30508
+ domain: orchestration
30509
+ verified: unverified
30510
+ ---
30511
+
30512
+ # Programmatic Tool Calling (PTC)
30513
+
30514
+ PTC means the model writes and runs code that calls platform tools itself, instead
30515
+ of calling tools one at a time. Reach for it when orchestration is multi-step,
30516
+ data-heavy, or needs a loop/branch that would be wasteful as a chain of model tool
30517
+ calls.
30518
+
30519
+ ## Where the code runs
30520
+
30521
+ Run the code with \`code_eval\` (backed by \`shell_exec\`). Each execution process is
30522
+ injected with two environment variables:
30523
+
30524
+ - \`API_GATEWAY_URL\` \u2014 the gateway base URL.
30525
+ - \`OPEN_API_TOKEN\` \u2014 a short-lived bearer token **self-bound to this agent**. It is
30526
+ not reusable elsewhere.
30527
+
30528
+ The token already encodes this agent's authority: the platform reflects the agent's
30529
+ own configured middleware into Open grants, plus a small always-on baseline
30530
+ (\`kb\` read-only and \`sandbox:self\`). So the callable set = that baseline \u222A the
30531
+ domains your configured middleware exposes \u2014 and nothing beyond it.
30532
+
30533
+ ## Client
30534
+
30535
+ The sandbox must have the zero-dependency \`@axiom-lattice/open-mcp-client\` package.
30536
+ Get it one of two ways:
30537
+
30538
+ - **Preinstalled in the sandbox image** (recommended): \`open-mcp\` is on \`PATH\`; for
30539
+ \`require("@axiom-lattice/open-mcp-client")\` to resolve, install it into the working
30540
+ directory or point \`NODE_PATH\` at the global \`node_modules\`.
30541
+ - **Install it in this run** when the sandbox has network + npm:
30542
+
30543
+ \`\`\`bash
30544
+ npm install @axiom-lattice/open-mcp-client
30545
+ # or use without installing:
30546
+ npx @axiom-lattice/open-mcp-client tools list --json
30547
+ \`\`\`
30548
+
30549
+ \`\`\`js
30550
+ const { OpenMcpClient } = require("@axiom-lattice/open-mcp-client");
30551
+ const client = OpenMcpClient.fromEnv(); // reads API_GATEWAY_URL + OPEN_API_TOKEN
30552
+ \`\`\`
30553
+
30554
+ CLI equivalents (config comes from the same env vars):
30555
+
30556
+ \`\`\`bash
30557
+ open-mcp tools list --json
30558
+ open-mcp tools call kb_search --json '{"query":"refund policy"}'
30559
+ \`\`\`
30560
+
30561
+ ## Discover, then call
30562
+
30563
+ 1. **Discover first.** Call \`await client.listTools()\`. This returns exactly the
30564
+ tools this agent is authorized to call (the server filters by the token's grants
30565
+ on every request, so it is the source of truth).
30566
+ 2. **Only call discovered tools.** Pick a tool from that list and call
30567
+ \`await client.callTool(name, args)\` with args matching the tool's \`inputSchema\`.
30568
+ 3. **Print the final result as JSON to stdout** so the calling agent can consume it.
30569
+ Keep intermediate logs minimal.
30570
+
30571
+ An unconfigured domain isn't registered, so calling it returns an \`isError\`
30572
+ tool-not-found result; a registered tool refused by the token's grants surfaces
30573
+ \`GRANT_DENIED\`. Either way the capability is **not configured for this agent**.
30574
+ Do not retry or work around it \u2014 drop that step or tell the user which
30575
+ middleware/domain must be enabled. A denied call is expected behavior, not a
30576
+ transient failure.
30577
+
30578
+ ## Tips
30579
+
30580
+ - Fan out with \`Promise.all\` over many items instead of many sequential model turns;
30581
+ that is the main win of PTC.
30582
+ - Inspect \`tool.inputSchema\` before calling to avoid schema-validation errors.
30583
+ - Reduce large results in code and return only the fields the agent needs.
30584
+ - The token is short-lived; for very long jobs re-read the env per \`code_eval\` run
30585
+ rather than caching the token in a file.
30586
+
30587
+ ## Worked example
30588
+
30589
+ \`\`\`js
30590
+ const { OpenMcpClient } = require("@axiom-lattice/open-mcp-client");
30591
+
30592
+ async function main() {
30593
+ const client = OpenMcpClient.fromEnv(); // API_GATEWAY_URL + OPEN_API_TOKEN
30594
+ const tools = await client.listTools(); // exactly what this agent may call
30595
+ const pick = tools.find((t) => t.name === "kb_search") ?? tools[0];
30596
+ const out = await client.callTool(pick.name, { query: "refund policy" });
30597
+ console.log(JSON.stringify({ called: pick.name, result: out.json() }));
30598
+ }
30599
+
30600
+ main().catch((err) => { console.error(String(err)); process.exit(1); });
30601
+ \`\`\`
30602
+
30603
+ Run this in \`code_eval\`. The first run proves env + client wiring; then extend it
30604
+ into the real fan-out/aggregation the task needs.
30605
+ `
29975
30606
  };
29976
30607
  function getBuiltInSkillMeta(name) {
29977
30608
  const content = BUILTIN_SKILLS[name];
@@ -33105,9 +33736,9 @@ async function getOrCreateCollectionVectorStore(name, embeddingKey, tenantId2, p
33105
33736
  const pendingInitialization = pendingVectorStoreInitializations.get(storeKey);
33106
33737
  if (pendingInitialization) return pendingInitialization;
33107
33738
  const initialization = (async () => {
33108
- const provider = vectorStoreProviderManager.getProvider(providerKey);
33739
+ const provider2 = vectorStoreProviderManager.getProvider(providerKey);
33109
33740
  const tableName = buildTableName(tenantId2, name);
33110
- const vectorStore = await provider.create({ tableName, embeddingKey });
33741
+ const vectorStore = await provider2.create({ tableName, embeddingKey });
33111
33742
  vectorStoreLatticeManager.registerLattice(storeKey, vectorStore);
33112
33743
  return vectorStore;
33113
33744
  })();
@@ -33125,9 +33756,9 @@ async function getCollectionEntryCount(name, embeddingKey, tenantId2) {
33125
33756
  return vs.getDocumentCount();
33126
33757
  }
33127
33758
  try {
33128
- const provider = vectorStoreProviderManager.getProvider("default");
33129
- if (provider.getEntryCount) {
33130
- return provider.getEntryCount(buildTableName(tenantId2, name));
33759
+ const provider2 = vectorStoreProviderManager.getProvider("default");
33760
+ if (provider2.getEntryCount) {
33761
+ return provider2.getEntryCount(buildTableName(tenantId2, name));
33131
33762
  }
33132
33763
  } catch {
33133
33764
  }
@@ -33139,9 +33770,9 @@ async function listCollectionEntries(name, embeddingKey, tenantId2) {
33139
33770
  return vs.getAllDocuments();
33140
33771
  }
33141
33772
  try {
33142
- const provider = vectorStoreProviderManager.getProvider("default");
33143
- if (provider.listEntries) {
33144
- return provider.listEntries(buildTableName(tenantId2, name));
33773
+ const provider2 = vectorStoreProviderManager.getProvider("default");
33774
+ if (provider2.listEntries) {
33775
+ return provider2.listEntries(buildTableName(tenantId2, name));
33145
33776
  }
33146
33777
  } catch {
33147
33778
  }
@@ -33151,9 +33782,9 @@ async function removeCollectionVectorStore(name, tenantId2) {
33151
33782
  const storeKey = buildKey(tenantId2, name);
33152
33783
  const tableName = buildTableName(tenantId2, name);
33153
33784
  try {
33154
- const provider = vectorStoreProviderManager.getProvider("default");
33155
- if (provider.dropTable) {
33156
- await provider.dropTable(tableName);
33785
+ const provider2 = vectorStoreProviderManager.getProvider("default");
33786
+ if (provider2.dropTable) {
33787
+ await provider2.dropTable(tableName);
33157
33788
  }
33158
33789
  } catch {
33159
33790
  }
@@ -33885,7 +34516,7 @@ function parseRecursiveGrepOutput(output) {
33885
34516
  }
33886
34517
 
33887
34518
  // src/sandbox_lattice/packDirectory.ts
33888
- var import_node_crypto5 = require("crypto");
34519
+ var import_node_crypto6 = require("crypto");
33889
34520
  var posix2 = __toESM(require("path/posix"));
33890
34521
  var MAX_PACK_ENTRIES = 1e4;
33891
34522
  var MAX_PACK_FILE_BYTES = 100 * 1024 * 1024;
@@ -33911,7 +34542,7 @@ function buildRemotePackDirectoryCommand(parentDir, sourceName, archiveName) {
33911
34542
  const parent = quotePosixShellArg(parentDir);
33912
34543
  const name = quotePosixShellArg(sourceName);
33913
34544
  const lock = quotePosixShellArg(`.${archiveName}.lock`);
33914
- const tempDir = quotePosixShellArg(`.${archiveName}.tmp-${(0, import_node_crypto5.randomBytes)(16).toString("hex")}`);
34545
+ const tempDir = quotePosixShellArg(`.${archiveName}.tmp-${(0, import_node_crypto6.randomBytes)(16).toString("hex")}`);
33915
34546
  const archivePath = joinPhysicalPath(parentDir, archiveName);
33916
34547
  const archive = quotePosixShellArg(archivePath);
33917
34548
  return [
@@ -35469,12 +36100,12 @@ var DaytonaProvider = class {
35469
36100
 
35470
36101
  // src/sandbox_lattice/providers/LocalSandboxProvider.ts
35471
36102
  var path6 = __toESM(require("path"));
35472
- var os = __toESM(require("os"));
36103
+ var os2 = __toESM(require("os"));
35473
36104
  var fs4 = __toESM(require("fs/promises"));
35474
36105
 
35475
36106
  // src/sandbox_lattice/LocalSandboxInstance.ts
35476
36107
  var import_node_child_process = require("child_process");
35477
- var import_node_crypto6 = require("crypto");
36108
+ var import_node_crypto7 = require("crypto");
35478
36109
  var fs3 = __toESM(require("fs/promises"));
35479
36110
  var path5 = __toESM(require("path"));
35480
36111
  var posix5 = __toESM(require("path/posix"));
@@ -35496,6 +36127,32 @@ function assertRegularDeleteTarget(file, stat4) {
35496
36127
  function quoteShellArgument(value) {
35497
36128
  return `'${value.replace(/'/g, "'\\''")}'`;
35498
36129
  }
36130
+ var PLATFORM_SECRET_ENV_VARS = [
36131
+ "OPEN_TOKEN_SECRET",
36132
+ "AUTH_TOKEN_SECRET",
36133
+ "JWT_SECRET",
36134
+ "WEB_APP_ATTACHMENT_SECRET",
36135
+ "LATTICE_ENCRYPTION_KEY",
36136
+ "DATABASE_URL",
36137
+ "REDIS_PASSWORD",
36138
+ "MICROSANDBOX_API_KEY",
36139
+ "DAYTONA_API_KEY",
36140
+ "E2B_API_KEY",
36141
+ "OPENAI_API_KEY",
36142
+ "AZURE_OPENAI_API_KEY",
36143
+ "ANTHROPIC_API_KEY",
36144
+ "TAVILY_API_KEY",
36145
+ "VAPID_PRIVATE_KEY",
36146
+ "SUPABASE_KEY",
36147
+ "A2A_API_KEYS"
36148
+ ];
36149
+ function stripPlatformSecrets(env) {
36150
+ const rest = { ...env };
36151
+ for (const name of PLATFORM_SECRET_ENV_VARS) {
36152
+ delete rest[name];
36153
+ }
36154
+ return rest;
36155
+ }
35499
36156
  function validatePackEntryName(name) {
35500
36157
  if (!name || name === "." || name === ".." || name.includes("\\") || name.includes("/")) {
35501
36158
  throw new Error(`Cannot pack: unsafe entry name '${name}'`);
@@ -35668,7 +36325,7 @@ var LocalSandboxInstance = class {
35668
36325
  timeout: timeoutMs,
35669
36326
  maxBuffer: 10 * 1024 * 1024,
35670
36327
  // 10MB
35671
- env: { ...process.env, HOME: this.rootDir }
36328
+ env: stripPlatformSecrets({ ...process.env, HOME: this.rootDir })
35672
36329
  }
35673
36330
  );
35674
36331
  const output = stderr ? `${stdout}
@@ -35794,7 +36451,7 @@ ${errOut}`.trim() : out.trim();
35794
36451
  };
35795
36452
  await preflight(sourceHost);
35796
36453
  const lockName = `.${filename}.lock`;
35797
- const tempDirectoryName = `.${filename}.tmp-${(0, import_node_crypto6.randomBytes)(16).toString("hex")}`;
36454
+ const tempDirectoryName = `.${filename}.tmp-${(0, import_node_crypto7.randomBytes)(16).toString("hex")}`;
35798
36455
  const sourceOperand = `./${rootName}`;
35799
36456
  tempDirectoryHost = path5.join(realArchiveDirectory, tempDirectoryName);
35800
36457
  tempHost = path5.join(tempDirectoryHost, "archive.zip");
@@ -35933,7 +36590,7 @@ var LocalSandboxProvider = class {
35933
36590
  this.instances = /* @__PURE__ */ new Map();
35934
36591
  this.creating = /* @__PURE__ */ new Map();
35935
36592
  this.createResourceAddress = createResourceAddress;
35936
- this.basePath = config.basePath ?? path6.join(os.homedir(), ".axiom-local-sandbox");
36593
+ this.basePath = config.basePath ?? path6.join(os2.homedir(), ".axiom-local-sandbox");
35937
36594
  }
35938
36595
  async createSandbox(name, _config) {
35939
36596
  const existing = this.instances.get(name);
@@ -37309,6 +37966,10 @@ var TaskLifecycleNotifier = class {
37309
37966
  this.listeners = /* @__PURE__ */ new Set();
37310
37967
  this.tails = /* @__PURE__ */ new Map();
37311
37968
  }
37969
+ /** Installs (or clears) the durable outbox used to fan notifications to other replicas. */
37970
+ setOutbox(outbox2) {
37971
+ this.outbox = outbox2;
37972
+ }
37312
37973
  /**
37313
37974
  * Registers an async lifecycle listener.
37314
37975
  *
@@ -37329,6 +37990,25 @@ var TaskLifecycleNotifier = class {
37329
37990
  * @throws Error when the event is malformed or one or more listeners reject.
37330
37991
  */
37331
37992
  async publish(event) {
37993
+ const snapshot = snapshotNotification(event);
37994
+ const key5 = `${snapshot.tenantId}:${snapshot.taskId}`;
37995
+ const previous = this.tails.get(key5) ?? Promise.resolve();
37996
+ const delivery = previous.catch(() => void 0).then(() => this.deliver(snapshot));
37997
+ this.tails.set(key5, delivery);
37998
+ if (this.outbox) {
37999
+ try {
38000
+ await this.outbox.append({ topic: "task-lifecycle", payload: snapshot, originReplicaId: getReplicaId() });
38001
+ } catch {
38002
+ }
38003
+ }
38004
+ try {
38005
+ await delivery;
38006
+ } finally {
38007
+ if (this.tails.get(key5) === delivery) this.tails.delete(key5);
38008
+ }
38009
+ }
38010
+ /** Delivers one outbox payload to local listeners without re-appending (poller path). */
38011
+ async deliverFromOutbox(event) {
37332
38012
  const snapshot = snapshotNotification(event);
37333
38013
  const key5 = `${snapshot.tenantId}:${snapshot.taskId}`;
37334
38014
  const previous = this.tails.get(key5) ?? Promise.resolve();
@@ -37393,6 +38073,10 @@ var ProjectTaskAdmissionNotifier = class {
37393
38073
  if (!Number.isSafeInteger(timeout) || timeout <= 0) throw new Error("Invalid admission listener timeout");
37394
38074
  this.listenerTimeoutMs = timeout;
37395
38075
  }
38076
+ /** Installs (or clears) the durable outbox used to fan notifications to other replicas. */
38077
+ setOutbox(outbox2) {
38078
+ this.outbox = outbox2;
38079
+ }
37396
38080
  /**
37397
38081
  * Registers a Project task admission listener.
37398
38082
  *
@@ -37412,6 +38096,30 @@ var ProjectTaskAdmissionNotifier = class {
37412
38096
  * @returns A promise settled after every snapshotted listener settles.
37413
38097
  */
37414
38098
  async publish(event) {
38099
+ const snapshot = snapshotNotification2(event);
38100
+ const listeners = [...this.listeners];
38101
+ const key5 = `${snapshot.tenantId}\0${snapshot.taskId}`;
38102
+ const previous = this.tails.get(key5) ?? Promise.resolve();
38103
+ const delivery = previous.catch(() => void 0).then(() => this.deliver(snapshot, listeners));
38104
+ this.tails.set(key5, delivery);
38105
+ if (this.outbox) {
38106
+ try {
38107
+ await this.outbox.append({
38108
+ topic: "project-task-admission",
38109
+ payload: snapshot,
38110
+ originReplicaId: getReplicaId()
38111
+ });
38112
+ } catch {
38113
+ }
38114
+ }
38115
+ try {
38116
+ await delivery;
38117
+ } finally {
38118
+ if (this.tails.get(key5) === delivery) this.tails.delete(key5);
38119
+ }
38120
+ }
38121
+ /** Delivers one outbox payload to local listeners without re-appending (poller path). */
38122
+ async deliverFromOutbox(event) {
37415
38123
  const snapshot = snapshotNotification2(event);
37416
38124
  const listeners = [...this.listeners];
37417
38125
  const key5 = `${snapshot.tenantId}\0${snapshot.taskId}`;
@@ -37481,8 +38189,137 @@ function canonicalTimestamp(value) {
37481
38189
  return Number.isFinite(parsed.getTime()) && parsed.toISOString() === value;
37482
38190
  }
37483
38191
 
38192
+ // src/services/OutboxPoller.ts
38193
+ var OutboxPoller = class {
38194
+ /**
38195
+ * Creates a poller bound to one durable outbox.
38196
+ *
38197
+ * @param deps Store, owner replica id, topic handlers, and optional tuning.
38198
+ */
38199
+ constructor(deps) {
38200
+ this.deps = deps;
38201
+ this.cursor = null;
38202
+ this.polling = false;
38203
+ }
38204
+ /** Captures the starting cursor and begins polling. */
38205
+ async start() {
38206
+ this.cursor = await this.deps.store.latestId();
38207
+ this.timer = setInterval(() => {
38208
+ void this.pollOnce();
38209
+ }, this.deps.intervalMs ?? 1e3);
38210
+ this.timer.unref?.();
38211
+ }
38212
+ /** Stops polling. */
38213
+ stop() {
38214
+ if (this.timer) clearInterval(this.timer);
38215
+ this.timer = void 0;
38216
+ }
38217
+ /** Runs one drain pass (exported for tests). */
38218
+ async pollOnce() {
38219
+ if (this.polling) return;
38220
+ this.polling = true;
38221
+ try {
38222
+ const batchSize = this.deps.batchSize ?? 100;
38223
+ for (; ; ) {
38224
+ const events = await this.deps.store.listAfter(this.cursor, batchSize);
38225
+ if (events.length === 0) break;
38226
+ for (const event of events) {
38227
+ this.cursor = event.id;
38228
+ if (event.originReplicaId === this.deps.owner) continue;
38229
+ const handler = this.deps.handlers[event.topic];
38230
+ if (!handler) continue;
38231
+ try {
38232
+ await this.deliverWithTimeout(handler, event.payload);
38233
+ } catch {
38234
+ this.deps.log?.warn("Outbox delivery failed", { topic: event.topic, id: event.id });
38235
+ }
38236
+ }
38237
+ if (events.length < batchSize) break;
38238
+ }
38239
+ } catch (error) {
38240
+ this.deps.log?.warn("Outbox poll failed", { error: error instanceof Error ? error.message : String(error) });
38241
+ } finally {
38242
+ this.polling = false;
38243
+ }
38244
+ }
38245
+ /**
38246
+ * Runs one handler bounded by `handlerTimeoutMs` so a hung listener cannot
38247
+ * stall cross-replica delivery for this replica.
38248
+ */
38249
+ async deliverWithTimeout(handler, payload) {
38250
+ const timeoutMs = this.deps.handlerTimeoutMs ?? 5e3;
38251
+ let timer;
38252
+ const timeout = new Promise((_resolve, reject) => {
38253
+ timer = setTimeout(() => reject(new Error("Outbox handler timed out")), timeoutMs);
38254
+ timer.unref?.();
38255
+ });
38256
+ try {
38257
+ await Promise.race([Promise.resolve().then(() => handler(payload)), timeout]);
38258
+ } finally {
38259
+ if (timer) clearTimeout(timer);
38260
+ }
38261
+ }
38262
+ };
38263
+
37484
38264
  // src/services/ProjectMutationMutex.ts
37485
38265
  var import_protocols28 = require("@axiom-lattice/protocols");
38266
+
38267
+ // src/services/coordinationLease.ts
38268
+ var configured;
38269
+ function configureCoordinationLeaseStore(store) {
38270
+ configured = store;
38271
+ }
38272
+ function getCoordinationLeaseStore() {
38273
+ return configured;
38274
+ }
38275
+
38276
+ // src/services/withLease.ts
38277
+ var import_node_crypto8 = require("crypto");
38278
+ var LeaseAcquisitionTimeoutError = class extends Error {
38279
+ /**
38280
+ * @param key Lease key that could not be acquired.
38281
+ * @param holder Unique holder that attempted acquisition.
38282
+ * @param timeoutMs How long the caller waited before giving up.
38283
+ */
38284
+ constructor(key5, holder, timeoutMs) {
38285
+ super(`Could not acquire lease '${key5}' for '${holder}' within ${timeoutMs}ms`);
38286
+ this.key = key5;
38287
+ this.holder = holder;
38288
+ this.timeoutMs = timeoutMs;
38289
+ /** Stable machine-readable code. */
38290
+ this.code = "LEASE_ACQUISITION_TIMEOUT";
38291
+ this.name = "LeaseAcquisitionTimeoutError";
38292
+ }
38293
+ };
38294
+ function sleep(ms) {
38295
+ return new Promise((resolve4) => setTimeout(resolve4, ms));
38296
+ }
38297
+ async function withLease(store, key5, ownerLabel, fn, options = {}) {
38298
+ const ttlMs = options.ttlMs ?? 3e4;
38299
+ const acquireTimeoutMs = options.acquireTimeoutMs ?? 1e4;
38300
+ const retryDelayMs = options.retryDelayMs ?? 50;
38301
+ const heartbeatMs = options.heartbeatMs ?? Math.max(1e3, Math.floor(ttlMs / 3));
38302
+ const holder = `${ownerLabel}#${(0, import_node_crypto8.randomUUID)()}`;
38303
+ const deadline = Date.now() + acquireTimeoutMs;
38304
+ for (; ; ) {
38305
+ if (await store.tryAcquire(key5, holder, ttlMs)) break;
38306
+ if (Date.now() >= deadline) throw new LeaseAcquisitionTimeoutError(key5, holder, acquireTimeoutMs);
38307
+ await sleep(retryDelayMs);
38308
+ }
38309
+ const heartbeat = setInterval(() => {
38310
+ void store.renew(key5, holder, ttlMs).catch(() => void 0);
38311
+ }, heartbeatMs);
38312
+ heartbeat.unref?.();
38313
+ try {
38314
+ return await fn();
38315
+ } finally {
38316
+ clearInterval(heartbeat);
38317
+ await store.release(key5, holder).catch(() => void 0);
38318
+ }
38319
+ }
38320
+
38321
+ // src/services/ProjectMutationMutex.ts
38322
+ var MUTATION_LEASE_OPTIONS = { ttlMs: 6e4, acquireTimeoutMs: 9e4, retryDelayMs: 100 };
37486
38323
  function projectKey(scope) {
37487
38324
  const row = (0, import_protocols28.snapshotExactRecord)(scope, ["tenantId", "projectId"]);
37488
38325
  if (!row || typeof row.tenantId !== "string" || !row.tenantId || row.tenantId.includes("\0") || typeof row.projectId !== "string" || !row.projectId || row.projectId.includes("\0")) {
@@ -37498,9 +38335,12 @@ var ProjectMutationMutex = class {
37498
38335
  /** Runs one mutation after all earlier mutations for the same Tenant and Project. */
37499
38336
  async runExclusive(scope, operation, signal) {
37500
38337
  const key5 = projectKey(scope);
38338
+ const leaseKey = `project-mutation:${encodeURIComponent(scope.tenantId)}:${encodeURIComponent(scope.projectId)}`;
37501
38339
  await this.acquire(key5, signal);
37502
38340
  try {
37503
- return await operation();
38341
+ const store = getCoordinationLeaseStore();
38342
+ if (!store) return await operation();
38343
+ return await withLease(store, leaseKey, getReplicaId(), operation, MUTATION_LEASE_OPTIONS);
37504
38344
  } finally {
37505
38345
  this.release(key5);
37506
38346
  }
@@ -37937,11 +38777,11 @@ function snapshotDate(value) {
37937
38777
  }
37938
38778
 
37939
38779
  // src/services/TaskLifecycleService.ts
37940
- var import_node_crypto8 = require("crypto");
38780
+ var import_node_crypto10 = require("crypto");
37941
38781
  var import_protocols31 = require("@axiom-lattice/protocols");
37942
38782
 
37943
38783
  // src/middlewares/taskBelief.ts
37944
- var import_node_crypto7 = require("crypto");
38784
+ var import_node_crypto9 = require("crypto");
37945
38785
  function buildBeliefCompletionEventKey(taskId, result, impacts) {
37946
38786
  const canonical = JSON.stringify({
37947
38787
  beliefImpact: impacts.map((impact) => ({
@@ -37952,7 +38792,7 @@ function buildBeliefCompletionEventKey(taskId, result, impacts) {
37952
38792
  })),
37953
38793
  result: result.trim()
37954
38794
  });
37955
- return (0, import_node_crypto7.createHash)("sha256").update(canonical).digest("hex");
38795
+ return (0, import_node_crypto9.createHash)("sha256").update(canonical).digest("hex");
37956
38796
  }
37957
38797
  function buildCompletionEvidenceDetail(result, impacts, eventKey) {
37958
38798
  return { result, beliefImpact: impacts, eventKey };
@@ -38007,9 +38847,9 @@ var BELIEF_OWNER_WRITE_ATTEMPTS = 3;
38007
38847
  function isWellFormedImpact(impact) {
38008
38848
  return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(impact.key) && Number.isInteger(impact.after) && impact.after >= 0 && impact.after <= 100 && impact.basis.trim().length > 0 && impact.basis.length <= 1e3 && !/[\r\n]/.test(impact.basis);
38009
38849
  }
38010
- async function applyBeliefImpacts(taskStore, owner, impacts) {
38011
- if (impacts.length === 0) return { description: owner.description ?? "" };
38012
- let current = owner;
38850
+ async function applyBeliefImpacts(taskStore, owner2, impacts) {
38851
+ if (impacts.length === 0) return { description: owner2.description ?? "" };
38852
+ let current = owner2;
38013
38853
  for (let attempt = 0; attempt < BELIEF_OWNER_WRITE_ATTEMPTS; attempt += 1) {
38014
38854
  const parsed = (0, import_protocols31.parseTaskBeliefState)(current.description ?? "");
38015
38855
  const existingTable = parsed.success;
@@ -38138,14 +38978,14 @@ function sameBeliefOwner(left, right) {
38138
38978
  function sameBeliefOwnerIdentity(left, right) {
38139
38979
  return canonicalJson({ ...left, beliefState: void 0 }) === canonicalJson({ ...right, beliefState: void 0 });
38140
38980
  }
38141
- function canonicalBeliefOwner(owner) {
38981
+ function canonicalBeliefOwner(owner2) {
38142
38982
  const beliefState = {
38143
- entries: owner.beliefState.entries.map((entry) => ({
38983
+ entries: owner2.beliefState.entries.map((entry) => ({
38144
38984
  ...entry,
38145
38985
  basis: entry.basis.trim().replace(/\s+/g, " ")
38146
38986
  })).sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0)
38147
38987
  };
38148
- return { ...owner, beliefState };
38988
+ return { ...owner2, beliefState };
38149
38989
  }
38150
38990
  function eventKeyFor(taskId, result, impacts, beliefOwner) {
38151
38991
  const base = `belief-impact:${taskId}:${buildBeliefCompletionEventKey(taskId, result, impacts)}`;
@@ -38155,7 +38995,7 @@ function canonicalJson(value) {
38155
38995
  return JSON.stringify(value);
38156
38996
  }
38157
38997
  function sha256(value) {
38158
- return (0, import_node_crypto8.createHash)("sha256").update(canonicalJson(value)).digest("hex");
38998
+ return (0, import_node_crypto10.createHash)("sha256").update(canonicalJson(value)).digest("hex");
38159
38999
  }
38160
39000
  function sameEvidenceDetail(detail, taskId, result, impacts, eventKey, beliefOwner) {
38161
39001
  const storedOwner = detail.beliefOwner;
@@ -39512,8 +40352,8 @@ var TaskLifecycleService = class {
39512
40352
  return { task: existing, parent: parentResult, eventKey, evidence, result, impacts };
39513
40353
  }
39514
40354
  beliefOwnerSnapshot(task, parent) {
39515
- const owner = parent?.ownerType === "agent" ? parent : task;
39516
- const parsed = (0, import_protocols31.parseTaskBeliefState)(owner.description ?? "");
40355
+ const owner2 = parent?.ownerType === "agent" ? parent : task;
40356
+ const parsed = (0, import_protocols31.parseTaskBeliefState)(owner2.description ?? "");
39517
40357
  const beliefState = parsed.success ? parsed.state : { entries: [] };
39518
40358
  if (parent?.ownerType === "agent") {
39519
40359
  return {
@@ -40233,7 +41073,7 @@ function createTaskLifecycleService(deps) {
40233
41073
  }
40234
41074
 
40235
41075
  // src/services/TaskExecutionService.ts
40236
- var import_node_crypto9 = require("crypto");
41076
+ var import_node_crypto12 = require("crypto");
40237
41077
  var import_node_util3 = require("util");
40238
41078
  var import_protocols32 = require("@axiom-lattice/protocols");
40239
41079
 
@@ -40492,12 +41332,20 @@ var TaskRunRegistry = class {
40492
41332
  }
40493
41333
  };
40494
41334
  var taskRunRegistry = new TaskRunRegistry();
41335
+ async function safelyLive(probe, task) {
41336
+ try {
41337
+ return await probe(task);
41338
+ } catch {
41339
+ return true;
41340
+ }
41341
+ }
40495
41342
  async function reconcileStaleTaskRuns(input) {
40496
41343
  const registry4 = input.registry ?? taskRunRegistry;
40497
41344
  const converged = [];
40498
41345
  const convergedTasks = [];
40499
41346
  for (const task of input.tasks) {
40500
41347
  if (task.status !== "in_progress" || registry4.isLive(task.id)) continue;
41348
+ if (input.isTaskLive && await safelyLive(input.isTaskLive, task)) continue;
40501
41349
  const result = await input.lifecycle.failTask({
40502
41350
  tenantId: task.tenantId,
40503
41351
  taskId: task.id,
@@ -40518,7 +41366,8 @@ async function reconcileTenantTaskRuns(input) {
40518
41366
  tasks: candidates,
40519
41367
  actor: input.actor,
40520
41368
  lifecycle: input.lifecycle,
40521
- ...input.registry ? { registry: input.registry } : {}
41369
+ ...input.registry ? { registry: input.registry } : {},
41370
+ ...input.isTaskLive ? { isTaskLive: input.isTaskLive } : {}
40522
41371
  });
40523
41372
  }
40524
41373
  async function reconcileAllTenantTaskRuns(input) {
@@ -40534,7 +41383,8 @@ async function reconcileAllTenantTaskRuns(input) {
40534
41383
  taskStore: input.taskStore,
40535
41384
  lifecycle: input.lifecycle,
40536
41385
  ...input.registry ? { registry: input.registry } : {},
40537
- ...input.include ? { include: input.include } : {}
41386
+ ...input.include ? { include: input.include } : {},
41387
+ ...input.isTaskLive ? { isTaskLive: input.isTaskLive } : {}
40538
41388
  });
40539
41389
  } catch {
40540
41390
  continue;
@@ -40545,6 +41395,42 @@ async function reconcileAllTenantTaskRuns(input) {
40545
41395
  return { converged, convergedTasks };
40546
41396
  }
40547
41397
 
41398
+ // src/services/taskRunLease.ts
41399
+ var import_node_crypto11 = require("crypto");
41400
+ function taskRunLeaseKey(taskId) {
41401
+ return `task-run:${taskId}`;
41402
+ }
41403
+ async function tryAcquireTaskRun(input) {
41404
+ const key5 = taskRunLeaseKey(input.taskId);
41405
+ const holder = `${input.owner}#${(0, import_node_crypto11.randomUUID)()}`;
41406
+ if (!await input.store.tryAcquire(key5, holder, input.ttlMs)) return null;
41407
+ const heartbeatMs = input.heartbeatMs ?? Math.max(1e3, Math.floor(input.ttlMs / 3));
41408
+ let released = false;
41409
+ let lost = false;
41410
+ const heartbeat = setInterval(() => {
41411
+ void input.store.renew(key5, holder, input.ttlMs).then((renewed) => {
41412
+ if (released || renewed || lost) return;
41413
+ lost = true;
41414
+ try {
41415
+ input.onLost?.();
41416
+ } catch {
41417
+ }
41418
+ }).catch(() => void 0);
41419
+ }, heartbeatMs);
41420
+ heartbeat.unref?.();
41421
+ return {
41422
+ get lost() {
41423
+ return lost;
41424
+ },
41425
+ release: async () => {
41426
+ if (released) return;
41427
+ released = true;
41428
+ clearInterval(heartbeat);
41429
+ await input.store.release(key5, holder).catch(() => void 0);
41430
+ }
41431
+ };
41432
+ }
41433
+
40548
41434
  // src/services/ExecutionAuthorityRegistry.ts
40549
41435
  var ManagedTaskStartError = class extends Error {
40550
41436
  /**
@@ -40815,6 +41701,14 @@ async function prepareTaskDispatch(deps, input) {
40815
41701
 
40816
41702
  // src/services/TaskExecutionService.ts
40817
41703
  var import_langgraph14 = require("@langchain/langgraph");
41704
+ var TASK_RUN_LEASE_TTL_MS = 12e4;
41705
+ function getLeaseStore() {
41706
+ try {
41707
+ return storeLatticeManager.getStoreLattice("default", "lease").store;
41708
+ } catch {
41709
+ return void 0;
41710
+ }
41711
+ }
40818
41712
  function persistedExecutionObservation(item, attempted) {
40819
41713
  const detail = item.detail;
40820
41714
  const persistedObservation = toJsonSafeValue(detail?.observation);
@@ -40875,11 +41769,11 @@ function createTaskExecutionService(deps) {
40875
41769
  let threadId;
40876
41770
  let runId;
40877
41771
  try {
40878
- threadId = storedThreadId ?? (deps.createThreadId ?? import_node_crypto9.randomUUID)();
41772
+ threadId = storedThreadId ?? (deps.createThreadId ?? import_node_crypto12.randomUUID)();
40879
41773
  if (task.status === "interrupted" && reuseCurrentRun) {
40880
41774
  runId = storedRunId;
40881
41775
  } else if (task.status === "interrupted" || task.status === "failed") {
40882
- runId = (deps.createRunId ?? import_node_crypto9.randomUUID)();
41776
+ runId = (deps.createRunId ?? import_node_crypto12.randomUUID)();
40883
41777
  }
40884
41778
  } catch (error) {
40885
41779
  return {
@@ -40942,7 +41836,21 @@ function createTaskExecutionService(deps) {
40942
41836
  warnings.push(...begun.warnings);
40943
41837
  }
40944
41838
  taskRunRegistry.register(task.id);
41839
+ let taskRunLease = null;
40945
41840
  try {
41841
+ const leaseStore = getLeaseStore();
41842
+ if (leaseStore) {
41843
+ try {
41844
+ taskRunLease = await tryAcquireTaskRun({
41845
+ store: leaseStore,
41846
+ taskId: task.id,
41847
+ owner: getReplicaId(),
41848
+ ttlMs: TASK_RUN_LEASE_TTL_MS
41849
+ });
41850
+ } catch {
41851
+ taskRunLease = null;
41852
+ }
41853
+ }
40946
41854
  const invocation = {
40947
41855
  tenantId: tenantId2,
40948
41856
  workspaceId: task.workspaceId,
@@ -41110,6 +42018,7 @@ function createTaskExecutionService(deps) {
41110
42018
  }
41111
42019
  } finally {
41112
42020
  taskRunRegistry.unregister(task.id);
42021
+ if (taskRunLease) await taskRunLease.release();
41113
42022
  }
41114
42023
  };
41115
42024
  return {
@@ -41122,6 +42031,43 @@ function createTaskExecutionService(deps) {
41122
42031
  };
41123
42032
  }
41124
42033
 
42034
+ // src/services/leadership.ts
42035
+ var import_node_crypto13 = require("crypto");
42036
+ async function runAsLeader(store, name, fn, options = {}) {
42037
+ const key5 = `leader:${name}`;
42038
+ const owner2 = options.owner ?? getReplicaId();
42039
+ const holder = `${owner2}#${(0, import_node_crypto13.randomUUID)()}`;
42040
+ const ttlMs = options.ttlMs ?? 6e4;
42041
+ if (!await store.tryAcquire(key5, holder, ttlMs)) return { ran: false };
42042
+ const heartbeat = setInterval(() => {
42043
+ void store.renew(key5, holder, ttlMs).catch(() => void 0);
42044
+ }, Math.max(1e3, Math.floor(ttlMs / 3)));
42045
+ heartbeat.unref?.();
42046
+ try {
42047
+ const result = await fn();
42048
+ return { ran: true, result };
42049
+ } finally {
42050
+ clearInterval(heartbeat);
42051
+ if (!options.hold) await store.release(key5, holder).catch(() => void 0);
42052
+ }
42053
+ }
42054
+
42055
+ // src/services/pluginCatalogSync.ts
42056
+ var PLUGIN_CATALOG_CHANGED_TOPIC = "plugin-catalog-changed";
42057
+ var outbox;
42058
+ var owner = "unknown";
42059
+ function configurePluginCatalogSync(next) {
42060
+ outbox = next?.outbox;
42061
+ owner = next?.owner ?? "unknown";
42062
+ }
42063
+ async function publishPluginCatalogChanged(tenantId2) {
42064
+ if (!outbox || !tenantId2) return;
42065
+ try {
42066
+ await outbox.append({ topic: PLUGIN_CATALOG_CHANGED_TOPIC, payload: { tenantId: tenantId2 }, originReplicaId: owner });
42067
+ } catch {
42068
+ }
42069
+ }
42070
+
41125
42071
  // src/index.ts
41126
42072
  var Protocols = __toESM(require("@axiom-lattice/protocols"));
41127
42073
 
@@ -41264,9 +42210,9 @@ var createLoadSkillsTool = ({ skills, readAll, pluginSkills, pluginSkillOwners }
41264
42210
  if (!skillName) continue;
41265
42211
  if (await isPluginSkillManagedInSandbox(sandbox, skillName)) {
41266
42212
  const enabledDefinition = pluginSkills?.[skillName];
41267
- const owner = pluginSkillOwners?.[skillName];
41268
- const registered = owner && enabledDefinition ? PluginRegistry.getSkillForPlugin(owner, skillName) : void 0;
41269
- if (!registered || registered.definition !== enabledDefinition || !owner || !await isPluginSkillAuthorizedInSandbox(sandbox, skillName, owner) || await getPluginSkillManifestVersion(sandbox, skillName) !== enabledDefinition.version) {
42213
+ const owner2 = pluginSkillOwners?.[skillName];
42214
+ const registered = owner2 && enabledDefinition ? PluginRegistry.getSkillForPlugin(owner2, skillName) : void 0;
42215
+ if (!registered || registered.definition !== enabledDefinition || !owner2 || !await isPluginSkillAuthorizedInSandbox(sandbox, skillName, owner2) || await getPluginSkillManifestVersion(sandbox, skillName) !== enabledDefinition.version) {
41270
42216
  continue;
41271
42217
  }
41272
42218
  }
@@ -41292,8 +42238,8 @@ var createLoadSkillsTool = ({ skills, readAll, pluginSkills, pluginSkillOwners }
41292
42238
  const seenNames = new Set(filteredSkills.map((skill) => skill.name));
41293
42239
  try {
41294
42240
  for (const [name, definition] of Object.entries(pluginSkills ?? {})) {
41295
- const owner = pluginSkillOwners?.[name];
41296
- const registered = owner ? PluginRegistry.getSkillForPlugin(owner, name) : void 0;
42241
+ const owner2 = pluginSkillOwners?.[name];
42242
+ const registered = owner2 ? PluginRegistry.getSkillForPlugin(owner2, name) : void 0;
41297
42243
  if (!registered || registered.definition !== definition) continue;
41298
42244
  const { meta } = parseSkillFrontmatter(definition.content);
41299
42245
  if (!meta.name || !meta.description || seenNames.has(meta.name)) continue;
@@ -41330,24 +42276,24 @@ var import_zod52 = __toESM(require("zod"));
41330
42276
  var import_langchain63 = require("langchain");
41331
42277
 
41332
42278
  // src/sandbox_plugin/pluginSkillResolution.ts
41333
- async function resolveTenantPluginSkill(tenantId2, owner, skillName) {
42279
+ async function resolveTenantPluginSkill(tenantId2, owner2, skillName) {
41334
42280
  try {
41335
42281
  const { plugins } = await getSandboxPluginCatalog().ensureLoaded(tenantId2);
41336
- const plugin = plugins.find((candidate) => candidate.meta.type === owner);
42282
+ const plugin = plugins.find((candidate) => candidate.meta.type === owner2);
41337
42283
  return plugin?.skills?.[skillName];
41338
42284
  } catch {
41339
42285
  return void 0;
41340
42286
  }
41341
42287
  }
41342
- async function resolveEnabledPluginSkill(owner, skillName, enabledDefinition, tenantId2) {
41343
- if (!owner || !enabledDefinition) return void 0;
41344
- const registered = PluginRegistry.getSkillForPlugin(owner, skillName);
41345
- if (registered && registered.pluginType === owner && registered.definition === enabledDefinition) {
41346
- return { pluginType: owner, definition: enabledDefinition };
42288
+ async function resolveEnabledPluginSkill(owner2, skillName, enabledDefinition, tenantId2) {
42289
+ if (!owner2 || !enabledDefinition) return void 0;
42290
+ const registered = PluginRegistry.getSkillForPlugin(owner2, skillName);
42291
+ if (registered && registered.pluginType === owner2 && registered.definition === enabledDefinition) {
42292
+ return { pluginType: owner2, definition: enabledDefinition };
41347
42293
  }
41348
42294
  if (!tenantId2) return void 0;
41349
- const tenantDefinition = await resolveTenantPluginSkill(tenantId2, owner, skillName);
41350
- return tenantDefinition ? { pluginType: owner, definition: tenantDefinition } : void 0;
42295
+ const tenantDefinition = await resolveTenantPluginSkill(tenantId2, owner2, skillName);
42296
+ return tenantDefinition ? { pluginType: owner2, definition: tenantDefinition } : void 0;
41351
42297
  }
41352
42298
 
41353
42299
  // src/tool_lattice/skill/load_skill_content.ts
@@ -41409,10 +42355,10 @@ var createLoadSkillContentTool = (pluginSkills, pluginSkillOwnersOrStaticSelecti
41409
42355
  return result2;
41410
42356
  }
41411
42357
  const enabledDefinition = pluginSkills?.[input.skill_name];
41412
- const owner = pluginSkillOwners?.[input.skill_name];
42358
+ const owner2 = pluginSkillOwners?.[input.skill_name];
41413
42359
  const runConfig = _exe_config?.configurable?.runConfig || {};
41414
42360
  const pluginSkill = await resolveEnabledPluginSkill(
41415
- owner,
42361
+ owner2,
41416
42362
  input.skill_name,
41417
42363
  enabledDefinition,
41418
42364
  runConfig.tenantId
@@ -41519,9 +42465,9 @@ var createLoadSkillResourceTool = (pluginSkills, pluginSkillOwners) => {
41519
42465
  validateRelativePath(input.resource_path, "resource");
41520
42466
  const sandbox = await getSandboxFromExeConfig3(_exe_config);
41521
42467
  const definition = pluginSkills?.[input.skill_name];
41522
- const owner = pluginSkillOwners?.[input.skill_name];
42468
+ const owner2 = pluginSkillOwners?.[input.skill_name];
41523
42469
  const tenantId2 = _exe_config?.configurable?.runConfig?.tenantId;
41524
- const resolved = await resolveEnabledPluginSkill(owner, input.skill_name, definition, tenantId2);
42470
+ const resolved = await resolveEnabledPluginSkill(owner2, input.skill_name, definition, tenantId2);
41525
42471
  if (definition) {
41526
42472
  if (!resolved || !resolved.definition.resources?.[input.resource_path]) {
41527
42473
  return `Resource "${input.resource_path}" not found for skill "${input.skill_name}".`;
@@ -43748,9 +44694,9 @@ async function renderAvailableExecutors(rc, tenantId2) {
43748
44694
  const governed = authority.kind === "projectRoom" || authority.kind === "projectTask";
43749
44695
  const projectId = governed ? (authority.trusted.projectTask ?? authority.trusted.projectRoom)?.projectId : void 0;
43750
44696
  const cacheKey = `${tenantId2}:${governed ? `project:${projectId ?? ""}` : "ordinary"}`;
43751
- const cached = executorCache.get(cacheKey);
44697
+ const cached2 = executorCache.get(cacheKey);
43752
44698
  const now = Date.now();
43753
- if (cached && now - cached.at < EXECUTOR_CACHE_TTL_MS) return cached.text;
44699
+ if (cached2 && now - cached2.at < EXECUTOR_CACHE_TTL_MS) return cached2.text;
43754
44700
  const lines = [];
43755
44701
  try {
43756
44702
  if (governed && projectId) {
@@ -48031,6 +48977,7 @@ var sandboxPluginsPlugin = {
48031
48977
  }
48032
48978
  case "reload": {
48033
48979
  const loaded = await catalog2.reload(tenantId2);
48980
+ void publishPluginCatalogChanged(tenantId2);
48034
48981
  return JSON.stringify({ reloaded: true, plugins: loaded.plugins.length, diagnostics: loaded.diagnostics });
48035
48982
  }
48036
48983
  case "create":
@@ -48050,12 +48997,14 @@ var sandboxPluginsPlugin = {
48050
48997
  if (!check.manifest) return JSON.stringify({ written: false, diagnostics: check.diagnostics });
48051
48998
  await catalog2.writePlugin(tenantId2, input.type, input.manifest, handlers);
48052
48999
  const loaded = await catalog2.reload(tenantId2);
49000
+ void publishPluginCatalogChanged(tenantId2);
48053
49001
  return JSON.stringify({ written: true, diagnostics: loaded.diagnostics, hash: loaded.hash });
48054
49002
  }
48055
49003
  case "delete": {
48056
49004
  if (!input.type) return "type is required";
48057
49005
  await catalog2.deletePlugin(tenantId2, input.type);
48058
49006
  await catalog2.reload(tenantId2);
49007
+ void publishPluginCatalogChanged(tenantId2);
48059
49008
  return JSON.stringify({ deleted: true });
48060
49009
  }
48061
49010
  case "validate": {
@@ -48167,6 +49116,26 @@ function registerBuiltinPlugins() {
48167
49116
  }
48168
49117
  }
48169
49118
 
49119
+ // src/sandbox_plugin/provisionOpenClient.ts
49120
+ var import_node_fs = require("fs");
49121
+ var SDK_ROOT = "/root/.agents/sdk";
49122
+ var CLIENT_DIR = `${SDK_ROOT}/open-mcp-client`;
49123
+ var MANIFEST_PATH2 = `${SDK_ROOT}/.open-client-manifest.json`;
49124
+ async function provisionSandboxOpenClient(sandbox, bundle) {
49125
+ await sandbox.file.createDirectory(SDK_ROOT);
49126
+ await sandbox.file.createDirectory(CLIENT_DIR);
49127
+ await sandbox.file.writeFile(`${CLIENT_DIR}/index.js`, bundle.indexJs);
49128
+ await sandbox.file.writeFile(`${CLIENT_DIR}/cli.js`, bundle.cliJs);
49129
+ await sandbox.file.writeFile(MANIFEST_PATH2, JSON.stringify({ version: bundle.version }));
49130
+ }
49131
+ function buildBundleFromDist(distDir, version) {
49132
+ return {
49133
+ indexJs: (0, import_node_fs.readFileSync)(`${distDir}/index.js`, "utf8"),
49134
+ cliJs: (0, import_node_fs.readFileSync)(`${distDir}/cli.js`, "utf8"),
49135
+ version
49136
+ };
49137
+ }
49138
+
48170
49139
  // src/workflow/index.ts
48171
49140
  init_compile();
48172
49141
  init_parse_yaml();
@@ -48558,16 +49527,16 @@ var IdRemapper = class {
48558
49527
  };
48559
49528
 
48560
49529
  // src/util/directRoomId.ts
48561
- var import_node_crypto10 = require("crypto");
49530
+ var import_node_crypto14 = require("crypto");
48562
49531
  function generateDirectHumanProjectId(tenantId2, workspaceId, userId1, userId2) {
48563
49532
  const sortedUserIds = [userId1, userId2].sort();
48564
49533
  const parts = [tenantId2, workspaceId, ...sortedUserIds];
48565
- const hash = (0, import_node_crypto10.createHash)("sha256").update(parts.join(":")).digest("hex");
49534
+ const hash = (0, import_node_crypto14.createHash)("sha256").update(parts.join(":")).digest("hex");
48566
49535
  return `direct_human_${hash}`;
48567
49536
  }
48568
49537
  function generateDirectBotProjectId(tenantId2, workspaceId, userId, assistantId) {
48569
49538
  const parts = [tenantId2, workspaceId, userId, assistantId];
48570
- const hash = (0, import_node_crypto10.createHash)("sha256").update(parts.join(":")).digest("hex");
49539
+ const hash = (0, import_node_crypto14.createHash)("sha256").update(parts.join(":")).digest("hex");
48571
49540
  return `direct_bot_${hash}`;
48572
49541
  }
48573
49542
  function parseDirectRoomType(projectId) {
@@ -48636,10 +49605,12 @@ registerBuiltinPlugins();
48636
49605
  InMemoryChunkBuffer,
48637
49606
  InMemoryCollectionStore,
48638
49607
  InMemoryDatabaseConfigStore,
49608
+ InMemoryLeaseStore,
48639
49609
  InMemoryMailboxStore,
48640
49610
  InMemoryMenuStore,
48641
49611
  InMemoryModelProviderStore,
48642
49612
  InMemoryOpenAuditStore,
49613
+ InMemoryOutboxStore,
48643
49614
  InMemoryProjectBotMembershipStore,
48644
49615
  InMemoryProjectMembershipStore,
48645
49616
  InMemoryProjectRoomCoordinator,
@@ -48659,6 +49630,7 @@ registerBuiltinPlugins();
48659
49630
  LatticeEval,
48660
49631
  LatticeEvalProject,
48661
49632
  LatticeEvalSuite,
49633
+ LeaseAcquisitionTimeoutError,
48662
49634
  LocalSandboxInstance,
48663
49635
  LocalSandboxProvider,
48664
49636
  LoggerLatticeManager,
@@ -48681,7 +49653,10 @@ registerBuiltinPlugins();
48681
49653
  MicrosandboxServiceClient,
48682
49654
  ModelLatticeManager,
48683
49655
  MysqlDatabase,
49656
+ OutboxPoller,
48684
49657
  PACK_DIRECTORY_TIMEOUT_SECONDS,
49658
+ PLATFORM_SECRET_ENV_VARS,
49659
+ PLUGIN_CATALOG_CHANGED_TOPIC,
48685
49660
  PersonalAssistantConfig,
48686
49661
  PinoLoggerClient,
48687
49662
  PluginRegistry,
@@ -48702,6 +49677,8 @@ registerBuiltinPlugins();
48702
49677
  RepeatedToolCallError,
48703
49678
  RoomAgentMessageService,
48704
49679
  SAFE_PATH_COMPONENT,
49680
+ SANDBOX_OPEN_GATEWAY_URL_ENV,
49681
+ SANDBOX_OPEN_TOKEN_ENV,
48705
49682
  STTModelLattice,
48706
49683
  STTModelLatticeManager,
48707
49684
  SandboxFilesystem,
@@ -48742,11 +49719,13 @@ registerBuiltinPlugins();
48742
49719
  appendReconciliationWarning,
48743
49720
  appendUiFence,
48744
49721
  assertSafeUpstreamUrl,
49722
+ buildBundleFromDist,
48745
49723
  buildGrepResultsDict,
48746
49724
  buildInput,
48747
49725
  buildNamedVolumeName,
48748
49726
  buildRemotePackDirectoryCommand,
48749
49727
  buildSandboxMetadataEnv,
49728
+ buildSandboxOpenEnv,
48750
49729
  buildSkillFile,
48751
49730
  buildStateAnnotation,
48752
49731
  buildTableName,
@@ -48755,11 +49734,14 @@ registerBuiltinPlugins();
48755
49734
  checkEmptyContent,
48756
49735
  clearEncryptionKeyCache,
48757
49736
  clearEvalRunService,
49737
+ clearSandboxOpenCredentialProviderForTests,
48758
49738
  collectionLatticeManager,
48759
49739
  compileInternal,
48760
49740
  compileSandboxPlugin,
48761
49741
  compileWorkflow,
48762
49742
  computeSandboxName,
49743
+ configureCoordinationLeaseStore,
49744
+ configurePluginCatalogSync,
48763
49745
  configureStores,
48764
49746
  connectAllChannels,
48765
49747
  convertToConnection,
@@ -48791,6 +49773,7 @@ registerBuiltinPlugins();
48791
49773
  createSandboxResolver,
48792
49774
  createSchedulerMiddleware,
48793
49775
  createSharePayload,
49776
+ createShellExecTool,
48794
49777
  createTaskExecutionService,
48795
49778
  createTaskLifecycleService,
48796
49779
  createTaskMiddleware,
@@ -48837,6 +49820,7 @@ registerBuiltinPlugins();
48837
49820
  getCheckpointSaver,
48838
49821
  getChunkBuffer,
48839
49822
  getCollectionEntryCount,
49823
+ getCoordinationLeaseStore,
48840
49824
  getEmbeddingsClient,
48841
49825
  getEmbeddingsLattice,
48842
49826
  getEncryptionKey,
@@ -48848,15 +49832,18 @@ registerBuiltinPlugins();
48848
49832
  getOrCreateCollectionVectorStore,
48849
49833
  getPluginSkillManifestVersion,
48850
49834
  getQueueLattice,
49835
+ getReplicaId,
48851
49836
  getRuntimeCapabilityConfig,
48852
49837
  getRuntimeSkillSelection,
48853
49838
  getSTTClient,
48854
49839
  getSTTClientWithTenant,
48855
49840
  getSTTModelLattice,
48856
49841
  getSandBoxManager,
49842
+ getSandboxOpenCredential,
48857
49843
  getSandboxPluginCatalog,
48858
49844
  getScheduleLattice,
48859
49845
  getStoreLattice,
49846
+ getThreadRunOwner,
48860
49847
  getToolClient,
48861
49848
  getToolClientWithTenant,
48862
49849
  getToolDefinition,
@@ -48910,6 +49897,8 @@ registerBuiltinPlugins();
48910
49897
  projectMutationMutex,
48911
49898
  projectRoomPlugin,
48912
49899
  projectTaskAdmissionNotifier,
49900
+ provisionSandboxOpenClient,
49901
+ publishPluginCatalogChanged,
48913
49902
  queueLatticeManager,
48914
49903
  readLoopBreakerConfig,
48915
49904
  reconcileAllTenantTaskRuns,
@@ -48928,6 +49917,7 @@ registerBuiltinPlugins();
48928
49917
  registerModelLatticeForTenant,
48929
49918
  registerQueueLattice,
48930
49919
  registerSTTModelLattice,
49920
+ registerSandboxOpenCredentialProvider,
48931
49921
  registerSandboxProviderType,
48932
49922
  registerScheduleLattice,
48933
49923
  registerStoreLattice,
@@ -48938,6 +49928,7 @@ registerBuiltinPlugins();
48938
49928
  registerWorkflowRun,
48939
49929
  removeCollectionVectorStore,
48940
49930
  renderTemplate,
49931
+ resetReplicaIdForTests,
48941
49932
  resolveEnabledPluginSkill,
48942
49933
  resolveJudgeModelKey,
48943
49934
  resolvePath,
@@ -48945,6 +49936,7 @@ registerBuiltinPlugins();
48945
49936
  resolveSandboxPluginConnections,
48946
49937
  resolveTenantPluginSkill,
48947
49938
  roomCoordinatorConfig,
49939
+ runAsLeader,
48948
49940
  runRemotePackDirectory,
48949
49941
  safeTaskResponseId,
48950
49942
  sandboxLatticeManager,
@@ -48963,11 +49955,15 @@ registerBuiltinPlugins();
48963
49955
  storeLatticeManager,
48964
49956
  sttModelLatticeManager,
48965
49957
  taskLifecycleNotifier,
49958
+ taskRunLeaseKey,
48966
49959
  taskRunRegistry,
49960
+ threadRunLeaseKey,
48967
49961
  toJsonSchema,
48968
49962
  toSafeStateExpr,
48969
49963
  toolLatticeManager,
48970
49964
  truncateIfTooLong,
49965
+ tryAcquireTaskRun,
49966
+ tryAcquireThreadRun,
48971
49967
  unregisterTeammateAgent,
48972
49968
  unregisterWorkflowRun,
48973
49969
  updateFileData,
@@ -48983,6 +49979,7 @@ registerBuiltinPlugins();
48983
49979
  validateToolInput,
48984
49980
  vectorStoreLatticeManager,
48985
49981
  vectorStoreProviderManager,
49982
+ withLease,
48986
49983
  withTimeout,
48987
49984
  wrapMcpToolWithUi
48988
49985
  });