@axiom-lattice/local-stores 3.1.24 → 3.2.0

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @axiom-lattice/local-stores@3.1.24 build /home/runner/work/agentic/agentic/packages/local-stores
+> @axiom-lattice/local-stores@3.2.0 build /home/runner/work/agentic/agentic/packages/local-stores
 > tsup src/index.ts --format cjs,esm --dts --sourcemap
 
 CLI Building entry: src/index.ts
@@ -8,13 +8,13 @@
 CLI Target: es2020
 CJS Build start
 ESM Build start
-CJS dist/index.js     239.26 KB
-CJS dist/index.js.map 434.80 KB
-CJS ⚡️ Build success in 587ms
-ESM dist/index.mjs     234.63 KB
-ESM dist/index.mjs.map 431.04 KB
-ESM ⚡️ Build success in 599ms
+ESM dist/index.mjs     237.70 KB
+ESM dist/index.mjs.map 436.10 KB
+ESM ⚡️ Build success in 568ms
+CJS dist/index.js     242.32 KB
+CJS dist/index.js.map 439.84 KB
+CJS ⚡️ Build success in 569ms
 DTS Build start
-DTS ⚡️ Build success in 12512ms
-DTS dist/index.d.ts  44.04 KB
-DTS dist/index.d.mts 44.04 KB
+DTS ⚡️ Build success in 12893ms
+DTS dist/index.d.ts  44.67 KB
+DTS dist/index.d.mts 44.67 KB

package/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @axiom-lattice/local-stores
 
+## 3.2.0
+
+### Minor Changes
+
+- eafaac8: Multi-gateway (PostgreSQL-only) coordination primitives — Plans 1 & 2.
+
+  **Plan 1 — replica identity + distributed leases**
+
+  - `getReplicaId()` stable per-process replica identity (`AXIOM_REPLICA_ID` or `hostname:pid`).
+  - `LeaseStore` / `LeaseRecord` protocol, `InMemoryLeaseStore` (registered as the default `"lease"` store), and `PostgreSQLLeaseStore` (migration v185 `lattice_leases`; atomic self-expiring upsert).
+  - `withLease(store, key, ownerLabel, fn)` — distributed critical-section helper with a unique per-invocation holder, bounded acquisition, heartbeat, and release-in-finally.
+
+  **Plan 2 — atomic thread-message-queue claim + lease**
+
+  - `ClaimMessagesOptions` and `IMessageQueueStore.claimPendingMessages` / `reclaimExpiredProcessing`.
+  - PostgreSQL migration v186 adds `locked_by` / `locked_at` / `lease_expires_at`; claim is an atomic `UPDATE … WHERE id IN (SELECT … FOR UPDATE SKIP LOCKED)` so concurrent Gateway replicas receive disjoint sets.
+  - In-memory and local SQLite implementations match the same semantics.
+
+  No existing behavior changes: the legacy `markProcessing` path is untouched; `Agent` is not yet rewired to the new claim (that is a follow-up plan).
+
+- 6080389: Scheduler exactly-once across replicas (claim-before-execute).
+
+  - `ScheduleStorage.claimScheduledRun(taskId)` atomically transitions `pending -> running` (Memory/Local/PostgreSQL implementations).
+  - `DefaultScheduleClient.executeTask` claims before running the handler, so a fire armed on multiple replicas executes only once (losers skip). Retry/CRON-rearm paths re-set `PENDING`, so later fires are claimable. See ADR-158.
+
+- 7ecbd01: Recover scheduled tasks stuck `running` after a crash.
+
+  - `ScheduleStorage.requeueStaleRunning(olderThanMs)` requeues `running` tasks older than a threshold back to `pending` (Memory/Local/PostgreSQL).
+  - `DefaultScheduleClient.restore()` requeues stale `running` schedules (30-minute heuristic) before re-arming, so a CRON schedule is no longer permanently stalled by a crash that left an unsettled claim. See ADR-158.
+
+### Patch Changes
+
+- Updated dependencies [6addbbf]
+- Updated dependencies [66e6080]
+- Updated dependencies [360f9e9]
+- Updated dependencies [eafaac8]
+- Updated dependencies [b8c4c92]
+- Updated dependencies [4c99955]
+- Updated dependencies [167620b]
+- Updated dependencies [d3cd2d5]
+- Updated dependencies [396ad50]
+- Updated dependencies [eec9db7]
+- Updated dependencies [d4429f1]
+- Updated dependencies [6080389]
+- Updated dependencies [7ecbd01]
+- Updated dependencies [e5e7b0f]
+- Updated dependencies [00b179d]
+- Updated dependencies [5f3d14b]
+  - @axiom-lattice/core@6.3.0
+  - @axiom-lattice/protocols@4.7.0
+
 ## 3.1.24
 
 ### Patch Changes

package/dist/index.d.mts

@@ -1,7 +1,7 @@
 import { SqlJsStatic, Database } from 'sql.js';
 import { SqliteSaver } from '@langchain/langgraph-checkpoint-sqlite';
 import { ThreadStore, Thread, CreateThreadRequest, AssistantStore, Assistant, CreateAssistantRequest, WorkspaceStore, Workspace, CreateWorkspaceRequest, UpdateWorkspaceRequest, ProjectStore, ProjectFilter, Project, CreateProjectRequest, UpdateProjectRequest, UpdateProjectCapabilityBundlesResult, ProjectRoomStore, ProjectRoom, ProjectMembershipStore, ProjectMembership, ProjectHumanRole, ProjectMembershipMutationResult, ProjectBotMembershipStore, ProjectBotMembership, ProjectRoomMessageStore, ProjectRoomMessage, ProjectRoomReadStateStore, ProjectRoomReadState, UserStore, User, CreateUserRequest, UpdateUserRequest, TenantStore, Tenant, CreateTenantRequest, UpdateTenantRequest, UserTenantLinkStore, UserTenantLink, CreateUserTenantLinkRequest, UpdateUserTenantLinkRequest, DatabaseConfigStore, DatabaseConfigEntry, CreateDatabaseConfigRequest, UpdateDatabaseConfigRequest, ConnectionStore, ConnectionEntry, MetricsServerConfigStore, MetricsServerConfigEntry, CreateMetricsServerConfigRequest, UpdateMetricsServerConfigRequest, McpServerConfigStore, McpServerConfigEntry, CreateMcpServerConfigRequest, UpdateMcpServerConfigRequest, ModelProviderStore, ModelProviderEntry, CreateModelProviderRequest, UpdateModelProviderRequest, ModelProviderStatus, ModelProviderModelEntry, ProviderModelInput, WorkflowTrackingStore, CreateWorkflowRunRequest, WorkflowRun, UpdateWorkflowRunRequest, QueryWorkflowRunsOptions, QueryWorkflowRunsResult, CreateRunStepRequest, RunStep, UpdateRunStepRequest, StepType, EvalStore, EvalProject, CreateEvalProjectRequest, EvalSuite, CreateEvalSuiteRequest, EvalCase, CreateEvalCaseRequest, EvalRun, CreateEvalRunRequest, EvalRunResult, EvalProjectReport, BindingRegistry, Binding, CreateBindingInput, BindingMutablePatch, BindingListParams, ChannelInstallationStore, ChannelInstallation, ChannelInstallationType, CreateChannelInstallationInput, UpdateChannelInstallationRequest, A2AApiKeyStore, A2AApiKeyRecord, CreateA2AApiKeyInput, UpdateA2AApiKeyInput, A2AApiKeyEntry, OpenAuditStore, OpenAuditQuery, OpenAuditRecord, OpenAuditAppendInput, SkillStore, SkillStoreContext, Skill, CreateSkillRequest, ScheduleStorage, ScheduledTaskDefinition, ScheduledTaskStatus, ScheduleExecutionType, TaskStore, CreateTaskRequest, TaskItem, TaskListFilter, TaskDependentListQuery, UpdateTaskRequest, TaskMutationSnapshot, TaskWorkItemStore, CreateWorkItemRequest, TaskWorkItem, CreateWorkItemIfAbsentRequest, TaskWorkItemListFilter, ProjectLifecycleEventQuery, CapabilityBundleStore, CapabilityBundle, InternalCreateCapabilityBundleInput, InternalUpdateCapabilityBundleInput, CapabilityBundleDeleteResult, IConversationStore, ConversationRecord, CreateConversationInput } from '@axiom-lattice/protocols';
-import { IMessageQueueStore, AddMessageParams, PendingMessage, QueueScope, ThreadInfo } from '@axiom-lattice/core';
+import { IMessageQueueStore, AddMessageParams, PendingMessage, QueueScope, ClaimMessagesOptions, ThreadInfo } from '@axiom-lattice/core';
 
 /**
  * Shared SQLite database connection for local stores using sql.js (WASM).
@@ -720,6 +720,8 @@ declare class LocalThreadMessageQueueStore implements IMessageQueueStore {
     addMessageAtHead(params: AddMessageParams): Promise<PendingMessage>;
     getPendingMessages(threadId: string, scope?: QueueScope): Promise<PendingMessage[]>;
     getProcessingMessages(threadId: string, scope?: QueueScope): Promise<PendingMessage[]>;
+    claimPendingMessages(threadId: string, scope: QueueScope | undefined, options: ClaimMessagesOptions): Promise<PendingMessage[]>;
+    reclaimExpiredProcessing(threadId: string, scope?: QueueScope): Promise<number>;
     getQueueSize(threadId: string, scope?: QueueScope): Promise<number>;
     getThreadsWithPendingMessages(): Promise<ThreadInfo[]>;
     removeMessage(messageId: string, scope?: QueueScope): Promise<boolean>;
@@ -760,6 +762,15 @@ declare class LocalScheduleStorage implements ScheduleStorage {
     save(task: ScheduledTaskDefinition): Promise<void>;
     get(taskId: string): Promise<ScheduledTaskDefinition | null>;
     update(taskId: string, updates: Partial<ScheduledTaskDefinition>): Promise<void>;
+    claimScheduledRun(taskId: string): Promise<boolean>;
+    /**
+     * Requeues `running` tasks whose `updated_at` is older than `olderThanMs` back
+     * to `pending` (recovery after a crash that left a claim unsettled).
+     *
+     * @param olderThanMs Age threshold in milliseconds.
+     * @returns The number of tasks requeued.
+     */
+    requeueStaleRunning(olderThanMs: number): Promise<number>;
     delete(taskId: string): Promise<void>;
     getActiveTasks(): Promise<ScheduledTaskDefinition[]>;
     getTasksByType(taskType: string): Promise<ScheduledTaskDefinition[]>;

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import { SqlJsStatic, Database } from 'sql.js';
 import { SqliteSaver } from '@langchain/langgraph-checkpoint-sqlite';
 import { ThreadStore, Thread, CreateThreadRequest, AssistantStore, Assistant, CreateAssistantRequest, WorkspaceStore, Workspace, CreateWorkspaceRequest, UpdateWorkspaceRequest, ProjectStore, ProjectFilter, Project, CreateProjectRequest, UpdateProjectRequest, UpdateProjectCapabilityBundlesResult, ProjectRoomStore, ProjectRoom, ProjectMembershipStore, ProjectMembership, ProjectHumanRole, ProjectMembershipMutationResult, ProjectBotMembershipStore, ProjectBotMembership, ProjectRoomMessageStore, ProjectRoomMessage, ProjectRoomReadStateStore, ProjectRoomReadState, UserStore, User, CreateUserRequest, UpdateUserRequest, TenantStore, Tenant, CreateTenantRequest, UpdateTenantRequest, UserTenantLinkStore, UserTenantLink, CreateUserTenantLinkRequest, UpdateUserTenantLinkRequest, DatabaseConfigStore, DatabaseConfigEntry, CreateDatabaseConfigRequest, UpdateDatabaseConfigRequest, ConnectionStore, ConnectionEntry, MetricsServerConfigStore, MetricsServerConfigEntry, CreateMetricsServerConfigRequest, UpdateMetricsServerConfigRequest, McpServerConfigStore, McpServerConfigEntry, CreateMcpServerConfigRequest, UpdateMcpServerConfigRequest, ModelProviderStore, ModelProviderEntry, CreateModelProviderRequest, UpdateModelProviderRequest, ModelProviderStatus, ModelProviderModelEntry, ProviderModelInput, WorkflowTrackingStore, CreateWorkflowRunRequest, WorkflowRun, UpdateWorkflowRunRequest, QueryWorkflowRunsOptions, QueryWorkflowRunsResult, CreateRunStepRequest, RunStep, UpdateRunStepRequest, StepType, EvalStore, EvalProject, CreateEvalProjectRequest, EvalSuite, CreateEvalSuiteRequest, EvalCase, CreateEvalCaseRequest, EvalRun, CreateEvalRunRequest, EvalRunResult, EvalProjectReport, BindingRegistry, Binding, CreateBindingInput, BindingMutablePatch, BindingListParams, ChannelInstallationStore, ChannelInstallation, ChannelInstallationType, CreateChannelInstallationInput, UpdateChannelInstallationRequest, A2AApiKeyStore, A2AApiKeyRecord, CreateA2AApiKeyInput, UpdateA2AApiKeyInput, A2AApiKeyEntry, OpenAuditStore, OpenAuditQuery, OpenAuditRecord, OpenAuditAppendInput, SkillStore, SkillStoreContext, Skill, CreateSkillRequest, ScheduleStorage, ScheduledTaskDefinition, ScheduledTaskStatus, ScheduleExecutionType, TaskStore, CreateTaskRequest, TaskItem, TaskListFilter, TaskDependentListQuery, UpdateTaskRequest, TaskMutationSnapshot, TaskWorkItemStore, CreateWorkItemRequest, TaskWorkItem, CreateWorkItemIfAbsentRequest, TaskWorkItemListFilter, ProjectLifecycleEventQuery, CapabilityBundleStore, CapabilityBundle, InternalCreateCapabilityBundleInput, InternalUpdateCapabilityBundleInput, CapabilityBundleDeleteResult, IConversationStore, ConversationRecord, CreateConversationInput } from '@axiom-lattice/protocols';
-import { IMessageQueueStore, AddMessageParams, PendingMessage, QueueScope, ThreadInfo } from '@axiom-lattice/core';
+import { IMessageQueueStore, AddMessageParams, PendingMessage, QueueScope, ClaimMessagesOptions, ThreadInfo } from '@axiom-lattice/core';
 
 /**
  * Shared SQLite database connection for local stores using sql.js (WASM).
@@ -720,6 +720,8 @@ declare class LocalThreadMessageQueueStore implements IMessageQueueStore {
     addMessageAtHead(params: AddMessageParams): Promise<PendingMessage>;
     getPendingMessages(threadId: string, scope?: QueueScope): Promise<PendingMessage[]>;
     getProcessingMessages(threadId: string, scope?: QueueScope): Promise<PendingMessage[]>;
+    claimPendingMessages(threadId: string, scope: QueueScope | undefined, options: ClaimMessagesOptions): Promise<PendingMessage[]>;
+    reclaimExpiredProcessing(threadId: string, scope?: QueueScope): Promise<number>;
     getQueueSize(threadId: string, scope?: QueueScope): Promise<number>;
     getThreadsWithPendingMessages(): Promise<ThreadInfo[]>;
     removeMessage(messageId: string, scope?: QueueScope): Promise<boolean>;
@@ -760,6 +762,15 @@ declare class LocalScheduleStorage implements ScheduleStorage {
     save(task: ScheduledTaskDefinition): Promise<void>;
     get(taskId: string): Promise<ScheduledTaskDefinition | null>;
     update(taskId: string, updates: Partial<ScheduledTaskDefinition>): Promise<void>;
+    claimScheduledRun(taskId: string): Promise<boolean>;
+    /**
+     * Requeues `running` tasks whose `updated_at` is older than `olderThanMs` back
+     * to `pending` (recovery after a crash that left a claim unsettled).
+     *
+     * @param olderThanMs Age threshold in milliseconds.
+     * @returns The number of tasks requeued.
+     */
+    requeueStaleRunning(olderThanMs: number): Promise<number>;
     delete(taskId: string): Promise<void>;
     getActiveTasks(): Promise<ScheduledTaskDefinition[]>;
     getTasksByType(taskType: string): Promise<ScheduledTaskDefinition[]>;

package/dist/index.js

@@ -4580,6 +4580,9 @@ CREATE TABLE IF NOT EXISTS lt_thread_message_queue (
   custom_run_config TEXT,
   trusted_run_context TEXT,
   execution_mode TEXT CHECK (execution_mode IS NULL OR execution_mode = 'followup'),
+  locked_by TEXT,
+  locked_at TEXT,
+  lease_expires_at TEXT,
   created_at TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_lt_tmq_thread ON lt_thread_message_queue(thread_id, status);
@@ -4598,6 +4601,16 @@ var LocalThreadMessageQueueStore = class {
     if (!names.has("execution_mode")) {
       db.run("ALTER TABLE lt_thread_message_queue ADD COLUMN execution_mode TEXT CHECK (execution_mode IS NULL OR execution_mode = 'followup')");
     }
+    if (!names.has("locked_by")) {
+      db.run("ALTER TABLE lt_thread_message_queue ADD COLUMN locked_by TEXT");
+    }
+    if (!names.has("locked_at")) {
+      db.run("ALTER TABLE lt_thread_message_queue ADD COLUMN locked_at TEXT");
+    }
+    if (!names.has("lease_expires_at")) {
+      db.run("ALTER TABLE lt_thread_message_queue ADD COLUMN lease_expires_at TEXT");
+    }
+    db.run("CREATE INDEX IF NOT EXISTS idx_lt_tmq_lease ON lt_thread_message_queue(status, lease_expires_at)");
   }
   async addMessage(params) {
     const trusted = validateQueueTrust(params);
@@ -4722,6 +4735,37 @@ var LocalThreadMessageQueueStore = class {
     ).all(threadId, ...filter.params);
     return rows2.map(rowToMessage);
   }
+  async claimPendingMessages(threadId, scope, options) {
+    const filter = scopeClause(scope);
+    const limitClause = options.limit === void 0 ? "" : ` LIMIT ${Math.max(0, Math.floor(options.limit))}`;
+    const claimedRows = transaction(this.db, () => {
+      const candidates = this.db.prepare(
+        `SELECT * FROM lt_thread_message_queue WHERE thread_id = ? AND status = 'pending'${filter.sql} ORDER BY priority DESC, sequence_order ASC${limitClause}`
+      ).all(threadId, ...filter.params);
+      const startedAt = Date.now();
+      const lockedAt = new Date(startedAt).toISOString();
+      const leaseExpiresAt = new Date(startedAt + options.ttlMs).toISOString();
+      const claimed = [];
+      for (const row of candidates) {
+        const changes = write(
+          this.db,
+          `UPDATE lt_thread_message_queue SET status = 'processing', locked_by = ?, locked_at = ?, lease_expires_at = ? WHERE id = ? AND status = 'pending'`,
+          [options.owner, lockedAt, leaseExpiresAt, row.id]
+        );
+        if (changes > 0) claimed.push(row);
+      }
+      return claimed;
+    });
+    return claimedRows.map(rowToMessage);
+  }
+  async reclaimExpiredProcessing(threadId, scope) {
+    const filter = scopeClause(scope);
+    return transaction(this.db, () => write(
+      this.db,
+      `UPDATE lt_thread_message_queue SET status = 'pending', locked_by = NULL, locked_at = NULL, lease_expires_at = NULL WHERE thread_id = ? AND status = 'processing' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?${filter.sql}`,
+      [threadId, nowISO(), ...filter.params]
+    ));
+  }
   async getQueueSize(threadId, scope) {
     const filter = scopeClause(scope);
     const row = this.db.prepare(
@@ -4763,7 +4807,7 @@ var LocalThreadMessageQueueStore = class {
   async resetProcessingToPending(threadId, scope) {
     const filter = scopeClause(scope);
     const result = this.db.prepare(
-      `UPDATE lt_thread_message_queue SET status = 'pending' WHERE thread_id = ? AND status = 'processing'${filter.sql}`
+      `UPDATE lt_thread_message_queue SET status = 'pending', locked_by = NULL, locked_at = NULL, lease_expires_at = NULL WHERE thread_id = ? AND status = 'processing'${filter.sql}`
     ).run(threadId, ...filter.params);
     return result.changes;
   }
@@ -5149,6 +5193,27 @@ var LocalScheduleStorage = class {
       `UPDATE lt_scheduled_tasks SET ${setClauses.join(", ")} WHERE task_id = ?`
     ).run(...values);
   }
+  async claimScheduledRun(taskId) {
+    const result = this.db.prepare(
+      `UPDATE lt_scheduled_tasks SET status = 'running', updated_at = ? WHERE task_id = ? AND status = 'pending'`
+    ).run((/* @__PURE__ */ new Date()).toISOString(), taskId);
+    return result.changes > 0;
+  }
+  /**
+   * Requeues `running` tasks whose `updated_at` is older than `olderThanMs` back
+   * to `pending` (recovery after a crash that left a claim unsettled).
+   *
+   * @param olderThanMs Age threshold in milliseconds.
+   * @returns The number of tasks requeued.
+   */
+  async requeueStaleRunning(olderThanMs) {
+    const now = /* @__PURE__ */ new Date();
+    const cutoff = new Date(now.getTime() - olderThanMs).toISOString();
+    const result = this.db.prepare(
+      `UPDATE lt_scheduled_tasks SET status = 'pending', updated_at = ? WHERE status = 'running' AND updated_at < ?`
+    ).run(now.toISOString(), cutoff);
+    return result.changes;
+  }
   async delete(taskId) {
     this.db.prepare(`DELETE FROM lt_scheduled_tasks WHERE task_id = ?`).run(taskId);
   }