@memberjunction/task-graph 6.1.2 → 6.2.0-edge.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,7 +21,7 @@
21
21
  *
22
22
  * @module @memberjunction/task-graph
23
23
  */
24
- import { LogError, LogStatus } from '@memberjunction/core';
24
+ import { LogError, LogStatus, RunView } from '@memberjunction/core';
25
25
  import { TERMINAL_TASK_GRAPH_STATUSES } from '@memberjunction/ai-core-plus';
26
26
  import { MachineTaskSQL } from './task-predicates.js';
27
27
  /**
@@ -85,18 +85,34 @@ export class TaskClaimStore {
85
85
  constructor(instanceID, claimTTLSeconds) {
86
86
  this.instanceID = instanceID;
87
87
  this.claimTTLSeconds = claimTTLSeconds;
88
+ /** The last guarded write's failure, or null when the last one actually ran. */
89
+ this._lastWriteError = null;
90
+ this._consecutiveWriteFailures = 0;
91
+ }
92
+ /**
93
+ * Whether the most recent guarded write FAILED, as opposed to losing its race.
94
+ *
95
+ * The dispatcher reads this after a false return: "another instance won" and "this process
96
+ * cannot write to the database at all" produce the same `false`, and treating the second as the
97
+ * first is what let a dispatcher skip every task in the table, forever, in silence.
98
+ */
99
+ get LastWriteFailed() {
100
+ return this._lastWriteError !== null;
101
+ }
102
+ /** The last failure's message, for a caller that wants to say why it is stuck. */
103
+ get LastWriteError() {
104
+ return this._lastWriteError;
105
+ }
106
+ /** How many guarded writes have failed in a row. Reset by the first one that runs. */
107
+ get ConsecutiveWriteFailures() {
108
+ return this._consecutiveWriteFailures;
88
109
  }
89
110
  sql(provider) {
90
111
  return provider;
91
112
  }
92
- /** Schema-qualified `Task` table for the provider's configured core schema. */
93
- taskTable(provider) {
94
- const db = this.sql(provider);
95
- return `${db.QuoteIdentifier(db.MJCoreSchemaName)}.${db.QuoteIdentifier('Task')}`;
96
- }
97
- agentRunTable(provider) {
98
- const db = this.sql(provider);
99
- return `${db.QuoteIdentifier(db.MJCoreSchemaName)}.${db.QuoteIdentifier('AIAgentRun')}`;
113
+ /** The claim TTL as whole seconds, which is what the procedures take. */
114
+ ttlSeconds() {
115
+ return Math.max(0, Math.round(this.claimTTLSeconds));
100
116
  }
101
117
  /**
102
118
  * Writes a graph's cost rollup onto the submitting run, those four columns and no others.
@@ -108,16 +124,13 @@ export class TaskClaimStore {
108
124
  * the run `Paused` under a claimed marker, which no sweep re-enters.
109
125
  */
110
126
  async TrySetRunCostRollup(provider, runID, totals, contextUser) {
111
- const db = this.sql(provider);
112
- const num = (v) => (v == null ? 'NULL' : String(v));
113
- const sql = `
114
- UPDATE ${this.agentRunTable(provider)}
115
- SET ${db.QuoteIdentifier('TotalCostRollup')} = ${num(totals.Cost)},
116
- ${db.QuoteIdentifier('TotalTokensUsedRollup')} = ${num(totals.Tokens)},
117
- ${db.QuoteIdentifier('TotalPromptTokensUsedRollup')} = ${num(totals.PromptTokens)},
118
- ${db.QuoteIdentifier('TotalCompletionTokensUsedRollup')} = ${num(totals.CompletionTokens)}
119
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(runID)}'`;
120
- return (await this.affectedRows(db, sql, contextUser)) === 1;
127
+ return this.guardedWrite(provider, 'spTaskGraphSetRunCostRollup', [
128
+ { Name: 'AgentRunID', Value: runID },
129
+ { Name: 'TotalCostRollup', Value: totals.Cost },
130
+ { Name: 'TotalTokensUsedRollup', Value: totals.Tokens },
131
+ { Name: 'TotalPromptTokensUsedRollup', Value: totals.PromptTokens },
132
+ { Name: 'TotalCompletionTokensUsedRollup', Value: totals.CompletionTokens },
133
+ ], contextUser);
121
134
  }
122
135
  /**
123
136
  * Settles a parked agent run, guarded on it still being parked.
@@ -127,18 +140,11 @@ export class TaskClaimStore {
127
140
  * across instances rather than last-write-wins.
128
141
  */
129
142
  async TrySettleRun(provider, runID, succeeded, errorMessage, contextUser) {
130
- const db = this.sql(provider);
131
- const errorClause = errorMessage == null
132
- ? ''
133
- : `, ${db.QuoteIdentifier('ErrorMessage')} = CONCAT(COALESCE(${db.QuoteIdentifier('ErrorMessage')} + CHAR(10) + CHAR(10), ''), '${this.escape(errorMessage)}')`;
134
- const sql = `
135
- UPDATE ${this.agentRunTable(provider)}
136
- SET ${db.QuoteIdentifier('Status')} = '${succeeded ? 'Completed' : 'Failed'}',
137
- ${db.QuoteIdentifier('Success')} = ${succeeded ? 1 : 0},
138
- ${db.QuoteIdentifier('CompletedAt')} = SYSUTCDATETIME()${errorClause}
139
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(runID)}'
140
- AND ${db.QuoteIdentifier('Status')} = 'Paused'`;
141
- return (await this.affectedRows(db, sql, contextUser)) === 1;
143
+ return this.guardedWrite(provider, 'spTaskGraphSettleRun', [
144
+ { Name: 'AgentRunID', Value: runID },
145
+ { Name: 'Succeeded', Value: succeeded },
146
+ { Name: 'ErrorMessage', Value: errorMessage },
147
+ ], contextUser);
142
148
  }
143
149
  /**
144
150
  * Attempts to claim one task.
@@ -151,24 +157,15 @@ export class TaskClaimStore {
151
157
  * @returns true when this instance now owns the task
152
158
  */
153
159
  async TryClaim(provider, taskID, contextUser) {
154
- const db = this.sql(provider);
155
160
  // The lease is written AND compared on the database's clock (SYSUTCDATETIME), never this
156
161
  // process's. The claim protocol is multi-instance: a lease written from one host's clock and
157
162
  // judged expired against another's turns ordinary NTP skew into premature reclamation — the
158
163
  // task runs twice — or into a lease that outlives its worker. One clock, the only shared one.
159
- const ttlSeconds = Math.max(0, Math.round(this.claimTTLSeconds));
160
- const sql = `
161
- UPDATE ${this.taskTable(provider)}
162
- SET ${db.QuoteIdentifier('Status')} = 'In Progress',
163
- ${db.QuoteIdentifier('ClaimedBy')} = '${this.escape(this.instanceID)}',
164
- ${db.QuoteIdentifier('ClaimExpiresAt')} = DATEADD(SECOND, ${ttlSeconds}, SYSUTCDATETIME()),
165
- ${db.QuoteIdentifier('StartedAt')} = SYSUTCDATETIME()
166
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
167
- AND ${db.QuoteIdentifier('Status')} = 'Pending'
168
- AND (${db.QuoteIdentifier('ClaimedBy')} IS NULL
169
- OR ${db.QuoteIdentifier('ClaimExpiresAt')} IS NULL
170
- OR ${db.QuoteIdentifier('ClaimExpiresAt')} < SYSUTCDATETIME())`;
171
- return (await this.affectedRows(db, sql, contextUser)) === 1;
164
+ return this.guardedWrite(provider, 'spTaskGraphClaimTask', [
165
+ { Name: 'TaskID', Value: taskID },
166
+ { Name: 'ClaimedBy', Value: this.instanceID },
167
+ { Name: 'ClaimTTLSeconds', Value: this.ttlSeconds() },
168
+ ], contextUser);
172
169
  }
173
170
  /**
174
171
  * Extends this instance's claim on a task it is actively running.
@@ -180,16 +177,12 @@ export class TaskClaimStore {
180
177
  * @returns true when the claim was extended; false means this instance no longer owns the task
181
178
  */
182
179
  async Heartbeat(provider, taskID, contextUser) {
183
- const db = this.sql(provider);
184
180
  // Same single-clock rule as TryClaim: the renewal is computed on the database's clock.
185
- const ttlSeconds = Math.max(0, Math.round(this.claimTTLSeconds));
186
- const sql = `
187
- UPDATE ${this.taskTable(provider)}
188
- SET ${db.QuoteIdentifier('ClaimExpiresAt')} = DATEADD(SECOND, ${ttlSeconds}, SYSUTCDATETIME())
189
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
190
- AND ${db.QuoteIdentifier('ClaimedBy')} = '${this.escape(this.instanceID)}'
191
- AND ${db.QuoteIdentifier('Status')} = 'In Progress'`;
192
- return (await this.affectedRows(db, sql, contextUser)) === 1;
181
+ return this.guardedWrite(provider, 'spTaskGraphHeartbeat', [
182
+ { Name: 'TaskID', Value: taskID },
183
+ { Name: 'ClaimedBy', Value: this.instanceID },
184
+ { Name: 'ClaimTTLSeconds', Value: this.ttlSeconds() },
185
+ ], contextUser);
193
186
  }
194
187
  /**
195
188
  * Records a terminal outcome and releases the claim in one guarded statement.
@@ -201,31 +194,18 @@ export class TaskClaimStore {
201
194
  * @returns true when this instance's outcome was recorded
202
195
  */
203
196
  async CompleteClaimed(provider, taskID, outcome, contextUser) {
204
- const db = this.sql(provider);
205
- const sets = [
206
- `${db.QuoteIdentifier('Status')} = '${outcome.Status}'`,
207
- `${db.QuoteIdentifier('CompletedAt')} = SYSUTCDATETIME()`,
208
- `${db.QuoteIdentifier('PercentComplete')} = ${outcome.Status === 'Complete' ? 100 : 0}`,
209
- // Release the claim as part of the same atomic write — a separate release could be
210
- // interrupted, leaving a terminal task holding a claim that the sweep would then flag.
211
- `${db.QuoteIdentifier('ClaimedBy')} = NULL`,
212
- `${db.QuoteIdentifier('ClaimExpiresAt')} = NULL`,
213
- ];
214
- sets.push(`${db.QuoteIdentifier('OutputPayload')} = ${this.literalOrNull(outcome.OutputPayload)}`);
215
- sets.push(`${db.QuoteIdentifier('ErrorMessage')} = ${this.literalOrNull(outcome.ErrorMessage)}`);
216
- sets.push(`${db.QuoteIdentifier('AgentRunID')} = ${outcome.AgentRunID ? `'${this.escape(outcome.AgentRunID)}'` : 'NULL'}`);
217
- // Only when supplied — see the note on the parameter. `undefined` means "leave it alone",
218
- // which is not the same as an explicit null.
219
- if (outcome.Configuration !== undefined) {
220
- sets.push(`${db.QuoteIdentifier('Configuration')} = ${this.literalOrNull(outcome.Configuration)}`);
221
- }
222
- const sql = `
223
- UPDATE ${this.taskTable(provider)}
224
- SET ${sets.join(', ')}
225
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
226
- AND ${db.QuoteIdentifier('Status')} = 'In Progress'
227
- AND ${db.QuoteIdentifier('ClaimedBy')} = '${this.escape(this.instanceID)}'`;
228
- return (await this.affectedRows(db, sql, contextUser)) === 1;
197
+ return this.guardedWrite(provider, 'spTaskGraphCompleteClaimed', [
198
+ { Name: 'TaskID', Value: taskID },
199
+ { Name: 'ClaimedBy', Value: this.instanceID },
200
+ { Name: 'Status', Value: outcome.Status },
201
+ { Name: 'OutputPayload', Value: outcome.OutputPayload ?? null },
202
+ { Name: 'ErrorMessage', Value: outcome.ErrorMessage ?? null },
203
+ { Name: 'AgentRunID', Value: outcome.AgentRunID ?? null },
204
+ // `undefined` means "leave it alone", which is not the same as an explicit null — so the
205
+ // flag, not the value, decides whether the column is written.
206
+ { Name: 'Configuration', Value: outcome.Configuration ?? null },
207
+ { Name: 'SetConfiguration', Value: outcome.Configuration !== undefined },
208
+ ], contextUser);
229
209
  }
230
210
  /**
231
211
  * Reclaims tasks whose claims have lapsed, returning them to `Pending` so any instance can pick
@@ -246,34 +226,52 @@ export class TaskClaimStore {
246
226
  */
247
227
  async ReleaseExpiredClaims(provider, contextUser) {
248
228
  const db = this.sql(provider);
249
- // Capture what will be reclaimed BEFORE reclaiming, so the log names the tasks. The
250
- // subsequent UPDATE re-states the same predicate, so a task whose claim was refreshed in
251
- // between is correctly skipped rather than reclaimed on stale information.
252
- const candidates = await db.ExecuteSQL(`SELECT ${db.QuoteIdentifier('ID')}, ${db.QuoteIdentifier('Name')}, ${db.QuoteIdentifier('ClaimedBy')}
253
- FROM ${this.taskTable(provider)}
254
- WHERE ${db.QuoteIdentifier('Status')} = 'In Progress'
255
- AND ${MachineTaskSQL(db.QuoteIdentifier.bind(db))}
256
- AND ${db.QuoteIdentifier('ClaimedBy')} IS NOT NULL
257
- AND ${db.QuoteIdentifier('ClaimExpiresAt')} IS NOT NULL
258
- AND ${db.QuoteIdentifier('ClaimExpiresAt')} < SYSUTCDATETIME()`, undefined, undefined, contextUser);
259
- if (!candidates || candidates.length === 0)
229
+ // Read from the base VIEW, which is what the runtime roles are granted (#4575) — and which
230
+ // also keeps the "a dispatcher completes this task" definition in the one module that owns
231
+ // it, rather than restating it inside a procedure where it would drift.
232
+ //
233
+ // Capture what will be reclaimed BEFORE reclaiming, so the log can name the tasks. The
234
+ // procedure re-states the LEASE predicate, which is the part that has to be evaluated at
235
+ // write time: a claim refreshed in between is correctly skipped rather than reclaimed on
236
+ // stale information.
237
+ const candidates = await RunView.FromMetadataProvider(provider).RunView({
238
+ EntityName: 'MJ: Tasks',
239
+ ExtraFilter: `Status='In Progress' AND ${MachineTaskSQL()} AND ClaimedBy IS NOT NULL ` +
240
+ `AND ClaimExpiresAt IS NOT NULL AND ClaimExpiresAt < ${db.Dialect.CurrentTimestampUTC()}`,
241
+ Fields: ['ID', 'Name', 'ClaimedBy'],
242
+ ResultType: 'simple',
243
+ // The statement this replaced was an unbounded SELECT. Without this, `MJ: Tasks`
244
+ // contributes its UserViewMaxRows (1000) as a TOP clause, and the sweep would
245
+ // silently reclaim only the first 1000 expired claims per pass — a backlog that
246
+ // never drains, reported as a completed sweep.
247
+ IgnoreMaxRows: true,
248
+ // The claim protocol mutates these rows out from under any cache; a stale read here
249
+ // would reclaim a task somebody is still running.
250
+ BypassCache: true,
251
+ }, contextUser);
252
+ if (!candidates.Success) {
253
+ LogError(`[TaskGraph reconciliation] could not read expired-claim candidates: ${candidates.ErrorMessage}`);
260
254
  return [];
261
- const sql = `
262
- UPDATE ${this.taskTable(provider)}
263
- SET ${db.QuoteIdentifier('Status')} = 'Pending',
264
- ${db.QuoteIdentifier('ClaimedBy')} = NULL,
265
- ${db.QuoteIdentifier('ClaimExpiresAt')} = NULL
266
- WHERE ${db.QuoteIdentifier('Status')} = 'In Progress'
267
- AND ${MachineTaskSQL(db.QuoteIdentifier.bind(db))}
268
- AND ${db.QuoteIdentifier('ClaimedBy')} IS NOT NULL
269
- AND ${db.QuoteIdentifier('ClaimExpiresAt')} IS NOT NULL
270
- AND ${db.QuoteIdentifier('ClaimExpiresAt')} < SYSUTCDATETIME()`;
271
- const released = await this.affectedRows(db, sql, contextUser);
272
- const events = candidates.slice(0, released).map((c) => ({
273
- TaskID: c.ID,
274
- Action: 'ExpiredClaimReleased',
275
- Detail: `Claim held by '${c.ClaimedBy}' expired; task '${c.Name}' returned to Pending.`,
276
- }));
255
+ }
256
+ const rows = candidates.Results ?? [];
257
+ if (rows.length === 0)
258
+ return [];
259
+ // The procedure reports which ids it actually released, so the events name the tasks that
260
+ // were reclaimed rather than assuming they were the first N candidates.
261
+ const released = await this.callProc(provider, 'spTaskGraphReleaseExpiredClaims', [
262
+ { Name: 'TaskIDs', Value: JSON.stringify(rows.map((r) => r.ID)) },
263
+ ], contextUser);
264
+ if (released === null)
265
+ return [];
266
+ const byID = new Map(rows.map((r) => [r.ID.toLowerCase(), r]));
267
+ const events = released.map((r) => {
268
+ const candidate = byID.get(String(r.ID).toLowerCase());
269
+ return {
270
+ TaskID: String(r.ID),
271
+ Action: 'ExpiredClaimReleased',
272
+ Detail: `Claim held by '${candidate?.ClaimedBy ?? 'unknown'}' expired; task '${candidate?.Name ?? r.ID}' returned to Pending.`,
273
+ };
274
+ });
277
275
  for (const e of events) {
278
276
  LogStatus(`[TaskGraph reconciliation] ${e.Action}: ${e.Detail}`);
279
277
  }
@@ -288,13 +286,22 @@ export class TaskClaimStore {
288
286
  * excluded because for them this shape is legitimate, not anomalous.
289
287
  */
290
288
  async FindOrphanedInProgress(provider, contextUser) {
291
- const db = this.sql(provider);
292
- const rows = await db.ExecuteSQL(`SELECT ${db.QuoteIdentifier('ID')}, ${db.QuoteIdentifier('Name')}
293
- FROM ${this.taskTable(provider)}
294
- WHERE ${db.QuoteIdentifier('Status')} = 'In Progress'
295
- AND ${MachineTaskSQL(db.QuoteIdentifier.bind(db))}
296
- AND ${db.QuoteIdentifier('ClaimedBy')} IS NULL`, undefined, undefined, contextUser);
297
- const events = (rows ?? []).map((r) => ({
289
+ // From the view, for the same reason as the sweep above (#4575).
290
+ const result = await RunView.FromMetadataProvider(provider).RunView({
291
+ EntityName: 'MJ: Tasks',
292
+ ExtraFilter: `Status='In Progress' AND ${MachineTaskSQL()} AND ClaimedBy IS NULL`,
293
+ Fields: ['ID', 'Name'],
294
+ ResultType: 'simple',
295
+ // Unbounded for the same reason as the sweep above: this is a reconciliation
296
+ // report, where a quietly truncated answer is worse than a slow one.
297
+ IgnoreMaxRows: true,
298
+ BypassCache: true,
299
+ }, contextUser);
300
+ if (!result.Success) {
301
+ LogError(`[TaskGraph reconciliation] could not read orphaned In Progress tasks: ${result.ErrorMessage}`);
302
+ return [];
303
+ }
304
+ const events = (result.Results ?? []).map((r) => ({
298
305
  TaskID: r.ID,
299
306
  Action: 'OrphanedInProgressReleased',
300
307
  Detail: `Agent task '${r.Name}' is In Progress with no claim — no dispatcher owns it.`,
@@ -324,15 +331,11 @@ export class TaskClaimStore {
324
331
  * (someone else settled it) or the write failed
325
332
  */
326
333
  async TrySettleParent(provider, parentTaskID, status, percentComplete, contextUser) {
327
- const db = this.sql(provider);
328
- const sql = `
329
- UPDATE ${this.taskTable(provider)}
330
- SET ${db.QuoteIdentifier('Status')} = '${this.escape(status)}',
331
- ${db.QuoteIdentifier('PercentComplete')} = ${Number.isFinite(percentComplete) ? Math.round(percentComplete) : 0},
332
- ${db.QuoteIdentifier('CompletedAt')} = SYSUTCDATETIME()
333
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
334
- AND ${db.QuoteIdentifier('Status')} NOT IN (${TERMINAL_PARENT_STATUS_SQL})`;
335
- return (await this.affectedRows(db, sql, contextUser)) === 1;
334
+ return this.guardedWrite(provider, 'spTaskGraphSettleParent', [
335
+ { Name: 'ParentTaskID', Value: parentTaskID },
336
+ { Name: 'Status', Value: status },
337
+ { Name: 'PercentComplete', Value: Number.isFinite(percentComplete) ? Math.round(percentComplete) : 0 },
338
+ ], contextUser);
336
339
  }
337
340
  /**
338
341
  * Updates a graph parent's in-flight progress — column-scoped, and refused once it is terminal.
@@ -349,14 +352,11 @@ export class TaskClaimStore {
349
352
  * is exactly the kind of timing argument a guard replaces with a structural one.
350
353
  */
351
354
  async TryUpdateParentProgress(provider, parentTaskID, status, percentComplete, contextUser) {
352
- const db = this.sql(provider);
353
- const sql = `
354
- UPDATE ${this.taskTable(provider)}
355
- SET ${db.QuoteIdentifier('Status')} = '${this.escape(status)}',
356
- ${db.QuoteIdentifier('PercentComplete')} = ${Number.isFinite(percentComplete) ? Math.round(percentComplete) : 0}
357
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
358
- AND ${db.QuoteIdentifier('Status')} NOT IN (${TERMINAL_PARENT_STATUS_SQL})`;
359
- return (await this.affectedRows(db, sql, contextUser)) === 1;
355
+ return this.guardedWrite(provider, 'spTaskGraphUpdateParentProgress', [
356
+ { Name: 'ParentTaskID', Value: parentTaskID },
357
+ { Name: 'Status', Value: status },
358
+ { Name: 'PercentComplete', Value: Number.isFinite(percentComplete) ? Math.round(percentComplete) : 0 },
359
+ ], contextUser);
360
360
  }
361
361
  /**
362
362
  * Stamps a graph parent's start time, once, without touching anything else.
@@ -367,13 +367,10 @@ export class TaskClaimStore {
367
367
  * naturally once-only and safe to call on every pass.
368
368
  */
369
369
  async TryStampParentStart(provider, parentTaskID, startedAt, contextUser) {
370
- const db = this.sql(provider);
371
- const sql = `
372
- UPDATE ${this.taskTable(provider)}
373
- SET ${db.QuoteIdentifier('StartedAt')} = '${startedAt.toISOString()}'
374
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
375
- AND ${db.QuoteIdentifier('StartedAt')} IS NULL`;
376
- return (await this.affectedRows(db, sql, contextUser)) === 1;
370
+ return this.guardedWrite(provider, 'spTaskGraphStampParentStart', [
371
+ { Name: 'ParentTaskID', Value: parentTaskID },
372
+ { Name: 'StartedAt', Value: startedAt },
373
+ ], contextUser);
377
374
  }
378
375
  /**
379
376
  * Claims the right to deliver a graph's continuation — exactly once, across every instance.
@@ -406,19 +403,12 @@ export class TaskClaimStore {
406
403
  * @returns true when this instance won the right to deliver
407
404
  */
408
405
  async TryClaimContinuation(provider, parentTaskID, deliveredAs, workflowTaskTypeID, contextUser) {
409
- const db = this.sql(provider);
410
- const nowIso = new Date().toISOString();
411
- const payload = db.QuoteIdentifier('InputPayload');
412
- const sql = `
413
- UPDATE ${this.taskTable(provider)}
414
- SET ${payload} = JSON_MODIFY(
415
- JSON_MODIFY(${payload}, '$.continuationDeliveredAt', '${this.escape(nowIso)}'),
416
- '$.continuationDeliveredAs', '${this.escape(deliveredAs)}')
417
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
418
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
419
- AND ISJSON(${payload}) = 1
420
- AND JSON_VALUE(${payload}, '$.continuationDeliveredAt') IS NULL`;
421
- return (await this.affectedRows(db, sql, contextUser)) === 1;
406
+ return this.guardedWrite(provider, 'spTaskGraphClaimContinuation', [
407
+ { Name: 'ParentTaskID', Value: parentTaskID },
408
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
409
+ { Name: 'DeliveredAs', Value: deliveredAs },
410
+ { Name: 'DeliveredAt', Value: new Date().toISOString() },
411
+ ], contextUser);
422
412
  }
423
413
  /**
424
414
  * Skips one task, refusing if anything has taken it since the caller looked.
@@ -448,14 +438,10 @@ export class TaskClaimStore {
448
438
  * @returns true when this call is the one that skipped it; false means something else got there
449
439
  */
450
440
  async TrySkipPending(provider, taskID, workflowTaskTypeID, contextUser) {
451
- const db = this.sql(provider);
452
- const sql = `
453
- UPDATE ${this.taskTable(provider)}
454
- SET ${db.QuoteIdentifier('Status')} = 'Skipped'
455
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
456
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
457
- AND ${db.QuoteIdentifier('Status')} = 'Pending'`;
458
- return (await this.affectedRows(db, sql, contextUser)) === 1;
441
+ return this.guardedWrite(provider, 'spTaskGraphSkipPending', [
442
+ { Name: 'TaskID', Value: taskID },
443
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
444
+ ], contextUser);
459
445
  }
460
446
  /**
461
447
  * Stamps the human-notified marker, once, without touching anything else.
@@ -467,14 +453,10 @@ export class TaskClaimStore {
467
453
  * naturally once-only and the rowcount says which instance did it.
468
454
  */
469
455
  async TryMarkHumanNotified(provider, taskID, marker, contextUser) {
470
- const db = this.sql(provider);
471
- const sql = `
472
- UPDATE ${this.taskTable(provider)}
473
- SET ${db.QuoteIdentifier('ClaimedBy')} = '${this.escape(marker)}'
474
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
475
- AND ${db.QuoteIdentifier('Status')} = 'Pending'
476
- AND ${db.QuoteIdentifier('ClaimedBy')} IS NULL`;
477
- return (await this.affectedRows(db, sql, contextUser)) === 1;
456
+ return this.guardedWrite(provider, 'spTaskGraphMarkHumanNotified', [
457
+ { Name: 'TaskID', Value: taskID },
458
+ { Name: 'Marker', Value: marker },
459
+ ], contextUser);
478
460
  }
479
461
  /**
480
462
  * Cancels one task, refusing if it settled while the caller was looking elsewhere.
@@ -495,13 +477,9 @@ export class TaskClaimStore {
495
477
  * @returns true when this call cancelled it; false means it had already settled
496
478
  */
497
479
  async TryCancelTask(provider, taskID, contextUser) {
498
- const db = this.sql(provider);
499
- const sql = `
500
- UPDATE ${this.taskTable(provider)}
501
- SET ${db.QuoteIdentifier('Status')} = 'Cancelled'
502
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
503
- AND ${db.QuoteIdentifier('Status')} NOT IN (${TERMINAL_PARENT_STATUS_SQL})`;
504
- return (await this.affectedRows(db, sql, contextUser)) === 1;
480
+ return this.guardedWrite(provider, 'spTaskGraphCancelTask', [
481
+ { Name: 'TaskID', Value: taskID },
482
+ ], contextUser);
505
483
  }
506
484
  /**
507
485
  * Records, durably and once, that a graph is finishing early.
@@ -521,17 +499,11 @@ export class TaskClaimStore {
521
499
  * @returns true when this call is the one that declared it
522
500
  */
523
501
  async TryDeclareEarlyFinish(provider, parentTaskID, workflowTaskTypeID, contextUser) {
524
- const db = this.sql(provider);
525
- const payload = db.QuoteIdentifier('InputPayload');
526
- const nowIso = new Date().toISOString();
527
- const sql = `
528
- UPDATE ${this.taskTable(provider)}
529
- SET ${payload} = JSON_MODIFY(${payload}, '$.earlyFinishedAt', '${this.escape(nowIso)}')
530
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
531
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
532
- AND ISJSON(${payload}) = 1
533
- AND JSON_VALUE(${payload}, '$.earlyFinishedAt') IS NULL`;
534
- return (await this.affectedRows(db, sql, contextUser)) === 1;
502
+ return this.guardedWrite(provider, 'spTaskGraphDeclareEarlyFinish', [
503
+ { Name: 'ParentTaskID', Value: parentTaskID },
504
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
505
+ { Name: 'FinishedAt', Value: new Date().toISOString() },
506
+ ], contextUser);
535
507
  }
536
508
  /**
537
509
  * Records why a graph ended early, writing that column and no other.
@@ -551,13 +523,11 @@ export class TaskClaimStore {
551
523
  * conversation task or somebody's to-do.
552
524
  */
553
525
  async TrySetParentOutput(provider, parentTaskID, outputPayload, workflowTaskTypeID, contextUser) {
554
- const db = this.sql(provider);
555
- const sql = `
556
- UPDATE ${this.taskTable(provider)}
557
- SET ${db.QuoteIdentifier('OutputPayload')} = '${this.escape(outputPayload)}'
558
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
559
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'`;
560
- return (await this.affectedRows(db, sql, contextUser)) === 1;
526
+ return this.guardedWrite(provider, 'spTaskGraphSetParentOutput', [
527
+ { Name: 'ParentTaskID', Value: parentTaskID },
528
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
529
+ { Name: 'OutputPayload', Value: outputPayload },
530
+ ], contextUser);
561
531
  }
562
532
  /**
563
533
  * Clears a graph's debug state entirely — the "stop debugging this run" write.
@@ -568,15 +538,10 @@ export class TaskClaimStore {
568
538
  * read time — most sharply resurrecting a step allowance the dispatcher consumed in between.
569
539
  */
570
540
  async TryClearDebugState(provider, parentTaskID, workflowTaskTypeID, contextUser) {
571
- const db = this.sql(provider);
572
- const payload = db.QuoteIdentifier('InputPayload');
573
- const sql = `
574
- UPDATE ${this.taskTable(provider)}
575
- SET ${payload} = JSON_MODIFY(${payload}, '$.debug', NULL)
576
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
577
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
578
- AND ISJSON(${payload}) = 1`;
579
- return (await this.affectedRows(db, sql, contextUser)) === 1;
541
+ return this.guardedWrite(provider, 'spTaskGraphClearDebugState', [
542
+ { Name: 'ParentTaskID', Value: parentTaskID },
543
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
544
+ ], contextUser);
580
545
  }
581
546
  /**
582
547
  * One field of the debug bag, as a value the statement can write.
@@ -604,60 +569,18 @@ export class TaskClaimStore {
604
569
  async TryWriteDebugFields(provider, parentTaskID, fields, workflowTaskTypeID, contextUser) {
605
570
  if (fields.length === 0)
606
571
  return true;
607
- const db = this.sql(provider);
608
- const payload = db.QuoteIdentifier('InputPayload');
609
- // Innermost first, so the outermost JSON_MODIFY sees every prior change — and every write
610
- // starts from a payload whose containing objects are known to exist (see ensureObjects).
611
- const expression = fields.reduce((inner, field) => `JSON_MODIFY(${inner}, '${this.escape(field.Path)}', ${this.renderDebugValue(field.Value)})`, this.ensureObjects(payload, fields.map((f) => f.Path)));
612
- const sql = `
613
- UPDATE ${this.taskTable(provider)}
614
- SET ${payload} = ${expression}
615
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
616
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
617
- AND ISJSON(${payload}) = 1`;
618
- return (await this.affectedRows(db, sql, contextUser)) === 1;
619
- }
620
- /**
621
- * Wraps a payload expression so every object CONTAINING one of these paths exists.
622
- *
623
- * `JSON_MODIFY` does not create intermediate objects: writing `$.debug.paused` into a payload
624
- * with no `debug` key, or `$.debug.edgeOverrides."<id>"` with no override map yet, silently
625
- * changes nothing — which for a control verb means the write reports success (rowcount 1, the
626
- * row WAS updated, just not the way anyone meant) and the workflow never pauses. A graph only
627
- * acquires a `debug` key the first time somebody debugs it, so this is the NORMAL first call,
628
- * not an edge case.
629
- *
630
- * This hazard arrived WITH field-scoped writes and is the price of them: the whole-bag write
631
- * they replaced targeted `$.debug`, one level down from a root that always exists, so it
632
- * created the containing object as a side effect of every verb. Field-scoping is still the
633
- * right trade — it removes the step-resurrection class outright — but it moves the
634
- * container's existence from implicit to something this method has to guarantee.
635
- *
636
- * Each containing object is created only when absent, shallowest first, so an existing bag is
637
- * never replaced.
638
- */
639
- ensureObjects(payload, paths) {
640
- const parents = new Set();
641
- for (const path of paths) {
642
- for (const prefix of ContainingPaths(path))
643
- parents.add(prefix);
644
- }
645
- // Shallowest first: `$.debug` must exist before `$.debug.edgeOverrides` can be added to it.
646
- const ordered = [...parents].sort((a, b) => a.length - b.length);
647
- return ordered.reduce((inner, parent) => `CASE WHEN JSON_QUERY(${inner}, '${this.escape(parent)}') IS NULL` +
648
- ` THEN JSON_MODIFY(${inner}, '${this.escape(parent)}', JSON_QUERY('{}'))` +
649
- ` ELSE ${inner} END`, payload);
650
- }
651
- /** Renders one debug value as a SQL literal `JSON_MODIFY` will store with the right JSON type. */
652
- renderDebugValue(value) {
653
- switch (value.Kind) {
654
- // `NULL` in lax mode DELETES the key, which is what "this verb cleared it" should mean.
655
- case 'null': return 'NULL';
656
- case 'bool': return `CAST(${value.Value ? 1 : 0} AS BIT)`;
657
- case 'string': return `'${this.escape(value.Value)}'`;
658
- // JSON_QUERY keeps objects and arrays as JSON rather than storing them as a string.
659
- case 'json': return `JSON_QUERY('${this.escape(value.Value)}')`;
660
- }
572
+ // Containing objects are computed here rather than in SQL: `ContainingPaths` is pure, tested,
573
+ // and the rule it encodes (JSON_MODIFY does not create intermediate objects) is the kind of
574
+ // database behaviour that is easy to assume wrongly.
575
+ const containers = [...new Set(fields.flatMap((f) => ContainingPaths(f.Path)))]
576
+ // Shallowest first: `$.debug` must exist before `$.debug.edgeOverrides` can be added to it.
577
+ .sort((a, b) => a.length - b.length);
578
+ return this.guardedWrite(provider, 'spTaskGraphWriteDebugFields', [
579
+ { Name: 'ParentTaskID', Value: parentTaskID },
580
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
581
+ { Name: 'Containers', Value: JSON.stringify(containers) },
582
+ { Name: 'Fields', Value: JSON.stringify(fields.map((f) => this.renderDebugField(f))) },
583
+ ], contextUser);
661
584
  }
662
585
  /**
663
586
  * Consumes a paused graph's one-shot step allowance — exactly once, across every instance.
@@ -668,16 +591,10 @@ export class TaskClaimStore {
668
591
  * never release two waves.
669
592
  */
670
593
  async TryConsumeStepMarker(provider, parentTaskID, workflowTaskTypeID, contextUser) {
671
- const db = this.sql(provider);
672
- const payload = db.QuoteIdentifier('InputPayload');
673
- const sql = `
674
- UPDATE ${this.taskTable(provider)}
675
- SET ${payload} = JSON_MODIFY(${payload}, '$.debug.step', NULL)
676
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
677
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
678
- AND ISJSON(${payload}) = 1
679
- AND JSON_VALUE(${payload}, '$.debug.step') IS NOT NULL`;
680
- return (await this.affectedRows(db, sql, contextUser)) === 1;
594
+ return this.guardedWrite(provider, 'spTaskGraphConsumeStepMarker', [
595
+ { Name: 'ParentTaskID', Value: parentTaskID },
596
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
597
+ ], contextUser);
681
598
  }
682
599
  /**
683
600
  * Pauses a graph because an eligible task hit a breakpoint — once, whichever instance sees it
@@ -694,21 +611,11 @@ export class TaskClaimStore {
694
611
  * way past, so a breakpoint could not exist without its container already being there.
695
612
  */
696
613
  async TryPauseAtBreakpoint(provider, parentTaskID, breakpointTaskID, workflowTaskTypeID, contextUser) {
697
- const db = this.sql(provider);
698
- const payload = db.QuoteIdentifier('InputPayload');
699
- const base = this.ensureObjects(payload, ['$.debug.paused']);
700
- const sql = `
701
- UPDATE ${this.taskTable(provider)}
702
- SET ${payload} = JSON_MODIFY(JSON_MODIFY(JSON_MODIFY(${base},
703
- '$.debug.paused', CAST(1 AS BIT)),
704
- '$.debug.pausedReason', 'breakpoint'),
705
- '$.debug.pausedAtTaskID', '${this.escape(breakpointTaskID)}')
706
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(parentTaskID)}'
707
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
708
- AND ISJSON(${payload}) = 1
709
- AND (JSON_VALUE(${payload}, '$.debug.paused') IS NULL
710
- OR JSON_VALUE(${payload}, '$.debug.paused') = 'false')`;
711
- return (await this.affectedRows(db, sql, contextUser)) === 1;
614
+ return this.guardedWrite(provider, 'spTaskGraphPauseAtBreakpoint', [
615
+ { Name: 'ParentTaskID', Value: parentTaskID },
616
+ { Name: 'BreakpointTaskID', Value: breakpointTaskID },
617
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
618
+ ], contextUser);
712
619
  }
713
620
  /**
714
621
  * Replaces a task's input, guarded on the status the caller believes it is in.
@@ -726,15 +633,12 @@ export class TaskClaimStore {
726
633
  * retry (`Failed`).
727
634
  */
728
635
  async TryUpdateInputPayload(provider, taskID, inputPayload, expectedStatus, workflowTaskTypeID, contextUser) {
729
- const db = this.sql(provider);
730
- const value = inputPayload == null ? 'NULL' : `'${this.escape(inputPayload)}'`;
731
- const sql = `
732
- UPDATE ${this.taskTable(provider)}
733
- SET ${db.QuoteIdentifier('InputPayload')} = ${value}
734
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
735
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
736
- AND ${db.QuoteIdentifier('Status')} = '${this.escape(expectedStatus)}'`;
737
- return (await this.affectedRows(db, sql, contextUser)) === 1;
636
+ return this.guardedWrite(provider, 'spTaskGraphUpdateInputPayload', [
637
+ { Name: 'TaskID', Value: taskID },
638
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
639
+ { Name: 'InputPayload', Value: inputPayload },
640
+ { Name: 'ExpectedStatus', Value: expectedStatus },
641
+ ], contextUser);
738
642
  }
739
643
  /**
740
644
  * Marks a task Complete with an operator-supplied output — the escape hatch for a wedged or
@@ -758,46 +662,79 @@ export class TaskClaimStore {
758
662
  * closed rather than relocated.)
759
663
  */
760
664
  async TryForceComplete(provider, taskID, outputPayload, workflowTaskTypeID, contextUser) {
665
+ return this.guardedWrite(provider, 'spTaskGraphForceComplete', [
666
+ { Name: 'TaskID', Value: taskID },
667
+ { Name: 'TaskTypeID', Value: workflowTaskTypeID },
668
+ { Name: 'OutputPayload', Value: outputPayload },
669
+ ], contextUser);
670
+ }
671
+ /**
672
+ * Runs one guarded procedure and returns whether THIS instance won.
673
+ *
674
+ * A false here means the guard did not match — someone else won the race, or the row had already
675
+ * moved on. It does NOT mean the write failed; a failure returns false too, but records itself
676
+ * (see {@link LastWriteFailed}) so the caller can tell the two apart.
677
+ */
678
+ async guardedWrite(provider, procName, params, contextUser) {
679
+ const rows = await this.callProc(provider, procName, params, contextUser);
680
+ if (rows === null)
681
+ return false;
682
+ return Number(rows[0]?.AffectedRows ?? 0) === 1;
683
+ }
684
+ /**
685
+ * Calls one of the task-graph procedures, returning its rows — or `null` when the call FAILED.
686
+ *
687
+ * **Why procedures and not raw SQL** (#4575). These statements used to be sent as direct DML
688
+ * against the `Task` and `AIAgentRun` base tables. MJ grants its runtime roles SELECT on views
689
+ * and EXECUTE on procedures and never table-level DML, so under a least-privilege login every
690
+ * write was refused — and, because a refusal was reported as rowcount 0, the dispatcher read it
691
+ * as a lost race and skipped every task forever. The guards themselves are unchanged; they moved
692
+ * into procedures the runtime roles can actually execute.
693
+ *
694
+ * **Why `null` and not 0.** A statement that never ran is not a lost race, and collapsing the two
695
+ * is what kept that defect invisible. Callers still get `false` from {@link guardedWrite} — the
696
+ * dispatch loop must not fault on one bad write — but the failure is recorded and logged as a
697
+ * failure, so an inert dispatcher can say so.
698
+ */
699
+ async callProc(provider, procName, params, contextUser) {
761
700
  const db = this.sql(provider);
762
- const output = outputPayload == null ? 'NULL' : `'${this.escape(outputPayload)}'`;
763
- const sql = `
764
- UPDATE ${this.taskTable(provider)}
765
- SET ${db.QuoteIdentifier('Status')} = 'Complete',
766
- ${db.QuoteIdentifier('OutputPayload')} = ${output},
767
- ${db.QuoteIdentifier('ErrorMessage')} = NULL,
768
- ${db.QuoteIdentifier('CompletedAt')} = SYSUTCDATETIME(),
769
- ${db.QuoteIdentifier('PercentComplete')} = 100,
770
- ${db.QuoteIdentifier('ClaimedBy')} = NULL,
771
- ${db.QuoteIdentifier('ClaimExpiresAt')} = NULL
772
- WHERE ${db.QuoteIdentifier('ID')} = '${this.escape(taskID)}'
773
- AND ${db.QuoteIdentifier('TypeID')} = '${this.escape(workflowTaskTypeID)}'
774
- AND (${db.QuoteIdentifier('Status')} IN ('Pending','Failed','Blocked')
775
- OR (${db.QuoteIdentifier('Status')} = 'In Progress'
776
- AND (${db.QuoteIdentifier('ClaimExpiresAt')} IS NULL
777
- OR ${db.QuoteIdentifier('ClaimExpiresAt')} < SYSUTCDATETIME())))`;
778
- return (await this.affectedRows(db, sql, contextUser)) === 1;
779
- }
780
- /** Runs the affected-rows statement, returning 0 on error rather than throwing into the loop. */
781
- async affectedRows(db, sql, contextUser) {
701
+ // Named arguments on SQL Server, positional on PostgreSQL — the call wrapper itself belongs
702
+ // to the dialect, so neither form is spelled out here.
703
+ const placeholders = params.map((param, i) => db.PlatformKey === 'postgresql'
704
+ ? db.BuildParameterPlaceholder(i)
705
+ : `@${param.Name}=${db.BuildParameterPlaceholder(i)}`);
706
+ const call = db.Dialect.ProcedureCallSyntax(db.MJCoreSchemaName, procName, placeholders);
782
707
  try {
783
- // The count comes back as data rather than through a driver-specific rowsAffected
784
- // field. The wrapper is dialect-owned because `@@ROWCOUNT` is T-SQL only: emitted on
785
- // PostgreSQL it left a bare `ROWCOUNT` identifier, which folds to lowercase, so every
786
- // guarded write failed with `column "rowcount" does not exist`.
787
- const rows = await db.ExecuteSQL(db.Dialect.AffectedRowCountSQL(sql, 'AffectedRows'), undefined, undefined, contextUser);
788
- return Number(rows?.[0]?.AffectedRows ?? 0);
708
+ const rows = await db.ExecuteSQL(call, params.map((param) => param.Value), { isMutation: true, description: `TaskGraph ${procName}` }, contextUser);
709
+ this._lastWriteError = null;
710
+ this._consecutiveWriteFailures = 0;
711
+ return rows ?? [];
789
712
  }
790
713
  catch (e) {
791
- LogError(`[TaskGraph] guarded write failed: ${e instanceof Error ? e.message : String(e)}`);
792
- return 0;
714
+ const message = e instanceof Error ? e.message : String(e);
715
+ this._lastWriteError = `${procName}: ${message}`;
716
+ this._consecutiveWriteFailures++;
717
+ LogError(`[TaskGraph] guarded write ${procName} FAILED — the statement never ran, so this is NOT ` +
718
+ `a lost race (${this._consecutiveWriteFailures} consecutive). ${message}`);
719
+ return null;
793
720
  }
794
721
  }
795
- literalOrNull(value) {
796
- return value == null ? 'NULL' : `'${this.escape(value)}'`;
797
- }
798
- /** Single-quote escaping. Inputs here are UUIDs and JSON the dispatcher itself produced. */
799
- escape(value) {
800
- return value.replace(/'/g, "''");
722
+ /**
723
+ * Renders one debug-bag field for the procedure's `@Fields` argument.
724
+ *
725
+ * The `Kind` travels with the value because it decides the JSON type written: a boolean stored as
726
+ * the string `"true"` reads back as truthy-but-wrong, and an object stored as a string reads back
727
+ * as a string.
728
+ */
729
+ renderDebugField(field) {
730
+ const value = field.Value;
731
+ switch (value.Kind) {
732
+ case 'null': return { Path: field.Path, Kind: 'null', Value: null };
733
+ case 'bool': return { Path: field.Path, Kind: 'bool', Value: value.Value };
734
+ case 'string': return { Path: field.Path, Kind: 'string', Value: value.Value };
735
+ // Embedded as real JSON, not as a string, so the procedure's JSON_QUERY sees an object.
736
+ case 'json': return { Path: field.Path, Kind: 'json', Value: JSON.parse(value.Value) };
737
+ }
801
738
  }
802
739
  }
803
740
  //# sourceMappingURL=TaskClaimStore.js.map