@brandboostinggmbh/observable-workflows 0.19.0-beta.3 → 0.20.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/index.d.ts +47 -1
- package/dist/index.js +166 -91
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -358,6 +358,48 @@ The library provides comprehensive error handling:
|
|
|
358
358
|
- **Workflow Failures**: Marked with appropriate status and error information
|
|
359
359
|
- **Retry Logic**: Failed workflows can be retried with step result reuse
|
|
360
360
|
- **Validation**: Input validation and type checking
|
|
361
|
+
- **Automatic D1 Retry**: Built-in retry logic with exponential backoff for transient D1 network errors
|
|
362
|
+
|
|
363
|
+
### D1 Network Error Retry
|
|
364
|
+
|
|
365
|
+
The library automatically retries D1 database operations that fail due to transient network errors (e.g., "D1_ERROR: Network connection lost"). This makes workflows more resilient to temporary network issues.
|
|
366
|
+
|
|
367
|
+
**Default Retry Configuration:**
|
|
368
|
+
- Maximum retry attempts: 3
|
|
369
|
+
- Initial delay: 100ms
|
|
370
|
+
- Maximum delay: 5000ms
|
|
371
|
+
- Exponential backoff multiplier: 2
|
|
372
|
+
- Jitter: Enabled
|
|
373
|
+
|
|
374
|
+
**Custom Retry Configuration:**
|
|
375
|
+
|
|
376
|
+
You can customize the retry behavior when creating the workflow context:
|
|
377
|
+
|
|
378
|
+
```typescript
|
|
379
|
+
const workflowContext = createWorkflowContext({
|
|
380
|
+
D1: env.D1,
|
|
381
|
+
retryConfig: {
|
|
382
|
+
maxAttempts: 5, // Retry up to 5 times
|
|
383
|
+
initialDelayMs: 200, // Start with 200ms delay
|
|
384
|
+
maxDelayMs: 10000, // Cap delay at 10 seconds
|
|
385
|
+
backoffMultiplier: 3, // Triple the delay each time
|
|
386
|
+
useJitter: true, // Add randomization to prevent thundering herd
|
|
387
|
+
},
|
|
388
|
+
})
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
**Manual Retry for Custom Operations:**
|
|
392
|
+
|
|
393
|
+
You can also use the retry utility directly for custom D1 operations:
|
|
394
|
+
|
|
395
|
+
```typescript
|
|
396
|
+
import { retryD1Operation } from '@brandboostinggmbh/observable-workflows'
|
|
397
|
+
|
|
398
|
+
const result = await retryD1Operation(
|
|
399
|
+
() => db.prepare("SELECT * FROM custom_table").first(),
|
|
400
|
+
{ maxAttempts: 3, initialDelayMs: 100 }
|
|
401
|
+
)
|
|
402
|
+
```
|
|
361
403
|
|
|
362
404
|
## Best Practices
|
|
363
405
|
|
package/dist/index.d.ts
CHANGED
|
@@ -18,6 +18,38 @@ declare function createStepContext(context: StepContextOptions): Promise<{
|
|
|
18
18
|
declare function ensureTables(db: D1Database): Promise<void>;
|
|
19
19
|
//#endregion
|
|
20
20
|
//#region src/observableWorkflows/helperFunctions.d.ts
|
|
21
|
+
/**
|
|
22
|
+
* Configuration for retry behavior on D1 operations
|
|
23
|
+
*/
|
|
24
|
+
interface RetryConfig$1 {
|
|
25
|
+
/** Maximum number of retry attempts (default: 3) */
|
|
26
|
+
maxAttempts?: number;
|
|
27
|
+
/** Initial delay in milliseconds (default: 100) */
|
|
28
|
+
initialDelayMs?: number;
|
|
29
|
+
/** Maximum delay in milliseconds (default: 5000) */
|
|
30
|
+
maxDelayMs?: number;
|
|
31
|
+
/** Multiplier for exponential backoff (default: 2) */
|
|
32
|
+
backoffMultiplier?: number;
|
|
33
|
+
/** Whether to add jitter to delays (default: true) */
|
|
34
|
+
useJitter?: boolean;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Retry a D1 operation with exponential backoff for transient network errors
|
|
38
|
+
*
|
|
39
|
+
* @param operation - The async operation to retry
|
|
40
|
+
* @param config - Retry configuration options
|
|
41
|
+
* @returns The result of the operation
|
|
42
|
+
* @throws The last error if all retry attempts fail
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```typescript
|
|
46
|
+
* const result = await retryD1Operation(
|
|
47
|
+
* () => db.prepare("SELECT * FROM table").first(),
|
|
48
|
+
* { maxAttempts: 5, initialDelayMs: 200 }
|
|
49
|
+
* )
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
declare function retryD1Operation<T>(operation: () => Promise<T>, config?: RetryConfig$1): Promise<T>;
|
|
21
53
|
declare function finalizeWorkflowRecord(options: InternalWorkflowContextOptions, {
|
|
22
54
|
workflowStatus,
|
|
23
55
|
endTime,
|
|
@@ -605,11 +637,25 @@ type ExternalBlobStorage = {
|
|
|
605
637
|
get: (id: string) => Promise<string>;
|
|
606
638
|
delete: (...keys: string[]) => Promise<number>;
|
|
607
639
|
};
|
|
640
|
+
type RetryConfig = {
|
|
641
|
+
/** Maximum number of retry attempts (default: 3) */
|
|
642
|
+
maxAttempts?: number;
|
|
643
|
+
/** Initial delay in milliseconds (default: 100) */
|
|
644
|
+
initialDelayMs?: number;
|
|
645
|
+
/** Maximum delay in milliseconds (default: 5000) */
|
|
646
|
+
maxDelayMs?: number;
|
|
647
|
+
/** Multiplier for exponential backoff (default: 2) */
|
|
648
|
+
backoffMultiplier?: number;
|
|
649
|
+
/** Whether to add jitter to delays (default: true) */
|
|
650
|
+
useJitter?: boolean;
|
|
651
|
+
};
|
|
608
652
|
type WorkflowContextOptions = {
|
|
609
653
|
D1: D1Database;
|
|
610
654
|
idFactory?: () => string;
|
|
611
655
|
serializer?: Serializer;
|
|
612
656
|
externalBlobStorage?: ExternalBlobStorage;
|
|
657
|
+
/** Optional retry configuration for D1 operations (defaults: maxAttempts: 3, initialDelayMs: 100, maxDelayMs: 5000, backoffMultiplier: 2, useJitter: true) */
|
|
658
|
+
retryConfig?: RetryConfig;
|
|
613
659
|
};
|
|
614
660
|
type InternalWorkflowContextOptions = WorkflowContextOptions & Required<Pick<WorkflowContextOptions, 'serializer' | 'idFactory'>>;
|
|
615
661
|
type WorkflowContextInstance = {
|
|
@@ -906,4 +952,4 @@ type R2ExternalBlobStorageOptions = {
|
|
|
906
952
|
*/
|
|
907
953
|
declare function createR2ExternalBlobStorage(options: R2ExternalBlobStorageOptions): ExternalBlobStorage;
|
|
908
954
|
//#endregion
|
|
909
|
-
export { ConsoleWrapper, DateRangeFilter, ExternalBlobStorage, HandleWorkflowQueueMessageParams, InternalWorkflowContextOptions, Log, PossibleValueTypeNames, PossibleValueTypes, PropertyFilter, QueueWorkflowContextOptions, R2ExternalBlobStorageOptions, Serializer, Step, StepContextOptions, StepCtx, StepWorkflowStatus, StringFilter, ValueTypeMap, WorkflowContext, WorkflowContextInstance, WorkflowContextOptions, WorkflowEnqueueBatchItem, WorkflowFilter, WorkflowFunction, WorkflowProperty, WorkflowPropertyDefinition, WorkflowQueueMessage, WorkflowRun, WorkflowStatus, createCleanupManager, createLogAccessor, createQueueWorkflowContext, createR2ExternalBlobStorage, createStepContext, createWorkflowContext, createWorkflowDependencies, createWorkflowDependency, defaultIdFactory, defaultSerializer, defineWorkflow, deleteWorkflowDependency, deserializeWithExternalStorage, ensureTables, finalizeWorkflowRecord, insertStepRecordFull, insertWorkflowRecord, pushLogToDB, serializeWithExternalStorage, tryDeserializeObj, updateWorkflow, updateWorkflowName, upsertWorkflowProperty, workflowTableRowToWorkflowRun };
|
|
955
|
+
export { ConsoleWrapper, DateRangeFilter, ExternalBlobStorage, HandleWorkflowQueueMessageParams, InternalWorkflowContextOptions, Log, PossibleValueTypeNames, PossibleValueTypes, PropertyFilter, QueueWorkflowContextOptions, R2ExternalBlobStorageOptions, RetryConfig, Serializer, Step, StepContextOptions, StepCtx, StepWorkflowStatus, StringFilter, ValueTypeMap, WorkflowContext, WorkflowContextInstance, WorkflowContextOptions, WorkflowEnqueueBatchItem, WorkflowFilter, WorkflowFunction, WorkflowProperty, WorkflowPropertyDefinition, WorkflowQueueMessage, WorkflowRun, WorkflowStatus, createCleanupManager, createLogAccessor, createQueueWorkflowContext, createR2ExternalBlobStorage, createStepContext, createWorkflowContext, createWorkflowDependencies, createWorkflowDependency, defaultIdFactory, defaultSerializer, defineWorkflow, deleteWorkflowDependency, deserializeWithExternalStorage, ensureTables, finalizeWorkflowRecord, insertStepRecordFull, insertWorkflowRecord, pushLogToDB, retryD1Operation, serializeWithExternalStorage, tryDeserializeObj, updateWorkflow, updateWorkflowName, upsertWorkflowProperty, workflowTableRowToWorkflowRun };
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Detect the current schema version for each table
|
|
4
4
|
*/
|
|
5
5
|
async function detectSchemaVersion(db) {
|
|
6
|
-
const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
|
|
6
|
+
const workflowTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first());
|
|
7
7
|
let workflowTable = "missing";
|
|
8
8
|
if (workflowTableInfo) {
|
|
9
9
|
const hasInputRef = workflowTableInfo.sql.includes("inputRef");
|
|
@@ -16,7 +16,7 @@ async function detectSchemaVersion(db) {
|
|
|
16
16
|
else if (!hasInputRef && inputHasNotNull) workflowTable = "v1";
|
|
17
17
|
else workflowTable = "v1";
|
|
18
18
|
}
|
|
19
|
-
const stepTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='StepTable'`).first();
|
|
19
|
+
const stepTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='StepTable'`).first());
|
|
20
20
|
let stepTable = "missing";
|
|
21
21
|
if (stepTableInfo) {
|
|
22
22
|
const hasResultRef = stepTableInfo.sql.includes("resultRef");
|
|
@@ -26,7 +26,7 @@ async function detectSchemaVersion(db) {
|
|
|
26
26
|
else if (hasResultRef && hasErrorRef) stepTable = "v2";
|
|
27
27
|
else stepTable = "v1";
|
|
28
28
|
}
|
|
29
|
-
const logTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='LogTable'`).first();
|
|
29
|
+
const logTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='LogTable'`).first());
|
|
30
30
|
let logTable = "missing";
|
|
31
31
|
if (logTableInfo) {
|
|
32
32
|
const hasStepTableCascadeFK = logTableInfo.sql.includes("FOREIGN KEY (instanceId, stepName) REFERENCES StepTable(instanceId, stepName) ON DELETE CASCADE");
|
|
@@ -34,7 +34,7 @@ async function detectSchemaVersion(db) {
|
|
|
34
34
|
if (hasStepTableCascadeFK && hasWorkflowTableFK) logTable = "v5";
|
|
35
35
|
else logTable = "v1";
|
|
36
36
|
}
|
|
37
|
-
const workflowPropertiesInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowProperties'`).first();
|
|
37
|
+
const workflowPropertiesInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowProperties'`).first());
|
|
38
38
|
let workflowProperties = "missing";
|
|
39
39
|
if (workflowPropertiesInfo) workflowProperties = "v1";
|
|
40
40
|
const workflowDependenciesInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowDependencies'`).first();
|
|
@@ -53,10 +53,10 @@ async function detectSchemaVersion(db) {
|
|
|
53
53
|
* Adds inputRef column and makes input nullable
|
|
54
54
|
*/
|
|
55
55
|
async function migrateWorkflowTableV1ToV2(db) {
|
|
56
|
-
const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
|
|
56
|
+
const workflowTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first());
|
|
57
57
|
const hasInputRef = workflowTableInfo.sql.includes("inputRef");
|
|
58
58
|
const inputHasNotNull = workflowTableInfo.sql.includes("input TEXT NOT NULL");
|
|
59
|
-
if (!hasInputRef || inputHasNotNull) await db.batch([
|
|
59
|
+
if (!hasInputRef || inputHasNotNull) await retryD1Operation(() => db.batch([
|
|
60
60
|
db.prepare(`CREATE TABLE WorkflowTable_new (
|
|
61
61
|
instanceId TEXT NOT NULL,
|
|
62
62
|
workflowType TEXT NOT NULL,
|
|
@@ -78,16 +78,16 @@ async function migrateWorkflowTableV1ToV2(db) {
|
|
|
78
78
|
FROM WorkflowTable`),
|
|
79
79
|
db.prepare(`DROP TABLE WorkflowTable`),
|
|
80
80
|
db.prepare(`ALTER TABLE WorkflowTable_new RENAME TO WorkflowTable`)
|
|
81
|
-
]);
|
|
81
|
+
]));
|
|
82
82
|
}
|
|
83
83
|
/**
|
|
84
84
|
* Migrate WorkflowTable from V2/V3 to V4 schema
|
|
85
85
|
* Adds triggerId column with UNIQUE constraint
|
|
86
86
|
*/
|
|
87
87
|
async function migrateWorkflowTableV2V3ToV4(db) {
|
|
88
|
-
const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
|
|
88
|
+
const workflowTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first());
|
|
89
89
|
const hasTriggerId = workflowTableInfo.sql.includes("triggerId");
|
|
90
|
-
if (!hasTriggerId) await db.batch([
|
|
90
|
+
if (!hasTriggerId) await retryD1Operation(() => db.batch([
|
|
91
91
|
db.prepare(`CREATE TABLE WorkflowTable_new (
|
|
92
92
|
instanceId TEXT NOT NULL,
|
|
93
93
|
workflowType TEXT NOT NULL,
|
|
@@ -111,7 +111,7 @@ async function migrateWorkflowTableV2V3ToV4(db) {
|
|
|
111
111
|
FROM WorkflowTable`),
|
|
112
112
|
db.prepare(`DROP TABLE WorkflowTable`),
|
|
113
113
|
db.prepare(`ALTER TABLE WorkflowTable_new RENAME TO WorkflowTable`)
|
|
114
|
-
]);
|
|
114
|
+
]));
|
|
115
115
|
}
|
|
116
116
|
/**
|
|
117
117
|
* Migrate WorkflowTable from V4 to V6 schema
|
|
@@ -127,7 +127,7 @@ async function migrateWorkflowTableV4ToV6(db) {
|
|
|
127
127
|
*/
|
|
128
128
|
async function migrateWorkflowTable(db, currentVersion) {
|
|
129
129
|
if (currentVersion === "missing") {
|
|
130
|
-
await db.prepare(`CREATE TABLE WorkflowTable (
|
|
130
|
+
await retryD1Operation(() => db.prepare(`CREATE TABLE WorkflowTable (
|
|
131
131
|
instanceId TEXT NOT NULL,
|
|
132
132
|
workflowType TEXT NOT NULL,
|
|
133
133
|
workflowName TEXT NOT NULL,
|
|
@@ -144,7 +144,7 @@ async function migrateWorkflowTable(db, currentVersion) {
|
|
|
144
144
|
triggerId TEXT,
|
|
145
145
|
PRIMARY KEY (instanceId),
|
|
146
146
|
UNIQUE (triggerId)
|
|
147
|
-
)`).run();
|
|
147
|
+
)`).run());
|
|
148
148
|
return;
|
|
149
149
|
}
|
|
150
150
|
if (currentVersion === "v1") {
|
|
@@ -168,7 +168,7 @@ async function migrateWorkflowTable(db, currentVersion) {
|
|
|
168
168
|
*/
|
|
169
169
|
async function migrateStepTable(db, currentVersion) {
|
|
170
170
|
if (currentVersion === "missing") {
|
|
171
|
-
await db.prepare(`CREATE TABLE StepTable (
|
|
171
|
+
await retryD1Operation(() => db.prepare(`CREATE TABLE StepTable (
|
|
172
172
|
instanceId TEXT NOT NULL,
|
|
173
173
|
stepName TEXT NOT NULL,
|
|
174
174
|
stepStatus TEXT NOT NULL,
|
|
@@ -182,15 +182,15 @@ async function migrateStepTable(db, currentVersion) {
|
|
|
182
182
|
tenantId TEXT NOT NULL,
|
|
183
183
|
PRIMARY KEY (instanceId, stepName),
|
|
184
184
|
FOREIGN KEY (instanceId) REFERENCES WorkflowTable(instanceId) ON DELETE CASCADE
|
|
185
|
-
)`).run();
|
|
185
|
+
)`).run());
|
|
186
186
|
return;
|
|
187
187
|
}
|
|
188
188
|
if (currentVersion === "v1") {
|
|
189
|
-
const stepTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='StepTable'`).first();
|
|
189
|
+
const stepTableInfo = await retryD1Operation(() => db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='StepTable'`).first());
|
|
190
190
|
const hasResultRef = stepTableInfo.sql.includes("resultRef");
|
|
191
191
|
const hasErrorRef = stepTableInfo.sql.includes("errorRef");
|
|
192
|
-
if (!hasResultRef) await db.prepare(`ALTER TABLE StepTable ADD COLUMN resultRef TEXT`).run();
|
|
193
|
-
if (!hasErrorRef) await db.prepare(`ALTER TABLE StepTable ADD COLUMN errorRef TEXT`).run();
|
|
192
|
+
if (!hasResultRef) await retryD1Operation(() => db.prepare(`ALTER TABLE StepTable ADD COLUMN resultRef TEXT`).run());
|
|
193
|
+
if (!hasErrorRef) await retryD1Operation(() => db.prepare(`ALTER TABLE StepTable ADD COLUMN errorRef TEXT`).run());
|
|
194
194
|
}
|
|
195
195
|
}
|
|
196
196
|
/**
|
|
@@ -198,7 +198,7 @@ async function migrateStepTable(db, currentVersion) {
|
|
|
198
198
|
*/
|
|
199
199
|
async function migrateLogTable(db, currentVersion) {
|
|
200
200
|
if (currentVersion === "missing") {
|
|
201
|
-
await db.prepare(`CREATE TABLE LogTable (
|
|
201
|
+
await retryD1Operation(() => db.prepare(`CREATE TABLE LogTable (
|
|
202
202
|
instanceId TEXT NOT NULL,
|
|
203
203
|
stepName TEXT,
|
|
204
204
|
log TEXT NOT NULL,
|
|
@@ -208,7 +208,7 @@ async function migrateLogTable(db, currentVersion) {
|
|
|
208
208
|
tenantId TEXT NOT NULL,
|
|
209
209
|
FOREIGN KEY (instanceId, stepName) REFERENCES StepTable(instanceId, stepName) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
|
|
210
210
|
FOREIGN KEY (instanceId) REFERENCES WorkflowTable(instanceId) ON DELETE CASCADE
|
|
211
|
-
)`).run();
|
|
211
|
+
)`).run());
|
|
212
212
|
return;
|
|
213
213
|
}
|
|
214
214
|
}
|
|
@@ -217,7 +217,7 @@ async function migrateLogTable(db, currentVersion) {
|
|
|
217
217
|
*/
|
|
218
218
|
async function migrateWorkflowPropertiesTable(db, currentVersion) {
|
|
219
219
|
if (currentVersion === "missing") {
|
|
220
|
-
await db.prepare(`CREATE TABLE WorkflowProperties (
|
|
220
|
+
await retryD1Operation(() => db.prepare(`CREATE TABLE WorkflowProperties (
|
|
221
221
|
instanceId TEXT NOT NULL,
|
|
222
222
|
key TEXT NOT NULL,
|
|
223
223
|
value TEXT NOT NULL,
|
|
@@ -226,7 +226,7 @@ async function migrateWorkflowPropertiesTable(db, currentVersion) {
|
|
|
226
226
|
PRIMARY KEY (instanceId, key),
|
|
227
227
|
FOREIGN KEY (instanceId) REFERENCES WorkflowTable(instanceId)
|
|
228
228
|
ON DELETE CASCADE
|
|
229
|
-
)`).run();
|
|
229
|
+
)`).run());
|
|
230
230
|
return;
|
|
231
231
|
}
|
|
232
232
|
}
|
|
@@ -251,13 +251,17 @@ async function migrateWorkflowDependenciesTable(db, currentVersion) {
|
|
|
251
251
|
* Create necessary indexes
|
|
252
252
|
*/
|
|
253
253
|
async function createIndexes(db) {
|
|
254
|
-
const existingIndexes = await db.prepare(`SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'`).all();
|
|
254
|
+
const existingIndexes = await retryD1Operation(() => db.prepare(`SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'`).all());
|
|
255
255
|
const existingIndexNames = new Set(existingIndexes.results?.map((row) => row.name) || []);
|
|
256
256
|
const indexes = [
|
|
257
257
|
{
|
|
258
258
|
name: "idx_workflows_parent_instance_id",
|
|
259
259
|
sql: `CREATE INDEX idx_workflows_parent_instance_id ON WorkflowTable (parentInstanceId)`
|
|
260
260
|
},
|
|
261
|
+
{
|
|
262
|
+
name: "idx_workflows_parent_tenant",
|
|
263
|
+
sql: `CREATE INDEX idx_workflows_parent_tenant ON WorkflowTable (parentInstanceId, tenantId)`
|
|
264
|
+
},
|
|
261
265
|
{
|
|
262
266
|
name: "idx_workflows_trigger_id",
|
|
263
267
|
sql: `CREATE INDEX idx_workflows_trigger_id ON WorkflowTable (triggerId)`
|
|
@@ -337,7 +341,7 @@ async function createIndexes(db) {
|
|
|
337
341
|
];
|
|
338
342
|
const preparedStatements = [];
|
|
339
343
|
for (const index of indexes) if (!existingIndexNames.has(index.name)) preparedStatements.push(db.prepare(index.sql));
|
|
340
|
-
if (preparedStatements.length > 0) await db.batch(preparedStatements);
|
|
344
|
+
if (preparedStatements.length > 0) await retryD1Operation(() => db.batch(preparedStatements));
|
|
341
345
|
}
|
|
342
346
|
/**
|
|
343
347
|
* Main migration function that ensures all tables exist and are up-to-date.
|
|
@@ -346,7 +350,7 @@ async function createIndexes(db) {
|
|
|
346
350
|
* @param db - D1Database instance
|
|
347
351
|
*/
|
|
348
352
|
async function ensureTables(db) {
|
|
349
|
-
await db.prepare(`PRAGMA foreign_keys = ON`).run();
|
|
353
|
+
await retryD1Operation(() => db.prepare(`PRAGMA foreign_keys = ON`).run());
|
|
350
354
|
const schemaVersion = await detectSchemaVersion(db);
|
|
351
355
|
await migrateWorkflowTable(db, schemaVersion.workflowTable);
|
|
352
356
|
await migrateStepTable(db, schemaVersion.stepTable);
|
|
@@ -358,16 +362,85 @@ async function ensureTables(db) {
|
|
|
358
362
|
|
|
359
363
|
//#endregion
|
|
360
364
|
//#region src/observableWorkflows/helperFunctions.ts
|
|
365
|
+
/**
|
|
366
|
+
* Default retry configuration for D1 operations
|
|
367
|
+
*/
|
|
368
|
+
const DEFAULT_RETRY_CONFIG = {
|
|
369
|
+
maxAttempts: 3,
|
|
370
|
+
initialDelayMs: 100,
|
|
371
|
+
maxDelayMs: 5e3,
|
|
372
|
+
backoffMultiplier: 2,
|
|
373
|
+
useJitter: true
|
|
374
|
+
};
|
|
375
|
+
/**
|
|
376
|
+
* Check if an error is a transient D1 network error that should be retried
|
|
377
|
+
*/
|
|
378
|
+
function isRetriableD1Error(error) {
|
|
379
|
+
if (!error) return false;
|
|
380
|
+
const errorMessage = error.message || String(error);
|
|
381
|
+
const errorString = errorMessage.toLowerCase();
|
|
382
|
+
return errorString.includes("network connection lost") || errorString.includes("d1_error") || errorString.includes("connection reset") || errorString.includes("econnreset") || errorString.includes("timeout") || errorString.includes("etimedout");
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Calculate delay with exponential backoff and optional jitter
|
|
386
|
+
*/
|
|
387
|
+
function calculateDelay(attempt, config) {
|
|
388
|
+
const exponentialDelay = Math.min(config.initialDelayMs * Math.pow(config.backoffMultiplier, attempt), config.maxDelayMs);
|
|
389
|
+
if (config.useJitter) return Math.floor(Math.random() * exponentialDelay);
|
|
390
|
+
return exponentialDelay;
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Sleep for the specified number of milliseconds
|
|
394
|
+
*/
|
|
395
|
+
function sleep(ms) {
|
|
396
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* Retry a D1 operation with exponential backoff for transient network errors
|
|
400
|
+
*
|
|
401
|
+
* @param operation - The async operation to retry
|
|
402
|
+
* @param config - Retry configuration options
|
|
403
|
+
* @returns The result of the operation
|
|
404
|
+
* @throws The last error if all retry attempts fail
|
|
405
|
+
*
|
|
406
|
+
* @example
|
|
407
|
+
* ```typescript
|
|
408
|
+
* const result = await retryD1Operation(
|
|
409
|
+
* () => db.prepare("SELECT * FROM table").first(),
|
|
410
|
+
* { maxAttempts: 5, initialDelayMs: 200 }
|
|
411
|
+
* )
|
|
412
|
+
* ```
|
|
413
|
+
*/
|
|
414
|
+
async function retryD1Operation(operation, config = {}) {
|
|
415
|
+
const finalConfig = {
|
|
416
|
+
...DEFAULT_RETRY_CONFIG,
|
|
417
|
+
...config
|
|
418
|
+
};
|
|
419
|
+
let lastError;
|
|
420
|
+
for (let attempt = 0; attempt < finalConfig.maxAttempts; attempt++) try {
|
|
421
|
+
return await operation();
|
|
422
|
+
} catch (error) {
|
|
423
|
+
lastError = error;
|
|
424
|
+
if (!isRetriableD1Error(error)) throw error;
|
|
425
|
+
if (attempt < finalConfig.maxAttempts - 1) {
|
|
426
|
+
const delay = calculateDelay(attempt, finalConfig);
|
|
427
|
+
console.warn(`D1 operation failed with retriable error (attempt ${attempt + 1}/${finalConfig.maxAttempts}), retrying in ${delay}ms:`, error instanceof Error ? error.message : String(error));
|
|
428
|
+
await sleep(delay);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
console.error(`D1 operation failed after ${finalConfig.maxAttempts} attempts:`, lastError instanceof Error ? lastError.message : String(lastError));
|
|
432
|
+
throw lastError;
|
|
433
|
+
}
|
|
361
434
|
async function finalizeWorkflowRecord(options, { workflowStatus, endTime, instanceId, result }) {
|
|
362
435
|
if (result !== void 0) {
|
|
363
436
|
const { data: resultData, externalRef: resultRef } = await serializeWithExternalStorage(result, options.serializer, options.externalBlobStorage);
|
|
364
|
-
return options.D1.prepare(`UPDATE WorkflowTable
|
|
437
|
+
return retryD1Operation(() => options.D1.prepare(`UPDATE WorkflowTable
|
|
365
438
|
SET workflowStatus = ?, endTime = ?, result = ?, resultRef = ?
|
|
366
|
-
WHERE instanceId = ?`).bind(workflowStatus, endTime, resultData, resultRef, instanceId).run();
|
|
439
|
+
WHERE instanceId = ?`).bind(workflowStatus, endTime, resultData, resultRef, instanceId).run());
|
|
367
440
|
}
|
|
368
|
-
return options.D1.prepare(`UPDATE WorkflowTable
|
|
441
|
+
return retryD1Operation(() => options.D1.prepare(`UPDATE WorkflowTable
|
|
369
442
|
SET workflowStatus = ?, endTime = ?
|
|
370
|
-
WHERE instanceId = ?`).bind(workflowStatus, endTime, instanceId).run();
|
|
443
|
+
WHERE instanceId = ?`).bind(workflowStatus, endTime, instanceId).run());
|
|
371
444
|
}
|
|
372
445
|
/**
|
|
373
446
|
* Insert a new workflow record into the database.
|
|
@@ -398,7 +471,7 @@ async function insertWorkflowRecord(options, { instanceId, workflowType, workflo
|
|
|
398
471
|
(instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId, triggerId)
|
|
399
472
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null, triggerId ?? null);
|
|
400
473
|
if (!dependencies || dependencies.length === 0) {
|
|
401
|
-
const result = await insertWorkflowStatement.run();
|
|
474
|
+
const result = await retryD1Operation(() => insertWorkflowStatement.run());
|
|
402
475
|
return {
|
|
403
476
|
success: result.success,
|
|
404
477
|
meta: result.meta
|
|
@@ -412,7 +485,7 @@ async function insertWorkflowRecord(options, { instanceId, workflowType, workflo
|
|
|
412
485
|
tenantId,
|
|
413
486
|
createdAt
|
|
414
487
|
}));
|
|
415
|
-
const results = await options.D1.batch([insertWorkflowStatement, ...dependencyStatements]);
|
|
488
|
+
const results = await retryD1Operation(() => options.D1.batch([insertWorkflowStatement, ...dependencyStatements]));
|
|
416
489
|
const allSucceeded = results.every((result) => result.success);
|
|
417
490
|
return {
|
|
418
491
|
success: allSucceeded,
|
|
@@ -420,12 +493,12 @@ async function insertWorkflowRecord(options, { instanceId, workflowType, workflo
|
|
|
420
493
|
};
|
|
421
494
|
}
|
|
422
495
|
function insertStepRecordFull(context, { instanceId, name, status, metadata, startTime, endTime, result, error, resultRef, errorRef }) {
|
|
423
|
-
return context.D1.prepare(`INSERT INTO StepTable
|
|
496
|
+
return retryD1Operation(() => context.D1.prepare(`INSERT INTO StepTable
|
|
424
497
|
(instanceId, stepName, stepStatus, stepMetadata, startTime, endTime, result, error, resultRef, errorRef, tenantId)
|
|
425
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, name, status, metadata, startTime, endTime, result, error, resultRef ?? null, errorRef ?? null, context.tenantId).run();
|
|
498
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, name, status, metadata, startTime, endTime, result, error, resultRef ?? null, errorRef ?? null, context.tenantId).run());
|
|
426
499
|
}
|
|
427
500
|
async function getStepRecord(context, stepName, instanceId) {
|
|
428
|
-
const existingStep = await context.D1.prepare(`SELECT * FROM StepTable WHERE instanceId = ? AND stepName = ? AND tenantId = ?`).bind(instanceId, stepName, context.tenantId).first();
|
|
501
|
+
const existingStep = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM StepTable WHERE instanceId = ? AND stepName = ? AND tenantId = ?`).bind(instanceId, stepName, context.tenantId).first());
|
|
429
502
|
const row = existingStep;
|
|
430
503
|
if (row) {
|
|
431
504
|
const result = await deserializeWithExternalStorage(row.result, row.resultRef, context.serializer, context.externalBlobStorage);
|
|
@@ -499,9 +572,9 @@ function truncateLogMessage(message) {
|
|
|
499
572
|
}
|
|
500
573
|
function pushLogToDB(options, { instanceId, stepName, message, timestamp, type, logOrder, tenantId }) {
|
|
501
574
|
const truncatedMessage = truncateLogMessage(message);
|
|
502
|
-
return options.D1.prepare(`INSERT INTO LogTable
|
|
575
|
+
return retryD1Operation(() => options.D1.prepare(`INSERT INTO LogTable
|
|
503
576
|
(instanceId, stepName, log, timestamp, type, logOrder, tenantId)
|
|
504
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, stepName, truncatedMessage, timestamp, type, logOrder, tenantId).run();
|
|
577
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, stepName, truncatedMessage, timestamp, type, logOrder, tenantId).run());
|
|
505
578
|
}
|
|
506
579
|
var LogBatcher = class {
|
|
507
580
|
batch = [];
|
|
@@ -536,10 +609,12 @@ var LogBatcher = class {
|
|
|
536
609
|
if (this.batch.length === 0) return;
|
|
537
610
|
const logsToFlush = this.batch.splice(0);
|
|
538
611
|
try {
|
|
539
|
-
|
|
612
|
+
await retryD1Operation(async () => {
|
|
613
|
+
const statements = logsToFlush.map((entry) => this.options.D1.prepare(`INSERT INTO LogTable
|
|
540
614
|
(instanceId, stepName, log, timestamp, type, logOrder, tenantId)
|
|
541
615
|
VALUES (?, ?, ?, ?, ?, ?, ?)`).bind(entry.instanceId, entry.stepName, entry.message, entry.timestamp, entry.type, entry.logOrder, entry.tenantId));
|
|
542
|
-
|
|
616
|
+
return await this.options.D1.batch(statements);
|
|
617
|
+
});
|
|
543
618
|
} catch (error) {
|
|
544
619
|
console.error("Batch log insert failed, falling back to individual inserts:", error);
|
|
545
620
|
await Promise.all(logsToFlush.map((entry) => pushLogToDB(this.options, entry)));
|
|
@@ -667,9 +742,9 @@ async function workflowTableRowToWorkflowRun({ row, serializer, externalBlobStor
|
|
|
667
742
|
};
|
|
668
743
|
}
|
|
669
744
|
async function updateWorkflowName(context, instanceId, newWorkflowName) {
|
|
670
|
-
return await context.D1.prepare(`UPDATE WorkflowTable
|
|
745
|
+
return await retryD1Operation(() => context.D1.prepare(`UPDATE WorkflowTable
|
|
671
746
|
SET workflowName = ?
|
|
672
|
-
WHERE instanceId = ?`).bind(newWorkflowName, instanceId).run();
|
|
747
|
+
WHERE instanceId = ?`).bind(newWorkflowName, instanceId).run());
|
|
673
748
|
}
|
|
674
749
|
/**
|
|
675
750
|
* Update workflow fields by instanceId. Only provided fields will be updated.
|
|
@@ -746,7 +821,7 @@ async function updateWorkflow(context, instanceId, updates) {
|
|
|
746
821
|
};
|
|
747
822
|
bindings.push(instanceId);
|
|
748
823
|
const sql = `UPDATE WorkflowTable SET ${setClauses.join(", ")} WHERE instanceId = ?`;
|
|
749
|
-
return await context.D1.prepare(sql).bind(...bindings).run();
|
|
824
|
+
return await retryD1Operation(() => context.D1.prepare(sql).bind(...bindings).run());
|
|
750
825
|
}
|
|
751
826
|
/**
|
|
752
827
|
* Serialize a workflow property value for storage
|
|
@@ -779,9 +854,9 @@ function deserializeWorkflowPropertyValue(serializedValue, valueType) {
|
|
|
779
854
|
}
|
|
780
855
|
async function upsertWorkflowProperty({ context, instanceId, key, value, tenantId }) {
|
|
781
856
|
const { serializedValue, valueType } = serializeWorkflowPropertyValue(value);
|
|
782
|
-
const res = await context.D1.prepare(`INSERT OR REPLACE INTO WorkflowProperties
|
|
857
|
+
const res = await retryD1Operation(() => context.D1.prepare(`INSERT OR REPLACE INTO WorkflowProperties
|
|
783
858
|
(instanceId, key, value, valueType, tenantId)
|
|
784
|
-
VALUES (?, ?, ?, ?, ?)`).bind(instanceId, key, serializedValue, valueType, tenantId).run();
|
|
859
|
+
VALUES (?, ?, ?, ?, ?)`).bind(instanceId, key, serializedValue, valueType, tenantId).run());
|
|
785
860
|
if (res.error) {
|
|
786
861
|
console.error("Error inserting workflow property:", res.error);
|
|
787
862
|
return false;
|
|
@@ -837,12 +912,12 @@ function prepareWorkflowDependencyStatement({ D1, dependencyWorkflowId, dependen
|
|
|
837
912
|
* ```
|
|
838
913
|
*/
|
|
839
914
|
async function createWorkflowDependency({ context, dependencyWorkflowId, dependentWorkflowId, tenantId }) {
|
|
840
|
-
return await prepareWorkflowDependencyStatement({
|
|
915
|
+
return await retryD1Operation(() => prepareWorkflowDependencyStatement({
|
|
841
916
|
D1: context.D1,
|
|
842
917
|
dependencyWorkflowId,
|
|
843
918
|
dependentWorkflowId,
|
|
844
919
|
tenantId
|
|
845
|
-
}).run();
|
|
920
|
+
}).run());
|
|
846
921
|
}
|
|
847
922
|
/**
|
|
848
923
|
* Create multiple workflow dependencies in a single transaction.
|
|
@@ -875,7 +950,7 @@ async function createWorkflowDependencies({ context, dependencies, tenantId }) {
|
|
|
875
950
|
tenantId,
|
|
876
951
|
createdAt
|
|
877
952
|
}));
|
|
878
|
-
return await context.D1.batch(statements);
|
|
953
|
+
return await retryD1Operation(() => context.D1.batch(statements));
|
|
879
954
|
}
|
|
880
955
|
/**
|
|
881
956
|
* Delete a specific workflow dependency relationship.
|
|
@@ -892,8 +967,8 @@ async function createWorkflowDependencies({ context, dependencies, tenantId }) {
|
|
|
892
967
|
* ```
|
|
893
968
|
*/
|
|
894
969
|
async function deleteWorkflowDependency({ context, dependencyWorkflowId, dependentWorkflowId, tenantId }) {
|
|
895
|
-
return await context.D1.prepare(`DELETE FROM WorkflowDependencies
|
|
896
|
-
WHERE dependencyWorkflowId = ? AND dependentWorkflowId = ? AND tenantId = ?`).bind(dependencyWorkflowId, dependentWorkflowId, tenantId).run();
|
|
970
|
+
return await retryD1Operation(() => context.D1.prepare(`DELETE FROM WorkflowDependencies
|
|
971
|
+
WHERE dependencyWorkflowId = ? AND dependentWorkflowId = ? AND tenantId = ?`).bind(dependencyWorkflowId, dependentWorkflowId, tenantId).run());
|
|
897
972
|
}
|
|
898
973
|
|
|
899
974
|
//#endregion
|
|
@@ -913,7 +988,7 @@ const createCleanupManager = (context) => {
|
|
|
913
988
|
const getAffectedWorkflows = async (config, limit) => {
|
|
914
989
|
const { statuses, statusPlaceholders } = buildStatusesAndPlaceholders(config);
|
|
915
990
|
if (statuses.length === 0) return [];
|
|
916
|
-
const result = await context.D1.prepare(`
|
|
991
|
+
const result = await retryD1Operation(() => context.D1.prepare(`
|
|
917
992
|
SELECT instanceId, inputRef
|
|
918
993
|
FROM WorkflowTable
|
|
919
994
|
WHERE tenantId = ?
|
|
@@ -921,13 +996,13 @@ const createCleanupManager = (context) => {
|
|
|
921
996
|
AND workflowStatus IN (${statusPlaceholders})
|
|
922
997
|
ORDER BY instanceId
|
|
923
998
|
LIMIT ?
|
|
924
|
-
`).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses, limit).all();
|
|
999
|
+
`).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses, limit).all());
|
|
925
1000
|
return result.results;
|
|
926
1001
|
};
|
|
927
1002
|
const getAffectedSteps = async (config, limit) => {
|
|
928
1003
|
const { statuses, statusPlaceholders } = buildStatusesAndPlaceholders(config);
|
|
929
1004
|
if (statuses.length === 0) return [];
|
|
930
|
-
const result = await context.D1.prepare(`
|
|
1005
|
+
const result = await retryD1Operation(() => context.D1.prepare(`
|
|
931
1006
|
SELECT s.instanceId, s.stepName, s.stepStatus, s.errorRef, s.resultRef
|
|
932
1007
|
FROM StepTable s
|
|
933
1008
|
WHERE s.instanceId IN (
|
|
@@ -940,7 +1015,7 @@ const createCleanupManager = (context) => {
|
|
|
940
1015
|
LIMIT ?
|
|
941
1016
|
)
|
|
942
1017
|
ORDER BY s.instanceId, s.stepName
|
|
943
|
-
`).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses, limit).all();
|
|
1018
|
+
`).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses, limit).all());
|
|
944
1019
|
return result.results;
|
|
945
1020
|
};
|
|
946
1021
|
const collectExternalStorageKeys = (workflows, steps) => {
|
|
@@ -993,15 +1068,15 @@ const createCleanupManager = (context) => {
|
|
|
993
1068
|
}
|
|
994
1069
|
if (deletedWorkflows.length > 0) {
|
|
995
1070
|
console.log(`Proceeding to delete workflows from the database...`);
|
|
996
|
-
await context.D1.prepare(`PRAGMA foreign_keys = ON`).run();
|
|
1071
|
+
await retryD1Operation(() => context.D1.prepare(`PRAGMA foreign_keys = ON`).run());
|
|
997
1072
|
let totalDeletedCount = 0;
|
|
998
1073
|
for (let i = 0; i < deletedWorkflows.length; i += 100) {
|
|
999
1074
|
const batch = deletedWorkflows.slice(i, i + 100);
|
|
1000
1075
|
const workflowIds = batch.map(() => "?").join(", ");
|
|
1001
|
-
const result = await context.D1.prepare(`
|
|
1076
|
+
const result = await retryD1Operation(() => context.D1.prepare(`
|
|
1002
1077
|
DELETE FROM WorkflowTable
|
|
1003
1078
|
WHERE instanceId IN (${workflowIds})
|
|
1004
|
-
`).bind(...batch.map((w) => w.instanceId)).run();
|
|
1079
|
+
`).bind(...batch.map((w) => w.instanceId)).run());
|
|
1005
1080
|
totalDeletedCount += result.meta.changes || 0;
|
|
1006
1081
|
}
|
|
1007
1082
|
return {
|
|
@@ -1017,8 +1092,8 @@ const createCleanupManager = (context) => {
|
|
|
1017
1092
|
};
|
|
1018
1093
|
};
|
|
1019
1094
|
const cleanupOrphanedSteps = async (config, limit) => {
|
|
1020
|
-
await context.D1.prepare(`PRAGMA foreign_keys = ON`).run();
|
|
1021
|
-
const orphanedSteps = await context.D1.prepare(`
|
|
1095
|
+
await retryD1Operation(() => context.D1.prepare(`PRAGMA foreign_keys = ON`).run());
|
|
1096
|
+
const orphanedSteps = await retryD1Operation(() => context.D1.prepare(`
|
|
1022
1097
|
SELECT s.instanceId, s.stepName, s.errorRef, s.resultRef
|
|
1023
1098
|
FROM StepTable s
|
|
1024
1099
|
WHERE s.tenantId = ?
|
|
@@ -1028,7 +1103,7 @@ const createCleanupManager = (context) => {
|
|
|
1028
1103
|
)
|
|
1029
1104
|
ORDER BY s.instanceId, s.stepName
|
|
1030
1105
|
LIMIT ?
|
|
1031
|
-
`).bind(context.tenantId, context.tenantId, limit).all();
|
|
1106
|
+
`).bind(context.tenantId, context.tenantId, limit).all());
|
|
1032
1107
|
const orphanedStepResults = orphanedSteps.results;
|
|
1033
1108
|
let deletedExternalStorageKeysCount = 0;
|
|
1034
1109
|
if (config.deleteRefsFromExternalStorage && context.externalBlobStorage && orphanedStepResults.length > 0) {
|
|
@@ -1044,11 +1119,11 @@ const createCleanupManager = (context) => {
|
|
|
1044
1119
|
for (let i = 0; i < orphanedStepResults.length; i += 99) {
|
|
1045
1120
|
const batch = orphanedStepResults.slice(i, i + 99);
|
|
1046
1121
|
const instanceIds = batch.map(() => "?").join(", ");
|
|
1047
|
-
const deletedStepsResult = await context.D1.prepare(`
|
|
1122
|
+
const deletedStepsResult = await retryD1Operation(() => context.D1.prepare(`
|
|
1048
1123
|
DELETE FROM StepTable
|
|
1049
1124
|
WHERE tenantId = ?
|
|
1050
1125
|
AND instanceId IN (${instanceIds})
|
|
1051
|
-
`).bind(context.tenantId, ...batch.map((s) => s.instanceId)).run();
|
|
1126
|
+
`).bind(context.tenantId, ...batch.map((s) => s.instanceId)).run());
|
|
1052
1127
|
totalDeletedSteps += deletedStepsResult.meta.changes || 0;
|
|
1053
1128
|
}
|
|
1054
1129
|
return {
|
|
@@ -1064,8 +1139,8 @@ const createCleanupManager = (context) => {
|
|
|
1064
1139
|
};
|
|
1065
1140
|
};
|
|
1066
1141
|
const cleanupOrphanedLogs = async (limit) => {
|
|
1067
|
-
await context.D1.prepare(`PRAGMA foreign_keys = ON`).run();
|
|
1068
|
-
const deletedLogsResult = await context.D1.prepare(`
|
|
1142
|
+
await retryD1Operation(() => context.D1.prepare(`PRAGMA foreign_keys = ON`).run());
|
|
1143
|
+
const deletedLogsResult = await retryD1Operation(() => context.D1.prepare(`
|
|
1069
1144
|
DELETE FROM LogTable
|
|
1070
1145
|
WHERE tenantId = ?
|
|
1071
1146
|
AND (
|
|
@@ -1080,7 +1155,7 @@ const createCleanupManager = (context) => {
|
|
|
1080
1155
|
))
|
|
1081
1156
|
)
|
|
1082
1157
|
LIMIT ?
|
|
1083
|
-
`).bind(context.tenantId, context.tenantId, context.tenantId, limit).run();
|
|
1158
|
+
`).bind(context.tenantId, context.tenantId, context.tenantId, limit).run());
|
|
1084
1159
|
return { deletedOrphanedLogs: deletedLogsResult.meta.changes || 0 };
|
|
1085
1160
|
};
|
|
1086
1161
|
return {
|
|
@@ -1268,7 +1343,7 @@ const createLogAccessor = (context) => {
|
|
|
1268
1343
|
const limitClause = limit !== void 0 && actualOffset !== void 0 ? "LIMIT ? OFFSET ?" : "";
|
|
1269
1344
|
const sql = `SELECT * FROM StepTable ${whereClause} ORDER BY startTime ASC ${limitClause}`;
|
|
1270
1345
|
if (limit !== void 0 && actualOffset !== void 0) bindings.push(limit, actualOffset);
|
|
1271
|
-
result = await context.D1.prepare(sql).bind(...bindings).all();
|
|
1346
|
+
result = await retryD1Operation(() => context.D1.prepare(sql).bind(...bindings).all());
|
|
1272
1347
|
if (result.results) {
|
|
1273
1348
|
const steps = await Promise.all(result.results.map(async (row) => {
|
|
1274
1349
|
let deserializedResult = null;
|
|
@@ -1295,7 +1370,7 @@ const createLogAccessor = (context) => {
|
|
|
1295
1370
|
const listWorkflows = async (limit, offset, filter, options) => {
|
|
1296
1371
|
const { sql, bindings } = buildFilteredWorkflowQuery(filter, { ignoreTenant: options?.ignoreTenant });
|
|
1297
1372
|
if (options?.debugLogs) console.log("listWorkflows SQL:", sql, "Bindings:", JSON.stringify(bindings));
|
|
1298
|
-
const result = await context.D1.prepare(sql).bind(...bindings, limit, offset).all();
|
|
1373
|
+
const result = await retryD1Operation(() => context.D1.prepare(sql).bind(...bindings, limit, offset).all());
|
|
1299
1374
|
if (options?.debugLogs) console.log("listWorkflows SQL Query executed");
|
|
1300
1375
|
if (result.results) {
|
|
1301
1376
|
const workflows = await Promise.all(result.results.map((row) => workflowTableRowToWorkflowRun({
|
|
@@ -1312,7 +1387,7 @@ const createLogAccessor = (context) => {
|
|
|
1312
1387
|
for (let i = 0; i < instanceIds.length; i += BATCH_SIZE) {
|
|
1313
1388
|
const batchIds = instanceIds.slice(i, i + BATCH_SIZE);
|
|
1314
1389
|
const placeholders = batchIds.map(() => "?").join(", ");
|
|
1315
|
-
const propertiesResult = await context.D1.prepare(`SELECT * FROM WorkflowProperties WHERE instanceId IN (${placeholders})`).bind(...batchIds).all();
|
|
1390
|
+
const propertiesResult = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM WorkflowProperties WHERE instanceId IN (${placeholders})`).bind(...batchIds).all());
|
|
1316
1391
|
if (propertiesResult.results) for (const row of propertiesResult.results) {
|
|
1317
1392
|
const property = {
|
|
1318
1393
|
key: row.key,
|
|
@@ -1333,10 +1408,10 @@ const createLogAccessor = (context) => {
|
|
|
1333
1408
|
for (let i = 0; i < instanceIds.length; i += BATCH_SIZE) {
|
|
1334
1409
|
const batchIds = instanceIds.slice(i, i + BATCH_SIZE);
|
|
1335
1410
|
const placeholders = batchIds.map(() => "?").join(", ");
|
|
1336
|
-
const dependenciesResult = await context.D1.prepare(`SELECT w.*, wd.createdAt, wd.dependentWorkflowId
|
|
1411
|
+
const dependenciesResult = await retryD1Operation(() => context.D1.prepare(`SELECT w.*, wd.createdAt, wd.dependentWorkflowId
|
|
1337
1412
|
FROM WorkflowDependencies wd
|
|
1338
1413
|
JOIN WorkflowTable w ON wd.dependencyWorkflowId = w.instanceId
|
|
1339
|
-
WHERE wd.dependentWorkflowId IN (${placeholders})`).bind(...batchIds).all();
|
|
1414
|
+
WHERE wd.dependentWorkflowId IN (${placeholders})`).bind(...batchIds).all());
|
|
1340
1415
|
if (dependenciesResult.results) for (const row of dependenciesResult.results) {
|
|
1341
1416
|
const depWorkflow = await workflowTableRowToWorkflowRun({
|
|
1342
1417
|
row,
|
|
@@ -1363,10 +1438,10 @@ const createLogAccessor = (context) => {
|
|
|
1363
1438
|
for (let i = 0; i < instanceIds.length; i += BATCH_SIZE) {
|
|
1364
1439
|
const batchIds = instanceIds.slice(i, i + BATCH_SIZE);
|
|
1365
1440
|
const placeholders = batchIds.map(() => "?").join(", ");
|
|
1366
|
-
const dependentsResult = await context.D1.prepare(`SELECT w.*, wd.createdAt, wd.dependencyWorkflowId
|
|
1441
|
+
const dependentsResult = await retryD1Operation(() => context.D1.prepare(`SELECT w.*, wd.createdAt, wd.dependencyWorkflowId
|
|
1367
1442
|
FROM WorkflowDependencies wd
|
|
1368
1443
|
JOIN WorkflowTable w ON wd.dependentWorkflowId = w.instanceId
|
|
1369
|
-
WHERE wd.dependencyWorkflowId IN (${placeholders})`).bind(...batchIds).all();
|
|
1444
|
+
WHERE wd.dependencyWorkflowId IN (${placeholders})`).bind(...batchIds).all());
|
|
1370
1445
|
if (dependentsResult.results) for (const row of dependentsResult.results) {
|
|
1371
1446
|
const depWorkflow = await workflowTableRowToWorkflowRun({
|
|
1372
1447
|
row,
|
|
@@ -1391,7 +1466,7 @@ const createLogAccessor = (context) => {
|
|
|
1391
1466
|
return [];
|
|
1392
1467
|
};
|
|
1393
1468
|
const getWorkflowByParentId = async (parentInstanceId) => {
|
|
1394
|
-
const result = await context.D1.prepare(`SELECT * FROM WorkflowTable WHERE parentInstanceId = ? AND tenantId = ?`).bind(parentInstanceId, context.tenantId).all();
|
|
1469
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM WorkflowTable WHERE parentInstanceId = ? AND tenantId = ?`).bind(parentInstanceId, context.tenantId).all());
|
|
1395
1470
|
if (result.results) {
|
|
1396
1471
|
const workflows = await Promise.all(result.results.map((row) => workflowTableRowToWorkflowRun({
|
|
1397
1472
|
row,
|
|
@@ -1403,7 +1478,7 @@ const createLogAccessor = (context) => {
|
|
|
1403
1478
|
return null;
|
|
1404
1479
|
};
|
|
1405
1480
|
const getWorkflowByTriggerId = async (triggerId) => {
|
|
1406
|
-
const result = await context.D1.prepare(`SELECT * FROM WorkflowTable WHERE triggerId = ? AND tenantId = ?`).bind(triggerId, context.tenantId).first();
|
|
1481
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM WorkflowTable WHERE triggerId = ? AND tenantId = ?`).bind(triggerId, context.tenantId).first());
|
|
1407
1482
|
if (result) {
|
|
1408
1483
|
const workflow = await workflowTableRowToWorkflowRun({
|
|
1409
1484
|
row: result,
|
|
@@ -1415,12 +1490,12 @@ const createLogAccessor = (context) => {
|
|
|
1415
1490
|
return null;
|
|
1416
1491
|
};
|
|
1417
1492
|
const getWorkflowTypesByTenantId = async (tenantId) => {
|
|
1418
|
-
const result = await context.D1.prepare(`SELECT DISTINCT workflowType FROM WorkflowTable WHERE tenantId = ?`).bind(tenantId).all();
|
|
1493
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT DISTINCT workflowType FROM WorkflowTable WHERE tenantId = ?`).bind(tenantId).all());
|
|
1419
1494
|
if (!result.results) return [];
|
|
1420
1495
|
return result.results.map((row) => row.workflowType);
|
|
1421
1496
|
};
|
|
1422
1497
|
const getWorkflowProperties = async (instanceId) => {
|
|
1423
|
-
const result = await context.D1.prepare(`SELECT * FROM WorkflowProperties WHERE instanceId = ?`).bind(instanceId).all();
|
|
1498
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM WorkflowProperties WHERE instanceId = ?`).bind(instanceId).all());
|
|
1424
1499
|
if (!result.results) return [];
|
|
1425
1500
|
return result.results.map((row) => ({
|
|
1426
1501
|
key: row.key,
|
|
@@ -1430,7 +1505,7 @@ const createLogAccessor = (context) => {
|
|
|
1430
1505
|
};
|
|
1431
1506
|
/** This function gets the basic data of a workflow, without populating any of it's complex fields */
|
|
1432
1507
|
const getWorkflowShallow = async (instanceId, options) => {
|
|
1433
|
-
const result = await context.D1.prepare(`SELECT * FROM WorkflowTable WHERE instanceId = ? AND tenantId = ?`).bind(instanceId, context.tenantId).first();
|
|
1508
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM WorkflowTable WHERE instanceId = ? AND tenantId = ?`).bind(instanceId, context.tenantId).first());
|
|
1434
1509
|
if (!result) return null;
|
|
1435
1510
|
const workflow = await workflowTableRowToWorkflowRun({
|
|
1436
1511
|
row: result,
|
|
@@ -1461,7 +1536,7 @@ const createLogAccessor = (context) => {
|
|
|
1461
1536
|
retryWorkflow.isRetryOf = workflow;
|
|
1462
1537
|
});
|
|
1463
1538
|
workflow.isRetryOf = parentWorkflow;
|
|
1464
|
-
const allLogs = await context.D1.prepare(`SELECT * FROM LogTable WHERE instanceId = ? ORDER BY timestamp ASC, logOrder ASC`).bind(instanceId).all();
|
|
1539
|
+
const allLogs = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM LogTable WHERE instanceId = ? ORDER BY timestamp ASC, logOrder ASC`).bind(instanceId).all());
|
|
1465
1540
|
const logs = allLogs.results?.map((logRow) => ({
|
|
1466
1541
|
instanceId: logRow.instanceId,
|
|
1467
1542
|
stepName: logRow.stepName,
|
|
@@ -1477,10 +1552,10 @@ const createLogAccessor = (context) => {
|
|
|
1477
1552
|
const properties = await getWorkflowProperties(instanceId);
|
|
1478
1553
|
workflow.properties = properties;
|
|
1479
1554
|
if (populateDependencies) {
|
|
1480
|
-
const dependenciesResult = await context.D1.prepare(`SELECT w.*, wd.createdAt
|
|
1555
|
+
const dependenciesResult = await retryD1Operation(() => context.D1.prepare(`SELECT w.*, wd.createdAt
|
|
1481
1556
|
FROM WorkflowDependencies wd
|
|
1482
1557
|
JOIN WorkflowTable w ON wd.dependencyWorkflowId = w.instanceId
|
|
1483
|
-
WHERE wd.dependentWorkflowId = ? AND wd.tenantId = ?`).bind(instanceId, context.tenantId).all();
|
|
1558
|
+
WHERE wd.dependentWorkflowId = ? AND wd.tenantId = ?`).bind(instanceId, context.tenantId).all());
|
|
1484
1559
|
if (dependenciesResult.results) workflow.dependencies = await Promise.all(dependenciesResult.results.map(async (row) => {
|
|
1485
1560
|
const depWorkflow = await workflowTableRowToWorkflowRun({
|
|
1486
1561
|
row,
|
|
@@ -1496,10 +1571,10 @@ const createLogAccessor = (context) => {
|
|
|
1496
1571
|
}));
|
|
1497
1572
|
}
|
|
1498
1573
|
if (populateDependents) {
|
|
1499
|
-
const dependentsResult = await context.D1.prepare(`SELECT w.*, wd.createdAt
|
|
1574
|
+
const dependentsResult = await retryD1Operation(() => context.D1.prepare(`SELECT w.*, wd.createdAt
|
|
1500
1575
|
FROM WorkflowDependencies wd
|
|
1501
1576
|
JOIN WorkflowTable w ON wd.dependentWorkflowId = w.instanceId
|
|
1502
|
-
WHERE wd.dependencyWorkflowId = ? AND wd.tenantId = ?`).bind(instanceId, context.tenantId).all();
|
|
1577
|
+
WHERE wd.dependencyWorkflowId = ? AND wd.tenantId = ?`).bind(instanceId, context.tenantId).all());
|
|
1503
1578
|
if (dependentsResult.results) workflow.dependents = await Promise.all(dependentsResult.results.map(async (row) => {
|
|
1504
1579
|
const depWorkflow = await workflowTableRowToWorkflowRun({
|
|
1505
1580
|
row,
|
|
@@ -1517,7 +1592,7 @@ const createLogAccessor = (context) => {
|
|
|
1517
1592
|
return workflow;
|
|
1518
1593
|
};
|
|
1519
1594
|
const getStep = async (instanceId, stepName) => {
|
|
1520
|
-
const result = await context.D1.prepare(`SELECT * FROM StepTable WHERE instanceId = ? AND stepName = ? AND tenantId = ?`).bind(instanceId, stepName, context.tenantId).all();
|
|
1595
|
+
const result = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM StepTable WHERE instanceId = ? AND stepName = ? AND tenantId = ?`).bind(instanceId, stepName, context.tenantId).all());
|
|
1521
1596
|
if (!result.results || result.results.length === 0) return null;
|
|
1522
1597
|
const row = result.results[0];
|
|
1523
1598
|
if (!row) return null;
|
|
@@ -1534,7 +1609,7 @@ const createLogAccessor = (context) => {
|
|
|
1534
1609
|
error: deserializedError,
|
|
1535
1610
|
logs: []
|
|
1536
1611
|
};
|
|
1537
|
-
const logResult = await context.D1.prepare(`SELECT * FROM LogTable WHERE instanceId = ? AND stepName = ? ORDER BY logOrder ASC`).bind(instanceId, stepName).all();
|
|
1612
|
+
const logResult = await retryD1Operation(() => context.D1.prepare(`SELECT * FROM LogTable WHERE instanceId = ? AND stepName = ? ORDER BY logOrder ASC`).bind(instanceId, stepName).all());
|
|
1538
1613
|
if (logResult.results) step.logs = logResult.results.map((logRow) => ({
|
|
1539
1614
|
instanceId: logRow.instanceId,
|
|
1540
1615
|
stepName: logRow.stepName,
|
|
@@ -1545,10 +1620,10 @@ const createLogAccessor = (context) => {
|
|
|
1545
1620
|
return step;
|
|
1546
1621
|
};
|
|
1547
1622
|
const getPropertiesKeys = async (instanceId) => {
|
|
1548
|
-
const result = await context.D1.prepare(`
|
|
1623
|
+
const result = await retryD1Operation(() => context.D1.prepare(`
|
|
1549
1624
|
SELECT DISTINCT key, valueType FROM WorkflowProperties
|
|
1550
1625
|
WHERE tenantId = ? AND (? IS NULL OR instanceId = ?)
|
|
1551
|
-
`).bind(context.tenantId, instanceId ?? null, instanceId ?? null).all();
|
|
1626
|
+
`).bind(context.tenantId, instanceId ?? null, instanceId ?? null).all());
|
|
1552
1627
|
console.log(`debug getPropertiesKeys: ${JSON.stringify(result)}`);
|
|
1553
1628
|
if (!result.results) return [];
|
|
1554
1629
|
return result.results.map((row) => ({
|
|
@@ -1571,7 +1646,7 @@ const createLogAccessor = (context) => {
|
|
|
1571
1646
|
bindings.push(toTime);
|
|
1572
1647
|
}
|
|
1573
1648
|
const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(" AND ")}` : "";
|
|
1574
|
-
const result = await context.D1.prepare(`
|
|
1649
|
+
const result = await retryD1Operation(() => context.D1.prepare(`
|
|
1575
1650
|
SELECT
|
|
1576
1651
|
workflowType,
|
|
1577
1652
|
COUNT(*) as workflowCount,
|
|
@@ -1614,7 +1689,7 @@ const createLogAccessor = (context) => {
|
|
|
1614
1689
|
${whereClause}
|
|
1615
1690
|
GROUP BY workflowType
|
|
1616
1691
|
ORDER BY workflowCount DESC
|
|
1617
|
-
`).bind(...bindings).all();
|
|
1692
|
+
`).bind(...bindings).all());
|
|
1618
1693
|
if (!result.results) return [];
|
|
1619
1694
|
return result.results.map((row) => ({
|
|
1620
1695
|
workflowType: row.workflowType,
|
|
@@ -1927,9 +2002,9 @@ async function createStepContext(context) {
|
|
|
1927
2002
|
const stepStatus$1 = "completed";
|
|
1928
2003
|
const { data: resultData, externalRef: resultRef } = await serializeWithExternalStorage(result, context.serializer, context.externalBlobStorage);
|
|
1929
2004
|
const stepError$1 = null;
|
|
1930
|
-
await context.D1.prepare(`UPDATE StepTable
|
|
2005
|
+
await retryD1Operation(() => context.D1.prepare(`UPDATE StepTable
|
|
1931
2006
|
SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
|
|
1932
|
-
WHERE instanceId = ? AND stepName = ?`).bind(stepStatus$1, endTime, resultData, stepError$1, resultRef, null, instanceId, stepName).run();
|
|
2007
|
+
WHERE instanceId = ? AND stepName = ?`).bind(stepStatus$1, endTime, resultData, stepError$1, resultRef, null, instanceId, stepName).run());
|
|
1933
2008
|
await logBatcher.destroy();
|
|
1934
2009
|
return result;
|
|
1935
2010
|
} catch (error) {
|
|
@@ -1937,9 +2012,9 @@ async function createStepContext(context) {
|
|
|
1937
2012
|
const stepStatus$1 = "failed";
|
|
1938
2013
|
const stepResult$1 = null;
|
|
1939
2014
|
const { data: errorData, externalRef: errorRef } = await serializeWithExternalStorage(error, context.serializer, context.externalBlobStorage);
|
|
1940
|
-
await context.D1.prepare(`UPDATE StepTable
|
|
2015
|
+
await retryD1Operation(() => context.D1.prepare(`UPDATE StepTable
|
|
1941
2016
|
SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
|
|
1942
|
-
WHERE instanceId = ? AND stepName = ?`).bind(stepStatus$1, endTime, stepResult$1, errorData, null, errorRef, instanceId, stepName).run();
|
|
2017
|
+
WHERE instanceId = ? AND stepName = ?`).bind(stepStatus$1, endTime, stepResult$1, errorData, null, errorRef, instanceId, stepName).run());
|
|
1943
2018
|
await logBatcher.destroy();
|
|
1944
2019
|
throw error;
|
|
1945
2020
|
}
|
|
@@ -2094,7 +2169,7 @@ function createWorkflowContext(options) {
|
|
|
2094
2169
|
await ensureTables(options.D1);
|
|
2095
2170
|
ensuredTables = true;
|
|
2096
2171
|
}
|
|
2097
|
-
const oldRun = await options.D1.prepare(`SELECT input, workflowName, tenantId, inputRef FROM WorkflowTable WHERE instanceId = ? `).bind(retryInstanceId).first();
|
|
2172
|
+
const oldRun = await retryD1Operation(() => options.D1.prepare(`SELECT input, workflowName, tenantId, inputRef FROM WorkflowTable WHERE instanceId = ? `).bind(retryInstanceId).first());
|
|
2098
2173
|
const oldWorkflowName = oldRun?.workflowName;
|
|
2099
2174
|
const tenantId = oldRun?.tenantId;
|
|
2100
2175
|
if (!tenantId) throw new Error(`No tenantId found for instanceId ${retryInstanceId}`);
|
|
@@ -2200,4 +2275,4 @@ function createR2ExternalBlobStorage(options) {
|
|
|
2200
2275
|
}
|
|
2201
2276
|
|
|
2202
2277
|
//#endregion
|
|
2203
|
-
export { createCleanupManager, createLogAccessor, createQueueWorkflowContext, createR2ExternalBlobStorage, createStepContext, createWorkflowContext, createWorkflowDependencies, createWorkflowDependency, defaultIdFactory, defaultSerializer, defineWorkflow, deleteWorkflowDependency, deserializeWithExternalStorage, ensureTables, finalizeWorkflowRecord, insertStepRecordFull, insertWorkflowRecord, pushLogToDB, serializeWithExternalStorage, tryDeserializeObj, updateWorkflow, updateWorkflowName, upsertWorkflowProperty, workflowTableRowToWorkflowRun };
|
|
2278
|
+
export { createCleanupManager, createLogAccessor, createQueueWorkflowContext, createR2ExternalBlobStorage, createStepContext, createWorkflowContext, createWorkflowDependencies, createWorkflowDependency, defaultIdFactory, defaultSerializer, defineWorkflow, deleteWorkflowDependency, deserializeWithExternalStorage, ensureTables, finalizeWorkflowRecord, insertStepRecordFull, insertWorkflowRecord, pushLogToDB, retryD1Operation, serializeWithExternalStorage, tryDeserializeObj, updateWorkflow, updateWorkflowName, upsertWorkflowProperty, workflowTableRowToWorkflowRun };
|