@tstdl/base 0.93.154 → 0.93.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/orm/sqls/sqls.d.ts +5 -4
- package/orm/sqls/sqls.js +48 -30
- package/orm/tests/build-jsonb.test.d.ts +1 -0
- package/orm/tests/build-jsonb.test.js +39 -0
- package/orm/tests/query-converter-complex.test.js +24 -19
- package/orm/tests/sql-helpers.test.js +1 -1
- package/package.json +4 -4
- package/task-queue/postgres/task-queue.js +153 -168
- package/task-queue/tests/optimization-edge-cases.test.d.ts +1 -0
- package/task-queue/tests/optimization-edge-cases.test.js +127 -0
- package/test4.js +1 -1
- package/test5.js +11 -5
- package/testing/integration-setup.js +1 -1
package/orm/sqls/sqls.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* simplifying common SQL operations like generating UUIDs, working with intervals,
|
|
5
5
|
* and aggregating data.
|
|
6
6
|
*/
|
|
7
|
-
import { Column,
|
|
7
|
+
import { Column, type AnyColumn, type SQL, type SQLChunk, type SQLWrapper } from 'drizzle-orm';
|
|
8
8
|
import type { GetSelectTableSelection, SelectResultField, TableLike } from 'drizzle-orm/query-builders/select.types';
|
|
9
9
|
import type { EnumerationObject, EnumerationValue, Record } from '../../types/types.js';
|
|
10
10
|
import { type PgEnumFromEnumeration } from '../enums.js';
|
|
@@ -13,7 +13,6 @@ import type { Uuid } from '../types.js';
|
|
|
13
13
|
/** Represents valid units for PostgreSQL interval values. */
|
|
14
14
|
export type IntervalUnit = 'millennium' | 'millenniums' | 'millennia' | 'century' | 'centuries' | 'decade' | 'decades' | 'year' | 'years' | 'day' | 'days' | 'hour' | 'hours' | 'minute' | 'minutes' | 'second' | 'seconds' | 'millisecond' | 'milliseconds' | 'microsecond' | 'microseconds';
|
|
15
15
|
export type ExclusiveColumnCondition = Column | boolean | SQL;
|
|
16
|
-
export declare const simpleJsonKeyPattern: RegExp;
|
|
17
16
|
export type TsHeadlineOptions = {
|
|
18
17
|
/**
|
|
19
18
|
* The longest headline to output.
|
|
@@ -64,6 +63,7 @@ export declare const RANDOM_UUID_V4: SQL<Uuid>;
|
|
|
64
63
|
export declare const RANDOM_UUID_V7: SQL<Uuid>;
|
|
65
64
|
export declare const SQL_TRUE: SQL<boolean>;
|
|
66
65
|
export declare const SQL_FALSE: SQL<boolean>;
|
|
66
|
+
export declare const SQL_NULL: SQL<null>;
|
|
67
67
|
export declare function enumValue<T extends EnumerationObject>(enumeration: T, dbEnum: PgEnumFromEnumeration<T> | string | null, value: EnumerationValue<T>): SQL<string>;
|
|
68
68
|
/**
|
|
69
69
|
* Generates a SQL `CASE` expression to enforce strict, mutually exclusive column usage based on a discriminator value.
|
|
@@ -114,7 +114,7 @@ export declare function exclusiveNotNull(...columns: Column[]): SQL;
|
|
|
114
114
|
* that defines the default condition to apply when a `Column` is provided in `conditionMapping`.
|
|
115
115
|
* By default, it generates an `IS NOT NULL` check.
|
|
116
116
|
*/
|
|
117
|
-
export declare function enumerationCaseWhen<T extends EnumerationObject>(enumeration: T, discriminator: Column, conditionMapping: Record<EnumerationValue<T>, Column | [Column, ...Column[]] | boolean | SQL>, defaultColumnCondition?: (
|
|
117
|
+
export declare function enumerationCaseWhen<T extends EnumerationObject>(enumeration: T, discriminator: Column, conditionMapping: Record<EnumerationValue<T>, Column | [Column, ...Column[]] | boolean | SQL>, defaultColumnCondition?: (columns: Column | [Column, ...Column[]]) => SQL<unknown>): SQL;
|
|
118
118
|
export declare function array<T>(values: readonly (SQL<T> | SQLChunk | T)[]): SQL<T[]>;
|
|
119
119
|
export declare function autoAlias<T>(column: AnyColumn<{
|
|
120
120
|
data: T;
|
|
@@ -125,7 +125,7 @@ export declare function autoAlias<T>(column: AnyColumn<{
|
|
|
125
125
|
* @param unit - The unit of the interval (e.g., 'day', 'hour').
|
|
126
126
|
* @returns A Drizzle SQL object representing the interval.
|
|
127
127
|
*/
|
|
128
|
-
export declare function interval(value: number
|
|
128
|
+
export declare function interval(value: number | SQL<number>, unit: IntervalUnit): SQL;
|
|
129
129
|
/**
|
|
130
130
|
* Creates a PostgreSQL `array_agg` aggregate function call.
|
|
131
131
|
* Aggregates values from a column into a PostgreSQL array.
|
|
@@ -183,6 +183,7 @@ export declare function greatest<T extends (Column | SQL | SQL.Aliased | number)
|
|
|
183
183
|
[P in keyof T]: T[P] extends number ? Exclude<T[P], number> | SQL<number> : T[P];
|
|
184
184
|
}[number]>>;
|
|
185
185
|
export declare function greatest<T>(...values: T[]): SQL<SelectResultField<T>>;
|
|
186
|
+
export declare function power(base: number | SQLChunk, exponent: number | SQLChunk): SQL<number>;
|
|
186
187
|
export declare function unnest<T>(array: SQL<readonly T[]> | SQL.Aliased<readonly T[]> | Column): SQL<T>;
|
|
187
188
|
/**
|
|
188
189
|
* Creates a PostgreSQL array contains operator expression (@>).
|
package/orm/sqls/sqls.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* simplifying common SQL operations like generating UUIDs, working with intervals,
|
|
5
5
|
* and aggregating data.
|
|
6
6
|
*/
|
|
7
|
-
import { and, Column, eq, isSQLWrapper, sql,
|
|
7
|
+
import { and, Column, eq, isSQLWrapper, sql, isNotNull as sqlIsNotNull, isNull as sqlIsNull, Table } from 'drizzle-orm';
|
|
8
8
|
import { match, P } from 'ts-pattern';
|
|
9
9
|
import { distinct, toArray } from '../../utils/array/array.js';
|
|
10
10
|
import { objectEntries, objectValues } from '../../utils/object/object.js';
|
|
@@ -25,7 +25,6 @@ function isJsonb(value) {
|
|
|
25
25
|
}
|
|
26
26
|
return false;
|
|
27
27
|
}
|
|
28
|
-
export const simpleJsonKeyPattern = /^[a-zA-Z0-9_-]+$/u;
|
|
29
28
|
/** Drizzle SQL helper for getting the current transaction's timestamp. Returns a Date object. */
|
|
30
29
|
export const TRANSACTION_TIMESTAMP = sql `transaction_timestamp()`;
|
|
31
30
|
/** Drizzle SQL helper for generating a random UUID (v4). Returns a Uuid string. */
|
|
@@ -34,6 +33,7 @@ export const RANDOM_UUID_V4 = sql `gen_random_uuid()`;
|
|
|
34
33
|
export const RANDOM_UUID_V7 = sql `uuidv7()`;
|
|
35
34
|
export const SQL_TRUE = sql `TRUE`;
|
|
36
35
|
export const SQL_FALSE = sql `FALSE`;
|
|
36
|
+
export const SQL_NULL = sql `NULL`;
|
|
37
37
|
export function enumValue(enumeration, dbEnum, value) {
|
|
38
38
|
if (isNull(dbEnum)) {
|
|
39
39
|
const enumName = getEnumName(enumeration);
|
|
@@ -69,25 +69,45 @@ export function enumValue(enumeration, dbEnum, value) {
|
|
|
69
69
|
* @returns A SQL object representing the complete `CASE discriminator WHEN ... THEN ... ELSE FALSE` statement.
|
|
70
70
|
*/
|
|
71
71
|
export function exclusiveColumn(enumeration, discriminator, conditionMapping) {
|
|
72
|
+
// 1. Gather all unique columns across the entire mapping
|
|
72
73
|
const allColumns = objectValues(conditionMapping)
|
|
73
74
|
.filter(isNotNull)
|
|
74
|
-
.flatMap((value) => toArray(value).filter((
|
|
75
|
+
.flatMap((value) => toArray(value).filter((item) => isInstanceOf(item, Column)));
|
|
75
76
|
const participatingColumns = distinct(allColumns);
|
|
76
|
-
const
|
|
77
|
+
const kaseWhen = caseWhen(discriminator);
|
|
78
|
+
// 2. Iterate and build conditions safely
|
|
79
|
+
for (const [key, value] of objectEntries(conditionMapping)) {
|
|
77
80
|
if (isNull(value)) {
|
|
78
|
-
|
|
81
|
+
kaseWhen.when(enumValue(enumeration, null, key), SQL_FALSE);
|
|
82
|
+
continue;
|
|
79
83
|
}
|
|
80
|
-
const
|
|
84
|
+
const conditions = [];
|
|
85
|
+
const items = toArray(value);
|
|
86
|
+
// Identify required vs null columns
|
|
87
|
+
const requiredColumns = items.filter((item) => isInstanceOf(item, Column));
|
|
81
88
|
const nullColumns = participatingColumns.filter((column) => !requiredColumns.includes(column));
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
.
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
// Push column constraints
|
|
90
|
+
for (const col of requiredColumns) {
|
|
91
|
+
conditions.push(sqlIsNotNull(col));
|
|
92
|
+
}
|
|
93
|
+
for (const col of nullColumns) {
|
|
94
|
+
conditions.push(sqlIsNull(col));
|
|
95
|
+
}
|
|
96
|
+
// Handle custom conditions (SQL or booleans)
|
|
97
|
+
for (const item of items) {
|
|
98
|
+
if (!isInstanceOf(item, Column)) {
|
|
99
|
+
if (isBoolean(item)) {
|
|
100
|
+
if (!item) {
|
|
101
|
+
conditions.push(SQL_FALSE);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
conditions.push(item);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const finalCondition = and(...conditions) ?? SQL_TRUE;
|
|
110
|
+
kaseWhen.when(enumValue(enumeration, null, key), finalCondition);
|
|
91
111
|
}
|
|
92
112
|
return kaseWhen.else(SQL_FALSE);
|
|
93
113
|
}
|
|
@@ -116,20 +136,17 @@ export function exclusiveNotNull(...columns) {
|
|
|
116
136
|
* that defines the default condition to apply when a `Column` is provided in `conditionMapping`.
|
|
117
137
|
* By default, it generates an `IS NOT NULL` check.
|
|
118
138
|
*/
|
|
119
|
-
export function enumerationCaseWhen(enumeration, discriminator, conditionMapping, defaultColumnCondition = (
|
|
120
|
-
const
|
|
139
|
+
export function enumerationCaseWhen(enumeration, discriminator, conditionMapping, defaultColumnCondition = (columns) => isArray(columns) ? and(...columns.map((col) => sqlIsNotNull(col))) : sqlIsNotNull(columns)) {
|
|
140
|
+
const kaseWhen = caseWhen(discriminator);
|
|
121
141
|
for (const [key, value] of objectEntries(conditionMapping)) {
|
|
122
142
|
const condition = match(value)
|
|
123
|
-
.with(P.boolean, (bool) => bool ? SQL_TRUE : SQL_FALSE)
|
|
124
|
-
|
|
143
|
+
.with(P.boolean, (bool) => (bool ? SQL_TRUE : SQL_FALSE))
|
|
144
|
+
// Check for both a single Column OR an Array of Columns
|
|
145
|
+
.when((val) => isInstanceOf(val, Column) || (isArray(val) && val.every((v) => isInstanceOf(v, Column))), (cols) => defaultColumnCondition(cols))
|
|
125
146
|
.otherwise((rawSql) => rawSql);
|
|
126
|
-
|
|
147
|
+
kaseWhen.when(enumValue(enumeration, null, key), condition);
|
|
127
148
|
}
|
|
128
|
-
return
|
|
129
|
-
CASE ${discriminator}
|
|
130
|
-
${sql.join(whens, sql `\n`)}
|
|
131
|
-
ELSE FALSE
|
|
132
|
-
END`;
|
|
149
|
+
return kaseWhen.else(SQL_FALSE);
|
|
133
150
|
}
|
|
134
151
|
export function array(values) {
|
|
135
152
|
const chunks = values.map((value) => isSQLWrapper(value) ? value : sql `${value}`);
|
|
@@ -146,7 +163,7 @@ export function autoAlias(column) {
|
|
|
146
163
|
* @returns A Drizzle SQL object representing the interval.
|
|
147
164
|
*/
|
|
148
165
|
export function interval(value, unit) {
|
|
149
|
-
return sql `(${value} ||' ${sql.raw(unit)}')::interval`;
|
|
166
|
+
return sql `(${value} || ' ${sql.raw(unit)}')::interval`;
|
|
150
167
|
}
|
|
151
168
|
/**
|
|
152
169
|
* Creates a PostgreSQL `array_agg` aggregate function call.
|
|
@@ -224,6 +241,9 @@ export function greatest(...values) {
|
|
|
224
241
|
const sqlValues = values.map((value) => isNumber(value) ? sql.raw(String(value)) : value);
|
|
225
242
|
return sql `greatest(${sql.join(sqlValues, sql.raw(', '))})`;
|
|
226
243
|
}
|
|
244
|
+
export function power(base, exponent) {
|
|
245
|
+
return sql `power(${base}, ${exponent})`;
|
|
246
|
+
}
|
|
227
247
|
export function unnest(array) {
|
|
228
248
|
return sql `unnest(${array})`;
|
|
229
249
|
}
|
|
@@ -408,9 +428,7 @@ export function jsonbBuildObject(properties) {
|
|
|
408
428
|
const chunks = [];
|
|
409
429
|
for (const [key, value] of entries) {
|
|
410
430
|
if (isDefined(value)) {
|
|
411
|
-
|
|
412
|
-
const sqlKey = isSimpleKey ? sql.raw(`'${key}'`) : sql `${key}`;
|
|
413
|
-
chunks.push(sqlKey, buildJsonb(value));
|
|
431
|
+
chunks.push(sql `${key}::text`, buildJsonb(value));
|
|
414
432
|
}
|
|
415
433
|
}
|
|
416
434
|
if (chunks.length == 0) {
|
|
@@ -439,5 +457,5 @@ export function buildJsonb(value) {
|
|
|
439
457
|
if (isObject(value)) {
|
|
440
458
|
return jsonbBuildObject(value);
|
|
441
459
|
}
|
|
442
|
-
return markAsJsonb(sql
|
|
460
|
+
return markAsJsonb(sql `to_jsonb(${value})`);
|
|
443
461
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { PgDialect } from 'drizzle-orm/pg-core';
|
|
2
|
+
import { describe, expect, test } from 'vitest';
|
|
3
|
+
import { buildJsonb } from '../sqls/sqls.js';
|
|
4
|
+
describe('buildJsonb', () => {
|
|
5
|
+
const dialect = new PgDialect();
|
|
6
|
+
test('should build jsonb from simple object', () => {
|
|
7
|
+
const query = buildJsonb({ a: 1, b: 'foo' });
|
|
8
|
+
const { sql, params } = dialect.sqlToQuery(query);
|
|
9
|
+
expect(sql).toBe('jsonb_build_object($1::text, to_jsonb($2), $3::text, to_jsonb($4))');
|
|
10
|
+
expect(params).toEqual(['a', 1, 'b', 'foo']);
|
|
11
|
+
});
|
|
12
|
+
test('should build jsonb from object with non-simple keys', () => {
|
|
13
|
+
const query = buildJsonb({ 'Betriebs-Nr.': '18182952' });
|
|
14
|
+
const { sql, params } = dialect.sqlToQuery(query);
|
|
15
|
+
// This is what failed before: it lacked the ::text cast
|
|
16
|
+
expect(sql).toBe('jsonb_build_object($1::text, to_jsonb($2))');
|
|
17
|
+
expect(params).toEqual(['Betriebs-Nr.', '18182952']);
|
|
18
|
+
});
|
|
19
|
+
test('should build jsonb from nested structures', () => {
|
|
20
|
+
const query = buildJsonb({
|
|
21
|
+
additionalData: { 'Betriebs-Nr.': '18182952' },
|
|
22
|
+
tags: ['a', 'b']
|
|
23
|
+
});
|
|
24
|
+
const { sql, params } = dialect.sqlToQuery(query);
|
|
25
|
+
expect(sql).toBe('jsonb_build_object($1::text, jsonb_build_object($2::text, to_jsonb($3)), $4::text, jsonb_build_array(to_jsonb($5), to_jsonb($6)))');
|
|
26
|
+
expect(params).toEqual(['additionalData', 'Betriebs-Nr.', '18182952', 'tags', 'a', 'b']);
|
|
27
|
+
});
|
|
28
|
+
test('should handle numbers correctly', () => {
|
|
29
|
+
const query = buildJsonb({ score: 0.5 });
|
|
30
|
+
const { sql, params } = dialect.sqlToQuery(query);
|
|
31
|
+
expect(sql).toBe('jsonb_build_object($1::text, to_jsonb($2))');
|
|
32
|
+
expect(params).toEqual(['score', 0.5]);
|
|
33
|
+
});
|
|
34
|
+
test('should handle null and empty structures', () => {
|
|
35
|
+
expect(dialect.sqlToQuery(buildJsonb(null)).sql).toBe('\'null\'::jsonb');
|
|
36
|
+
expect(dialect.sqlToQuery(buildJsonb({})).sql).toBe('\'{}\'::jsonb');
|
|
37
|
+
expect(dialect.sqlToQuery(buildJsonb([])).sql).toBe('\'[]\'::jsonb');
|
|
38
|
+
});
|
|
39
|
+
});
|
|
@@ -7,13 +7,12 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
-
import { describe, expect, test } from 'vitest';
|
|
11
|
-
import { sql } from 'drizzle-orm';
|
|
12
10
|
import { PgDialect } from 'drizzle-orm/pg-core';
|
|
13
|
-
import {
|
|
11
|
+
import { Integer, StringProperty } from '../../schema/index.js';
|
|
12
|
+
import { describe, expect, test } from 'vitest';
|
|
13
|
+
import { Table } from '../decorators.js';
|
|
14
14
|
import { Entity } from '../entity.js';
|
|
15
|
-
import {
|
|
16
|
-
import { getDrizzleTableFromType, getColumnDefinitionsMap } from '../server/drizzle/schema-converter.js';
|
|
15
|
+
import { getColumnDefinitionsMap, getDrizzleTableFromType } from '../server/drizzle/schema-converter.js';
|
|
17
16
|
import { convertQuery } from '../server/query-converter.js';
|
|
18
17
|
describe('ORM Query Converter Complex', () => {
|
|
19
18
|
const dialect = new PgDialect();
|
|
@@ -49,10 +48,11 @@ describe('ORM Query Converter Complex', () => {
|
|
|
49
48
|
},
|
|
50
49
|
};
|
|
51
50
|
const condition = convertQuery(q, table, colMap);
|
|
52
|
-
const
|
|
53
|
-
expect(
|
|
54
|
-
expect(
|
|
55
|
-
expect(
|
|
51
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
52
|
+
expect(sql).toContain('setweight(to_tsvector($1, "test"."complex_items"."name"), \'A\')');
|
|
53
|
+
expect(sql).toContain('setweight(to_tsvector($2, "test"."complex_items"."description"), \'B\')');
|
|
54
|
+
expect(sql).toContain('websearch_to_tsquery($3, $4)');
|
|
55
|
+
expect(params).toEqual(['english', 'english', 'english', 'search term']);
|
|
56
56
|
});
|
|
57
57
|
test('should handle ParadeDB $parade match with tokenizer', () => {
|
|
58
58
|
const q = {
|
|
@@ -66,9 +66,10 @@ describe('ORM Query Converter Complex', () => {
|
|
|
66
66
|
},
|
|
67
67
|
};
|
|
68
68
|
const condition = convertQuery(q, table, colMap);
|
|
69
|
-
const
|
|
69
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
70
70
|
// Tokenizer is supported via JSON object syntax
|
|
71
|
-
expect(
|
|
71
|
+
expect(sql).toContain('"test"."complex_items"."name" @@@ jsonb_build_object($1::text, jsonb_build_object($2::text, to_jsonb($3), $4::text, jsonb_build_object($5::text, to_jsonb($6), $7::text, to_jsonb($8), $9::text, to_jsonb($10))))::pdb.query');
|
|
72
|
+
expect(params).toEqual(['match', 'value', 'test', 'tokenizer', 'type', 'ngram', 'min_gram', 3, 'max_gram', 3]);
|
|
72
73
|
});
|
|
73
74
|
test('should handle ParadeDB $parade range', () => {
|
|
74
75
|
const q = {
|
|
@@ -82,9 +83,10 @@ describe('ORM Query Converter Complex', () => {
|
|
|
82
83
|
},
|
|
83
84
|
};
|
|
84
85
|
const condition = convertQuery(q, table, colMap);
|
|
85
|
-
const
|
|
86
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
86
87
|
// This should fall back to convertParadeComparisonQuery with recursive jsonb build
|
|
87
|
-
expect(
|
|
88
|
+
expect(sql).toContain('"test"."complex_items"."value" @@@ jsonb_build_object($1::text, jsonb_build_object($2::text, jsonb_build_object($3::text, to_jsonb($4)), $5::text, jsonb_build_object($6::text, to_jsonb($7))))::pdb.query');
|
|
89
|
+
expect(params).toEqual(['range', 'lower_bound', 'included', 10, 'upper_bound', 'excluded', 20]);
|
|
88
90
|
});
|
|
89
91
|
test('should handle ParadeDB $parade phrasePrefix', () => {
|
|
90
92
|
const q = {
|
|
@@ -95,8 +97,9 @@ describe('ORM Query Converter Complex', () => {
|
|
|
95
97
|
},
|
|
96
98
|
};
|
|
97
99
|
const condition = convertQuery(q, table, colMap);
|
|
98
|
-
const
|
|
99
|
-
expect(
|
|
100
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
101
|
+
expect(sql).toContain('"test"."complex_items"."name" @@@ jsonb_build_object($1::text, jsonb_build_object($2::text, jsonb_build_array(to_jsonb($3), to_jsonb($4)), $5::text, to_jsonb($6)))::pdb.query');
|
|
102
|
+
expect(params).toEqual(['phrase_prefix', 'phrases', 'hello', 'wor', 'max_expansions', 10]);
|
|
100
103
|
});
|
|
101
104
|
test('should handle ParadeDB $parade regexPhrase', () => {
|
|
102
105
|
const q = {
|
|
@@ -107,8 +110,9 @@ describe('ORM Query Converter Complex', () => {
|
|
|
107
110
|
},
|
|
108
111
|
};
|
|
109
112
|
const condition = convertQuery(q, table, colMap);
|
|
110
|
-
const
|
|
111
|
-
expect(
|
|
113
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
114
|
+
expect(sql).toContain('"test"."complex_items"."name" @@@ jsonb_build_object($1::text, jsonb_build_object($2::text, jsonb_build_array(to_jsonb($3), to_jsonb($4)), $5::text, to_jsonb($6)))::pdb.query');
|
|
115
|
+
expect(params).toEqual(['regex_phrase', 'regexes', 'he.*', 'wo.*', 'slop', 1]);
|
|
112
116
|
});
|
|
113
117
|
test('should handle ParadeDB top-level moreLikeThis', () => {
|
|
114
118
|
const q = {
|
|
@@ -120,7 +124,8 @@ describe('ORM Query Converter Complex', () => {
|
|
|
120
124
|
},
|
|
121
125
|
};
|
|
122
126
|
const condition = convertQuery(q, table, colMap);
|
|
123
|
-
const
|
|
124
|
-
expect(
|
|
127
|
+
const { sql, params } = dialect.sqlToQuery(condition);
|
|
128
|
+
expect(sql).toContain('"test"."complex_items"."id" @@@ jsonb_build_object($1::text, jsonb_build_object($2::text, to_jsonb($3), $4::text, to_jsonb(ARRAY[$5, $6])))::pdb.query');
|
|
129
|
+
expect(params).toEqual(['more_like_this', 'key_value', '123', 'fields', 'name', 'description']);
|
|
125
130
|
});
|
|
126
131
|
});
|
|
@@ -42,7 +42,7 @@ describe('ORM SQL Helpers', () => {
|
|
|
42
42
|
test('interval should generate postgres interval syntax', () => {
|
|
43
43
|
const i = interval(5, 'days');
|
|
44
44
|
const sqlStr = dialect.sqlToQuery(i).sql;
|
|
45
|
-
expect(sqlStr).toContain("||' days')::interval");
|
|
45
|
+
expect(sqlStr).toContain("|| ' days')::interval");
|
|
46
46
|
});
|
|
47
47
|
test('coalesce should join multiple columns', () => {
|
|
48
48
|
const c = coalesce(testTable.colA, testTable.colB, sql `'default'`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tstdl/base",
|
|
3
|
-
"version": "0.93.
|
|
3
|
+
"version": "0.93.156",
|
|
4
4
|
"author": "Patrick Hein",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -152,8 +152,8 @@
|
|
|
152
152
|
"type-fest": "^5.4"
|
|
153
153
|
},
|
|
154
154
|
"peerDependencies": {
|
|
155
|
-
"@aws-sdk/client-s3": "^3.
|
|
156
|
-
"@aws-sdk/s3-request-presigner": "^3.
|
|
155
|
+
"@aws-sdk/client-s3": "^3.1002",
|
|
156
|
+
"@aws-sdk/s3-request-presigner": "^3.1002",
|
|
157
157
|
"@genkit-ai/google-genai": "^1.29",
|
|
158
158
|
"@google-cloud/storage": "^7.19",
|
|
159
159
|
"@toon-format/toon": "^2.1.0",
|
|
@@ -168,7 +168,7 @@
|
|
|
168
168
|
"handlebars": "^4.7",
|
|
169
169
|
"mjml": "^4.18",
|
|
170
170
|
"nodemailer": "^8.0",
|
|
171
|
-
"pg": "^8.
|
|
171
|
+
"pg": "^8.20",
|
|
172
172
|
"playwright": "^1.58",
|
|
173
173
|
"preact": "^10.28",
|
|
174
174
|
"preact-render-to-string": "^6.6",
|
|
@@ -60,15 +60,14 @@ import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists
|
|
|
60
60
|
import { filter, merge, throttleTime } from 'rxjs';
|
|
61
61
|
import { CancellationSignal } from '../../cancellation/index.js';
|
|
62
62
|
import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
|
|
63
|
-
import { serializeError, TimeoutError } from '../../errors/index.js';
|
|
63
|
+
import { NotFoundError, serializeError, TimeoutError } from '../../errors/index.js';
|
|
64
64
|
import { afterResolve, inject, provide, Singleton } from '../../injector/index.js';
|
|
65
65
|
import { Logger } from '../../logger/index.js';
|
|
66
66
|
import { MessageBus } from '../../message-bus/index.js';
|
|
67
|
-
import { arrayOverlaps, caseWhen, coalesce, enumValue,
|
|
68
|
-
import {
|
|
67
|
+
import { arrayOverlaps, caseWhen, coalesce, enumValue, greatest, interval, jsonbBuildObject, least, power, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
+
import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
69
69
|
import { RateLimiter } from '../../rate-limit/index.js';
|
|
70
70
|
import { distinct, toArray } from '../../utils/array/array.js';
|
|
71
|
-
import { currentTimestamp } from '../../utils/date-time.js';
|
|
72
71
|
import { Timer } from '../../utils/timer.js';
|
|
73
72
|
import { cancelableTimeout } from '../../utils/timing.js';
|
|
74
73
|
import { isArray, isDefined, isNotNull, isNull, isNumber, isString, isUndefined } from '../../utils/type-guards.js';
|
|
@@ -76,10 +75,9 @@ import { millisecondsPerMinute, millisecondsPerSecond } from '../../utils/units.
|
|
|
76
75
|
import { defaultQueueConfig, queueableOrWaitableStatuses, queueableStatuses, TaskDependencyType, TaskQueue, TaskStatus, terminalStatuses } from '../task-queue.js';
|
|
77
76
|
import { ensureTaskError } from '../task.error.js';
|
|
78
77
|
import { PostgresTaskQueueModuleConfig } from './module.js';
|
|
79
|
-
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable,
|
|
78
|
+
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskStatus, task as taskTable } from './schemas.js';
|
|
80
79
|
import { PostgresTask, PostgresTaskArchive } from './task.model.js';
|
|
81
80
|
let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
82
|
-
#database = inject(Database);
|
|
83
81
|
#repository = injectRepository(PostgresTask);
|
|
84
82
|
#archiveRepository = injectRepository(PostgresTaskArchive);
|
|
85
83
|
#config = this.config;
|
|
@@ -399,13 +397,17 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
399
397
|
.set({
|
|
400
398
|
unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
|
|
401
399
|
unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} + ${updates.completeIncrement}`,
|
|
402
|
-
status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)),
|
|
400
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), TaskStatus.Waiting).else(taskTable.status),
|
|
403
401
|
})
|
|
404
402
|
.from(updates)
|
|
405
403
|
.where(eq(taskTable.id, updates.taskId))
|
|
406
404
|
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
405
|
+
const notifiedNamespaces = new Set();
|
|
407
406
|
for (const row of updatedRows) {
|
|
408
|
-
|
|
407
|
+
if (!notifiedNamespaces.has(row.namespace)) {
|
|
408
|
+
this.notify(row.namespace);
|
|
409
|
+
notifiedNamespaces.add(row.namespace);
|
|
410
|
+
}
|
|
409
411
|
}
|
|
410
412
|
}
|
|
411
413
|
async has(id, options) {
|
|
@@ -535,23 +537,29 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
535
537
|
await this.cancelMany([id], options);
|
|
536
538
|
}
|
|
537
539
|
async cancelMany(ids, options) {
|
|
540
|
+
if (ids.length == 0) {
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
538
543
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
539
|
-
const
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
544
|
+
const cancelledRows = await tx.pgTransaction.execute(sql `
|
|
545
|
+
WITH RECURSIVE task_tree AS (
|
|
546
|
+
SELECT id FROM ${taskTable} WHERE ${inArray(taskTable.id, ids)}
|
|
547
|
+
UNION ALL
|
|
548
|
+
SELECT child.id FROM ${taskTable} child JOIN task_tree parent ON child.parent_id = parent.id
|
|
549
|
+
)
|
|
550
|
+
UPDATE ${taskTable}
|
|
551
|
+
SET
|
|
552
|
+
status = ${enumValue(TaskStatus, taskStatus, TaskStatus.Cancelled)},
|
|
553
|
+
token = NULL,
|
|
554
|
+
complete_timestamp = ${TRANSACTION_TIMESTAMP}
|
|
555
|
+
FROM task_tree
|
|
556
|
+
WHERE
|
|
557
|
+
${taskTable.id} = task_tree.id
|
|
558
|
+
AND ${taskTable.status} NOT IN (${sql.join(terminalStatuses.map((s) => enumValue(TaskStatus, taskStatus, s)), sql `, `)})
|
|
559
|
+
RETURNING ${taskTable.id} as id, ${taskTable.namespace} as namespace
|
|
560
|
+
`);
|
|
561
|
+
if (cancelledRows.rows.length > 0) {
|
|
562
|
+
await this.resolveDependenciesMany(cancelledRows.rows.map((row) => ({ id: row.id, status: TaskStatus.Cancelled, namespace: row.namespace })), { transaction: tx });
|
|
555
563
|
}
|
|
556
564
|
});
|
|
557
565
|
}
|
|
@@ -705,42 +713,29 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
705
713
|
if (isNull(task.token)) {
|
|
706
714
|
return undefined;
|
|
707
715
|
}
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
if (isDefined(updatedRow)) {
|
|
724
|
-
return await this.#repository.mapToEntity(updatedRow);
|
|
725
|
-
}
|
|
726
|
-
const [existingRow] = await tx.pgTransaction
|
|
727
|
-
.select({ startTimestamp: taskTable.startTimestamp })
|
|
728
|
-
.from(taskTable)
|
|
729
|
-
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)));
|
|
730
|
-
if (isDefined(existingRow) && isNotNull(existingRow.startTimestamp) && (currentTimestamp() - existingRow.startTimestamp) > this.maxExecutionTime) {
|
|
731
|
-
await tx.pgTransaction
|
|
732
|
-
.update(taskTable)
|
|
733
|
-
.set({
|
|
734
|
-
status: TaskStatus.TimedOut,
|
|
735
|
-
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
736
|
-
error: { code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' },
|
|
737
|
-
})
|
|
738
|
-
.where(eq(taskTable.id, task.id));
|
|
739
|
-
await this.resolveDependenciesMany([{ id: task.id, status: TaskStatus.TimedOut, namespace: task.namespace }], { transaction: tx });
|
|
740
|
-
this.notify();
|
|
741
|
-
}
|
|
716
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
717
|
+
const exceededMaxExecutionTime = lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`);
|
|
718
|
+
const [updatedRow] = await session
|
|
719
|
+
.update(taskTable)
|
|
720
|
+
.set({
|
|
721
|
+
status: caseWhen(exceededMaxExecutionTime, enumValue(TaskStatus, taskStatus, TaskStatus.TimedOut)).else(taskTable.status),
|
|
722
|
+
visibilityDeadline: caseWhen(exceededMaxExecutionTime, null).else(sql `${TRANSACTION_TIMESTAMP} + ${interval(this.visibilityTimeout, 'milliseconds')}`),
|
|
723
|
+
completeTimestamp: caseWhen(exceededMaxExecutionTime, TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
|
|
724
|
+
error: caseWhen(exceededMaxExecutionTime, jsonbBuildObject({ code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' })).else(taskTable.error),
|
|
725
|
+
progress: caseWhen(exceededMaxExecutionTime, taskTable.progress).else(isDefined(options?.progress) ? options.progress : taskTable.progress),
|
|
726
|
+
state: caseWhen(exceededMaxExecutionTime, taskTable.state).else(isDefined(options?.state) ? options.state : taskTable.state),
|
|
727
|
+
})
|
|
728
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
|
|
729
|
+
.returning();
|
|
730
|
+
if (isUndefined(updatedRow)) {
|
|
742
731
|
return undefined;
|
|
743
|
-
}
|
|
732
|
+
}
|
|
733
|
+
if (updatedRow.status == TaskStatus.TimedOut) {
|
|
734
|
+
await this.resolveDependencies(task.id, TaskStatus.TimedOut, { namespace: task.namespace, transaction: options?.transaction });
|
|
735
|
+
this.notify();
|
|
736
|
+
return undefined;
|
|
737
|
+
}
|
|
738
|
+
return await this.#repository.mapToEntity(updatedRow);
|
|
744
739
|
}
|
|
745
740
|
async touchMany(tasks, options) {
|
|
746
741
|
if (tasks.length == 0) {
|
|
@@ -778,34 +773,24 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
778
773
|
}
|
|
779
774
|
async complete(task, options) {
|
|
780
775
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
781
|
-
const [freshTask] = await tx.pgTransaction
|
|
782
|
-
.select({ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
783
|
-
.from(taskTable)
|
|
784
|
-
.where(eq(taskTable.id, task.id))
|
|
785
|
-
.for('update');
|
|
786
|
-
if (isUndefined(freshTask)) {
|
|
787
|
-
return;
|
|
788
|
-
}
|
|
789
|
-
const hasActiveChildren = freshTask.unresolvedCompleteDependencies > 0;
|
|
790
|
-
const nextStatus = hasActiveChildren ? TaskStatus.WaitingChildren : TaskStatus.Completed;
|
|
791
776
|
const [updatedTask] = await tx.pgTransaction.update(taskTable)
|
|
792
777
|
.set({
|
|
793
|
-
status:
|
|
778
|
+
status: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.WaitingChildren)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Completed)),
|
|
794
779
|
token: null,
|
|
795
780
|
result: options?.result,
|
|
796
|
-
progress:
|
|
797
|
-
completeTimestamp: (
|
|
781
|
+
progress: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), task.progress).else(sql.raw('1')),
|
|
782
|
+
completeTimestamp: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), null).else(TRANSACTION_TIMESTAMP),
|
|
798
783
|
visibilityDeadline: null,
|
|
799
784
|
})
|
|
800
785
|
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
|
|
801
|
-
.returning({ id: taskTable.id });
|
|
786
|
+
.returning({ id: taskTable.id, status: taskTable.status });
|
|
802
787
|
if (isUndefined(updatedTask)) {
|
|
803
788
|
return;
|
|
804
789
|
}
|
|
805
|
-
if (
|
|
790
|
+
if (updatedTask.status == TaskStatus.Completed) {
|
|
806
791
|
await this.#circuitBreaker.recordSuccess();
|
|
807
792
|
}
|
|
808
|
-
await this.resolveDependencies(task.id,
|
|
793
|
+
await this.resolveDependencies(task.id, updatedTask.status, { namespace: task.namespace, transaction: tx });
|
|
809
794
|
});
|
|
810
795
|
}
|
|
811
796
|
async completeMany(tasks, options) {
|
|
@@ -855,9 +840,9 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
855
840
|
const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
|
|
856
841
|
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
857
842
|
const delay = isRetryable
|
|
858
|
-
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
843
|
+
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
|
|
859
844
|
: 0;
|
|
860
|
-
const nextSchedule =
|
|
845
|
+
const nextSchedule = sql `${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')}`;
|
|
861
846
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
862
847
|
const [updatedRow] = await tx.pgTransaction
|
|
863
848
|
.update(taskTable)
|
|
@@ -866,7 +851,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
866
851
|
token: null,
|
|
867
852
|
error: serializeError(error),
|
|
868
853
|
visibilityDeadline: null,
|
|
869
|
-
scheduleTimestamp: nextSchedule,
|
|
854
|
+
scheduleTimestamp: isRetryable ? nextSchedule : taskTable.scheduleTimestamp,
|
|
870
855
|
startTimestamp: null,
|
|
871
856
|
completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
|
|
872
857
|
})
|
|
@@ -889,31 +874,31 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
889
874
|
const isRetryable = (task.tries < this.maxTries);
|
|
890
875
|
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
891
876
|
const delay = isRetryable
|
|
892
|
-
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
877
|
+
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
|
|
893
878
|
: 0;
|
|
894
|
-
const nextSchedule =
|
|
895
|
-
const completeTimestamp = (nextStatus == TaskStatus.Dead) ?
|
|
896
|
-
return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}
|
|
879
|
+
const nextSchedule = sql `(${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')})`;
|
|
880
|
+
const completeTimestamp = (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null;
|
|
881
|
+
return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}::${taskStatus}, ${serializeError(error)}::jsonb, ${nextSchedule}::timestamptz, ${completeTimestamp}::timestamptz)`;
|
|
897
882
|
});
|
|
898
883
|
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
899
884
|
.select({
|
|
900
|
-
updateId: sql `(id)
|
|
901
|
-
updateToken: sql `(token)
|
|
902
|
-
updateTries: sql `(tries)
|
|
903
|
-
updateStatus: sql `(status)
|
|
904
|
-
updateError: sql `(error)
|
|
905
|
-
updateSchedule: sql `(schedule_timestamp)
|
|
906
|
-
updateComplete: sql `(complete_timestamp)
|
|
885
|
+
updateId: sql `(id)`.as('update_id'),
|
|
886
|
+
updateToken: sql `(token)`.as('update_token'),
|
|
887
|
+
updateTries: sql `(tries)`.as('update_tries'),
|
|
888
|
+
updateStatus: sql `(status)`.as('update_status'),
|
|
889
|
+
updateError: sql `(error)`.as('update_error'),
|
|
890
|
+
updateSchedule: sql `(schedule_timestamp)`.as('update_schedule'),
|
|
891
|
+
updateComplete: sql `(complete_timestamp)`.as('update_complete'),
|
|
907
892
|
})
|
|
908
893
|
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, tries, status, error, schedule_timestamp, complete_timestamp)`));
|
|
909
894
|
const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
|
|
910
895
|
.update(taskTable)
|
|
911
896
|
.set({
|
|
912
|
-
status: sql `${updates.updateStatus}
|
|
897
|
+
status: sql `${updates.updateStatus}`,
|
|
913
898
|
token: null,
|
|
914
899
|
error: sql `${updates.updateError}`,
|
|
915
900
|
visibilityDeadline: null,
|
|
916
|
-
scheduleTimestamp:
|
|
901
|
+
scheduleTimestamp: caseWhen(eq(updates.updateStatus, TaskStatus.Retrying), updates.updateSchedule).else(taskTable.scheduleTimestamp),
|
|
917
902
|
startTimestamp: null,
|
|
918
903
|
completeTimestamp: sql `${updates.updateComplete}`,
|
|
919
904
|
})
|
|
@@ -934,41 +919,41 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
934
919
|
await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
|
|
935
920
|
}
|
|
936
921
|
async resolveDependenciesMany(tasks, options) {
|
|
937
|
-
|
|
922
|
+
const tasksToResolve = tasks.filter((t) => terminalStatuses.includes(t.status));
|
|
923
|
+
if (tasksToResolve.length == 0) {
|
|
938
924
|
return;
|
|
939
925
|
}
|
|
940
|
-
const taskStatusMap = new Map(tasks.map((t) => [t.id, t.status]));
|
|
941
926
|
const notifiedNamespaces = new Set();
|
|
942
927
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
943
|
-
const
|
|
944
|
-
|
|
928
|
+
const taskValues = tasksToResolve.map((t) => sql `(${t.id}::uuid, ${t.status}::${taskStatus})`);
|
|
929
|
+
// 1. CTE: Load the incoming terminal tasks into a memory table
|
|
930
|
+
const resolvedTasks = tx.pgTransaction.$with('resolved_tasks').as((qb) => qb
|
|
945
931
|
.select({
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
type: taskDependencyTable.type,
|
|
949
|
-
requiredStatuses: taskDependencyTable.requiredStatuses,
|
|
932
|
+
resolvedId: sql `(id)`.as('resolved_id'),
|
|
933
|
+
resolvedStatus: sql `(status)`.as('resolved_status'),
|
|
950
934
|
})
|
|
951
|
-
.from(
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
}
|
|
935
|
+
.from(sql `(VALUES ${sql.join(taskValues, sql `, `)}) AS t(id, status)`));
|
|
936
|
+
// 2. CTE: Atomically delete all edges pointing to these terminal tasks and return them
|
|
937
|
+
const deletedEdges = tx.pgTransaction.$with('deleted_edges').as(() => tx.pgTransaction
|
|
938
|
+
.delete(taskDependencyTable)
|
|
939
|
+
.where(inArray(taskDependencyTable.dependencyTaskId, tx.pgTransaction.select({ id: resolvedTasks.resolvedId }).from(resolvedTasks)))
|
|
940
|
+
.returning());
|
|
941
|
+
// 3. Execute: Join deleted edges with their resolving status to determine if they matched the required status
|
|
942
|
+
const resolvedEdges = await tx.pgTransaction
|
|
943
|
+
.with(resolvedTasks, deletedEdges)
|
|
944
|
+
.select({
|
|
945
|
+
taskId: deletedEdges.taskId,
|
|
946
|
+
dependencyTaskId: deletedEdges.dependencyTaskId,
|
|
947
|
+
type: deletedEdges.type,
|
|
948
|
+
isMatched: sql `${resolvedTasks.resolvedStatus} = ANY(${deletedEdges.requiredStatuses})`.as('is_matched'),
|
|
949
|
+
})
|
|
950
|
+
.from(deletedEdges)
|
|
951
|
+
.innerJoin(resolvedTasks, eq(deletedEdges.dependencyTaskId, resolvedTasks.resolvedId));
|
|
969
952
|
if (resolvedEdges.length == 0) {
|
|
970
953
|
return;
|
|
971
954
|
}
|
|
955
|
+
// Extract skipped dependencies (terminal status but not a matched status)
|
|
956
|
+
const abortOnDependencyFailureTaskIds = distinct(resolvedEdges.filter((d) => !d.isMatched).map((d) => d.taskId));
|
|
972
957
|
const sortedResolvedEdges = resolvedEdges.toSorted((a, b) => {
|
|
973
958
|
const idCompare = a.taskId.localeCompare(b.taskId);
|
|
974
959
|
if (idCompare != 0) {
|
|
@@ -980,18 +965,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
980
965
|
}
|
|
981
966
|
return a.type.localeCompare(b.type);
|
|
982
967
|
});
|
|
983
|
-
const edgeValues = sortedResolvedEdges.map((e) => sql `(${e.taskId}::uuid, ${e.dependencyTaskId}::uuid, ${e.type}::text)`);
|
|
984
|
-
await tx.pgTransaction.execute(sql `
|
|
985
|
-
DELETE FROM ${taskDependencyTable}
|
|
986
|
-
WHERE (task_id, dependency_task_id, type) IN (
|
|
987
|
-
SELECT t.task_id, t.dependency_task_id, t.type::${taskDependencyType}
|
|
988
|
-
FROM (VALUES ${sql.join(edgeValues, sql `, `)}) AS t(task_id, dependency_task_id, type)
|
|
989
|
-
)
|
|
990
|
-
`);
|
|
991
968
|
const terminalTasks = [];
|
|
992
969
|
const skippedTaskIds = new Set();
|
|
993
|
-
if (abortOnDependencyFailureTaskIds.
|
|
994
|
-
const sortedAbortIds =
|
|
970
|
+
if (abortOnDependencyFailureTaskIds.length > 0) {
|
|
971
|
+
const sortedAbortIds = abortOnDependencyFailureTaskIds.toSorted();
|
|
995
972
|
const dependentTasks = await tx.pgTransaction
|
|
996
973
|
.select({ id: taskTable.id, namespace: taskTable.namespace, abortOnDependencyFailure: taskTable.abortOnDependencyFailure, status: taskTable.status })
|
|
997
974
|
.from(taskTable)
|
|
@@ -1054,7 +1031,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1054
1031
|
.set({
|
|
1055
1032
|
unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
|
|
1056
1033
|
unresolvedCompleteDependencies: greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`),
|
|
1057
|
-
status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)),
|
|
1034
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)), TaskStatus.Pending).else(caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TaskStatus.Completed).else(taskTable.status)),
|
|
1058
1035
|
progress: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), 1).else(taskTable.progress),
|
|
1059
1036
|
completeTimestamp: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
|
|
1060
1037
|
token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
|
|
@@ -1086,47 +1063,50 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1086
1063
|
}
|
|
1087
1064
|
}
|
|
1088
1065
|
async maintenance(options) {
|
|
1089
|
-
await
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
this.processPriorityAging(options),
|
|
1095
|
-
]);
|
|
1066
|
+
await this.processExpirations(options);
|
|
1067
|
+
await this.processZombieRetries(options);
|
|
1068
|
+
await this.processZombieExhaustions(options);
|
|
1069
|
+
await this.processHardTimeouts(options);
|
|
1070
|
+
await this.processPriorityAging(options);
|
|
1096
1071
|
await this.performArchival(options);
|
|
1097
1072
|
await this.performArchivePurge(options);
|
|
1098
1073
|
}
|
|
1099
1074
|
async performArchival(options) {
|
|
1075
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1100
1076
|
while (true) {
|
|
1101
|
-
const
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1077
|
+
const childTaskTable = aliasedTable(taskTable, 'childTask');
|
|
1078
|
+
const selection = session.$with('selection').as((qb) => qb
|
|
1079
|
+
.select({ id: taskTable.id })
|
|
1080
|
+
.from(taskTable)
|
|
1081
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, terminalStatuses), lte(taskTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}`), notExists(session
|
|
1082
|
+
.select({ id: childTaskTable.id })
|
|
1083
|
+
.from(childTaskTable)
|
|
1084
|
+
.where(eq(childTaskTable.parentId, taskTable.id))), notExists(session
|
|
1085
|
+
.select({ taskId: taskDependencyTable.taskId })
|
|
1086
|
+
.from(taskDependencyTable)
|
|
1087
|
+
.where(eq(taskDependencyTable.dependencyTaskId, taskTable.id)))))
|
|
1088
|
+
.limit(1000)
|
|
1089
|
+
.for('update', { skipLocked: true }));
|
|
1090
|
+
const deleted = session.$with('deleted').as(() => session
|
|
1091
|
+
.delete(taskTable)
|
|
1092
|
+
.where(inArray(taskTable.id, session.select().from(selection)))
|
|
1093
|
+
.returning());
|
|
1094
|
+
const inserted = session.$with('inserted').as(() => session
|
|
1095
|
+
.insert(taskArchiveTable)
|
|
1096
|
+
.select(session.select().from(deleted))
|
|
1097
|
+
.returning({ id: taskArchiveTable.id }));
|
|
1098
|
+
const [result] = await session
|
|
1099
|
+
.with(selection, deleted, inserted)
|
|
1100
|
+
.select({ count: count() })
|
|
1101
|
+
.from(inserted);
|
|
1102
|
+
if ((result?.count ?? 0) < 1000) {
|
|
1123
1103
|
break;
|
|
1124
1104
|
}
|
|
1125
1105
|
}
|
|
1126
1106
|
}
|
|
1127
1107
|
async performArchivePurge(options) {
|
|
1128
|
-
const session = options?.transaction?.pgTransaction ?? this.#
|
|
1129
|
-
const selection = session.$with('
|
|
1108
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1109
|
+
const selection = session.$with('selection').as((qb) => qb
|
|
1130
1110
|
.select({ id: taskArchiveTable.id })
|
|
1131
1111
|
.from(taskArchiveTable)
|
|
1132
1112
|
.where(and(eq(taskArchiveTable.namespace, this.#namespace), lte(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
|
|
@@ -1143,7 +1123,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1143
1123
|
}
|
|
1144
1124
|
}
|
|
1145
1125
|
async processExpirations(options) {
|
|
1146
|
-
const expiredSelection = this.#
|
|
1126
|
+
const expiredSelection = this.#repository.session.$with('expired_selection').as((qb) => qb
|
|
1147
1127
|
.select({ id: taskTable.id })
|
|
1148
1128
|
.from(taskTable)
|
|
1149
1129
|
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableOrWaitableStatuses), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
@@ -1173,7 +1153,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1173
1153
|
}
|
|
1174
1154
|
}
|
|
1175
1155
|
async processZombieRetries(options) {
|
|
1176
|
-
const session = options?.transaction?.pgTransaction ?? this.#
|
|
1156
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1177
1157
|
const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
|
|
1178
1158
|
.select({ id: taskTable.id })
|
|
1179
1159
|
.from(taskTable)
|
|
@@ -1189,7 +1169,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1189
1169
|
token: null,
|
|
1190
1170
|
visibilityDeadline: null,
|
|
1191
1171
|
startTimestamp: null,
|
|
1192
|
-
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`,
|
|
1172
|
+
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(least(this.retryDelayMaximum, sql `${this.retryDelayMinimum} * ${power(this.retryDelayGrowth, sql `${taskTable.tries} - 1`)}`), 'milliseconds')}`,
|
|
1193
1173
|
error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
|
|
1194
1174
|
})
|
|
1195
1175
|
.where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
|
|
@@ -1200,7 +1180,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1200
1180
|
}
|
|
1201
1181
|
}
|
|
1202
1182
|
async processZombieExhaustions(options) {
|
|
1203
|
-
const zombieExhaustionSelection = this.#
|
|
1183
|
+
const zombieExhaustionSelection = this.#repository.session.$with('selection').as((qb) => qb
|
|
1204
1184
|
.select({ id: taskTable.id })
|
|
1205
1185
|
.from(taskTable)
|
|
1206
1186
|
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
|
|
@@ -1231,7 +1211,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1231
1211
|
}
|
|
1232
1212
|
}
|
|
1233
1213
|
async processHardTimeouts(options) {
|
|
1234
|
-
const timeoutSelection = this.#
|
|
1214
|
+
const timeoutSelection = this.#repository.session.$with('selection').as((qb) => qb
|
|
1235
1215
|
.select({ id: taskTable.id })
|
|
1236
1216
|
.from(taskTable)
|
|
1237
1217
|
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
@@ -1262,7 +1242,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1262
1242
|
}
|
|
1263
1243
|
}
|
|
1264
1244
|
async processPriorityAging(options) {
|
|
1265
|
-
const session = options?.transaction?.pgTransaction ?? this.#
|
|
1245
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1266
1246
|
const agingSelection = session.$with('aging_selection').as((qb) => qb
|
|
1267
1247
|
.select({ id: taskTable.id })
|
|
1268
1248
|
.from(taskTable)
|
|
@@ -1285,8 +1265,8 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1285
1265
|
}
|
|
1286
1266
|
}
|
|
1287
1267
|
async restart(id, options) {
|
|
1288
|
-
const
|
|
1289
|
-
await
|
|
1268
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1269
|
+
const [updatedTask] = await session
|
|
1290
1270
|
.update(taskTable)
|
|
1291
1271
|
.set({
|
|
1292
1272
|
status: TaskStatus.Pending,
|
|
@@ -1301,7 +1281,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1301
1281
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1302
1282
|
state: (options?.resetState == true) ? null : undefined,
|
|
1303
1283
|
})
|
|
1304
|
-
.where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, terminalStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))))
|
|
1284
|
+
.where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, terminalStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))))
|
|
1285
|
+
.returning();
|
|
1286
|
+
if (isUndefined(updatedTask)) {
|
|
1287
|
+
throw new NotFoundError('Task not found or not in a restartable state.');
|
|
1288
|
+
}
|
|
1289
|
+
this.notify(updatedTask.namespace);
|
|
1305
1290
|
}
|
|
1306
1291
|
notify(namespace = this.#namespace) {
|
|
1307
1292
|
this.#messageBus.publishAndForget(namespace);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { and, eq, sql } from 'drizzle-orm';
|
|
2
|
+
import { beforeAll, describe, expect, vi } from 'vitest';
|
|
3
|
+
import { inject } from '../../injector/index.js';
|
|
4
|
+
import { TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
5
|
+
import { injectRepository } from '../../orm/server/index.js';
|
|
6
|
+
import { setupIntegrationTest, testInInjector } from '../../testing/index.js';
|
|
7
|
+
import { timeout } from '../../utils/timing.js';
|
|
8
|
+
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, task as taskTable } from '../postgres/schemas.js';
|
|
9
|
+
import { PostgresTaskQueue } from '../postgres/task-queue.js';
|
|
10
|
+
import { PostgresTask, PostgresTaskArchive } from '../postgres/task.model.js';
|
|
11
|
+
import { TaskDependencyType, TaskStatus } from '../task-queue.js';
|
|
12
|
+
describe('Task Queue Optimization Edge Cases', () => {
|
|
13
|
+
let context;
|
|
14
|
+
beforeAll(async () => {
|
|
15
|
+
context = await setupIntegrationTest({ modules: { taskQueue: true } });
|
|
16
|
+
});
|
|
17
|
+
testInInjector('should notify unique namespaces exactly once in incrementCounters', () => context.injector, async () => {
|
|
18
|
+
const q1 = inject(PostgresTaskQueue, 'ns-1');
|
|
19
|
+
const q2 = inject(PostgresTaskQueue, 'ns-2');
|
|
20
|
+
const notifySpy = vi.spyOn(PostgresTaskQueue.prototype, 'notify');
|
|
21
|
+
const [t1] = await q1.enqueueMany([{ type: 't1', data: {} }], { returnTasks: true });
|
|
22
|
+
const [t2] = await q2.enqueueMany([{ type: 't2', data: {} }], { returnTasks: true });
|
|
23
|
+
notifySpy.mockClear();
|
|
24
|
+
await q1.incrementCounters([
|
|
25
|
+
{ taskId: t1.id, dependencyTaskId: 'some-dep', type: TaskDependencyType.Schedule },
|
|
26
|
+
{ taskId: t2.id, dependencyTaskId: 'some-dep', type: TaskDependencyType.Schedule },
|
|
27
|
+
{ taskId: t1.id, dependencyTaskId: 'other-dep', type: TaskDependencyType.Schedule },
|
|
28
|
+
]);
|
|
29
|
+
const notifiedNamespaces = notifySpy.mock.calls.map(call => call[0]);
|
|
30
|
+
expect(notifiedNamespaces).toContain('ns-1');
|
|
31
|
+
expect(notifiedNamespaces).toContain('ns-2');
|
|
32
|
+
expect(notifiedNamespaces.filter(n => n == 'ns-1').length).toBe(1);
|
|
33
|
+
expect(notifiedNamespaces.filter(n => n == 'ns-2').length).toBe(1);
|
|
34
|
+
});
|
|
35
|
+
testInInjector('should resolve edge and abort on unmatched terminal status in resolveDependenciesMany', () => context.injector, async () => {
|
|
36
|
+
const queue = inject(PostgresTaskQueue, 'test-namespace');
|
|
37
|
+
const repository = injectRepository(PostgresTask);
|
|
38
|
+
const [parent] = await queue.enqueueMany([{ type: 'parent', data: {}, abortOnDependencyFailure: true }], { returnTasks: true });
|
|
39
|
+
const [child] = await queue.enqueueMany([{ type: 'child', data: {} }], { returnTasks: true });
|
|
40
|
+
await repository.session.insert(taskDependencyTable).values({
|
|
41
|
+
taskId: parent.id,
|
|
42
|
+
dependencyTaskId: child.id,
|
|
43
|
+
type: TaskDependencyType.Schedule,
|
|
44
|
+
requiredStatuses: [TaskStatus.Completed],
|
|
45
|
+
});
|
|
46
|
+
await repository.session.update(taskTable).set({ unresolvedScheduleDependencies: 1, status: TaskStatus.Waiting }).where(eq(taskTable.id, parent.id));
|
|
47
|
+
await queue.resolveDependenciesMany([{ id: child.id, status: TaskStatus.Dead, namespace: 'test-namespace' }]);
|
|
48
|
+
const updatedParent = await queue.getTask(parent.id);
|
|
49
|
+
expect(updatedParent.status).toBe(TaskStatus.Skipped);
|
|
50
|
+
});
|
|
51
|
+
testInInjector('should handle hard timeout during touch', () => context.injector, async () => {
|
|
52
|
+
// Configure with small maxExecutionTime
|
|
53
|
+
const queue = inject(PostgresTaskQueue, { namespace: 'timeout-test', maxExecutionTime: 10 });
|
|
54
|
+
const [task] = await queue.enqueueMany([{ type: 'test', data: {} }], { returnTasks: true });
|
|
55
|
+
const [runningTask] = await queue.dequeueMany(1);
|
|
56
|
+
await timeout(50);
|
|
57
|
+
const result = await queue.touch(runningTask);
|
|
58
|
+
expect(result).toBeUndefined();
|
|
59
|
+
const updated = await queue.getTask(runningTask.id);
|
|
60
|
+
expect(updated.status).toBe(TaskStatus.TimedOut);
|
|
61
|
+
});
|
|
62
|
+
testInInjector('should handle non-existent tasks in complete and fail', () => context.injector, async () => {
|
|
63
|
+
const queue = inject(PostgresTaskQueue, 'missing-test');
|
|
64
|
+
const fakeTask = { id: crypto.randomUUID(), token: crypto.randomUUID(), tries: 0 };
|
|
65
|
+
await expect(queue.complete(fakeTask)).resolves.toBeUndefined();
|
|
66
|
+
await expect(queue.fail(fakeTask, new Error('fail'))).resolves.toBeUndefined();
|
|
67
|
+
});
|
|
68
|
+
testInInjector('should handle terminal tasks with no dependents in resolveDependenciesMany', () => context.injector, async () => {
|
|
69
|
+
const queue = inject(PostgresTaskQueue, 'no-deps-test');
|
|
70
|
+
const [task] = await queue.enqueueMany([{ type: 'test', data: {} }], { returnTasks: true });
|
|
71
|
+
await queue.resolveDependenciesMany([{ id: task.id, status: TaskStatus.Completed }]);
|
|
72
|
+
});
|
|
73
|
+
testInInjector('should handle archival and purge in maintenance', () => context.injector, async () => {
|
|
74
|
+
const namespace = `archival-test-${crypto.randomUUID()}`;
|
|
75
|
+
// Configure with small retention
|
|
76
|
+
const queue = inject(PostgresTaskQueue, { namespace, retention: 1000, archiveRetention: 1000 });
|
|
77
|
+
const repository = injectRepository(PostgresTask);
|
|
78
|
+
const archiveRepository = injectRepository(PostgresTaskArchive);
|
|
79
|
+
const [task] = await queue.enqueueMany([{ type: 'test', data: {} }], { returnTasks: true });
|
|
80
|
+
await repository.session.update(taskTable)
|
|
81
|
+
.set({
|
|
82
|
+
status: TaskStatus.Completed,
|
|
83
|
+
completeTimestamp: sql `${TRANSACTION_TIMESTAMP} - interval '2 seconds'`,
|
|
84
|
+
})
|
|
85
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.namespace, namespace)));
|
|
86
|
+
await queue.performArchival();
|
|
87
|
+
const archived = await archiveRepository.load(task.id);
|
|
88
|
+
expect(archived).toBeDefined();
|
|
89
|
+
await repository.session.update(taskArchiveTable)
|
|
90
|
+
.set({ completeTimestamp: sql `${TRANSACTION_TIMESTAMP} - interval '2 seconds'` })
|
|
91
|
+
.where(and(eq(taskArchiveTable.id, task.id), eq(taskArchiveTable.namespace, namespace)));
|
|
92
|
+
await queue.performArchivePurge();
|
|
93
|
+
const purged = await archiveRepository.load(task.id).catch(() => undefined);
|
|
94
|
+
expect(purged).toBeUndefined();
|
|
95
|
+
});
|
|
96
|
+
testInInjector('should notify on restart', () => context.injector, async () => {
|
|
97
|
+
const namespace = `restart-test-${crypto.randomUUID()}`;
|
|
98
|
+
const queue = inject(PostgresTaskQueue, namespace);
|
|
99
|
+
const notifySpy = vi.spyOn(PostgresTaskQueue.prototype, 'notify');
|
|
100
|
+
const [task] = await queue.enqueueMany([{ type: 'test', data: {} }], { returnTasks: true });
|
|
101
|
+
await queue.cancelMany([task.id]);
|
|
102
|
+
notifySpy.mockClear();
|
|
103
|
+
await queue.restart(task.id);
|
|
104
|
+
expect(notifySpy).toHaveBeenCalledWith(namespace);
|
|
105
|
+
});
|
|
106
|
+
testInInjector('should use exponential backoff for zombies', () => context.injector, async () => {
|
|
107
|
+
const namespace = `zombie-backoff-${crypto.randomUUID()}`;
|
|
108
|
+
// Configure with standard growth
|
|
109
|
+
const queue = inject(PostgresTaskQueue, { namespace, retryDelayMinimum: 1000, retryDelayGrowth: 2 });
|
|
110
|
+
const repository = injectRepository(PostgresTask);
|
|
111
|
+
const [task] = await queue.enqueueMany([{ type: 'test', data: {} }], { returnTasks: true });
|
|
112
|
+
await queue.dequeueMany(1);
|
|
113
|
+
await repository.session.update(taskTable)
|
|
114
|
+
.set({
|
|
115
|
+
status: TaskStatus.Running,
|
|
116
|
+
visibilityDeadline: sql `${TRANSACTION_TIMESTAMP} - interval '1 minute'`,
|
|
117
|
+
tries: 2,
|
|
118
|
+
})
|
|
119
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.namespace, namespace)));
|
|
120
|
+
await queue.processZombieRetries();
|
|
121
|
+
const updated = await queue.getTask(task.id);
|
|
122
|
+
expect(updated.status).toBe(TaskStatus.Retrying);
|
|
123
|
+
const delay = updated.scheduleTimestamp - Date.now();
|
|
124
|
+
expect(delay).toBeGreaterThan(1000);
|
|
125
|
+
expect(delay).toBeLessThan(3000);
|
|
126
|
+
});
|
|
127
|
+
});
|
package/test4.js
CHANGED
|
@@ -23,7 +23,7 @@ const config = {
|
|
|
23
23
|
},
|
|
24
24
|
},
|
|
25
25
|
s3: {
|
|
26
|
-
endpoint: string('S3_ENDPOINT', 'http://localhost:
|
|
26
|
+
endpoint: string('S3_ENDPOINT', 'http://localhost:19552'),
|
|
27
27
|
accessKey: string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
28
28
|
secretKey: string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
29
29
|
bucket: string('S3_BUCKET', undefined),
|
package/test5.js
CHANGED
|
@@ -3,12 +3,18 @@ import { Application } from './application/application.js';
|
|
|
3
3
|
import { provideModule, provideSignalHandler } from './application/index.js';
|
|
4
4
|
import { PrettyPrintLogFormatter } from './logger/index.js';
|
|
5
5
|
import { provideConsoleLogTransport } from './logger/transports/console.js';
|
|
6
|
+
import { TaskQueue } from './task-queue/task-queue.js';
|
|
7
|
+
import { setupIntegrationTest } from './testing/integration-setup.js';
|
|
8
|
+
import { createArray } from './utils/array/array.js';
|
|
9
|
+
import { timedBenchmarkAsync } from './utils/benchmark.js';
|
|
6
10
|
async function main(_cancellationSignal) {
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
const { injector } = await setupIntegrationTest({ modules: { taskQueue: true } });
|
|
12
|
+
const queue1 = injector.resolve(TaskQueue, 'namespace-1');
|
|
13
|
+
const batch = createArray(1000, (i) => ({ type: 'test', data: { index: i } }));
|
|
14
|
+
const enqueueResult = await timedBenchmarkAsync(1000, async () => {
|
|
15
|
+
await queue1.enqueueMany(batch);
|
|
16
|
+
});
|
|
17
|
+
console.log(enqueueResult.operationsPerMillisecond * batch.length * 1000, 'items/s');
|
|
12
18
|
}
|
|
13
19
|
Application.run('Test', [
|
|
14
20
|
provideConsoleLogTransport(PrettyPrintLogFormatter),
|
|
@@ -125,7 +125,7 @@ export async function setupIntegrationTest(options = {}) {
|
|
|
125
125
|
if (options.modules?.objectStorage) {
|
|
126
126
|
const bucketPerModule = options.s3?.bucketPerModule ?? configParser.boolean('S3_BUCKET_PER_MODULE', true);
|
|
127
127
|
configureS3ObjectStorage({
|
|
128
|
-
endpoint: options.s3?.endpoint ?? configParser.string('S3_ENDPOINT', 'http://127.0.0.1:
|
|
128
|
+
endpoint: options.s3?.endpoint ?? configParser.string('S3_ENDPOINT', 'http://127.0.0.1:19552'),
|
|
129
129
|
accessKey: options.s3?.accessKey ?? configParser.string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
130
130
|
secretKey: options.s3?.secretKey ?? configParser.string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
131
131
|
bucket: bucketPerModule ? undefined : (options.s3?.bucket ?? configParser.string('S3_BUCKET', 'test-bucket')),
|