@fedify/sqlite 2.0.0-pr.490.2 → 2.0.0-pr.559.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +3 -3
- package/deno.json +7 -4
- package/dist/dist/sqlite.node.d.cts +2 -0
- package/dist/kv.cjs +30 -0
- package/dist/kv.d.cts +11 -6
- package/dist/kv.d.ts +8 -3
- package/dist/kv.js +30 -0
- package/dist/mod.cjs +3 -1
- package/dist/mod.d.cts +2 -1
- package/dist/mod.d.ts +2 -1
- package/dist/mod.js +2 -1
- package/dist/mq.cjs +393 -0
- package/dist/mq.d.cts +112 -0
- package/dist/mq.d.ts +113 -0
- package/dist/mq.js +392 -0
- package/package.json +22 -8
- package/src/kv.test.ts +94 -1
- package/src/kv.ts +53 -3
- package/src/mod.ts +1 -0
- package/src/mq.test.ts +24 -0
- package/src/mq.ts +623 -0
- package/tsdown.config.ts +7 -1
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
|
|
7
7
|
This package provides a SQLite-based [`KvStore`] implementation for [Fedify].
|
|
8
8
|
|
|
9
|
-
[JSR]: https://jsr.io/@fedify/sqlite
|
|
10
9
|
[JSR badge]: https://jsr.io/badges/@fedify/sqlite
|
|
11
|
-
[
|
|
10
|
+
[JSR]: https://jsr.io/@fedify/sqlite
|
|
12
11
|
[npm badge]: https://img.shields.io/npm/v/@fedify/sqlite?logo=npm
|
|
13
|
-
[
|
|
12
|
+
[npm]: https://www.npmjs.com/package/@fedify/sqlite
|
|
14
13
|
[`KvStore`]: https://jsr.io/@fedify/fedify/doc/federation/~/KvStore
|
|
14
|
+
[Fedify]: https://fedify.dev/
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
Usage
|
package/deno.json
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fedify/sqlite",
|
|
3
|
-
"version": "2.0.0-pr.
|
|
3
|
+
"version": "2.0.0-pr.559.5+39b87b25",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/mod.ts",
|
|
7
|
-
"./kv": "./src/kv.ts"
|
|
7
|
+
"./kv": "./src/kv.ts",
|
|
8
|
+
"./mq": "./src/mq.ts"
|
|
8
9
|
},
|
|
9
10
|
"imports": {
|
|
10
11
|
"#sqlite": "./src/sqlite.node.ts"
|
|
@@ -16,11 +17,13 @@
|
|
|
16
17
|
],
|
|
17
18
|
"publish": {
|
|
18
19
|
"exclude": [
|
|
19
|
-
"
|
|
20
|
+
"**/*.test.ts",
|
|
21
|
+
"!dist/",
|
|
22
|
+
"tsdown.config.ts"
|
|
20
23
|
]
|
|
21
24
|
},
|
|
22
25
|
"tasks": {
|
|
23
26
|
"check": "deno fmt --check && deno lint && deno check",
|
|
24
|
-
"test": "deno test --allow-net --allow-env --doc --no-check=leaks"
|
|
27
|
+
"test": "deno test --allow-net --allow-env --allow-read --allow-write --doc --no-check=leaks"
|
|
25
28
|
}
|
|
26
29
|
}
|
package/dist/kv.cjs
CHANGED
|
@@ -134,6 +134,36 @@ var SqliteKvStore = class SqliteKvStore {
|
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
136
|
/**
|
|
137
|
+
* {@inheritDoc KvStore.list}
|
|
138
|
+
* @since 1.10.0
|
|
139
|
+
*/
|
|
140
|
+
async *list(prefix) {
|
|
141
|
+
this.initialize();
|
|
142
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
143
|
+
let results;
|
|
144
|
+
if (prefix == null || prefix.length === 0) results = this.#db.prepare(`
|
|
145
|
+
SELECT key, value
|
|
146
|
+
FROM "${this.#tableName}"
|
|
147
|
+
WHERE expires IS NULL OR expires > ?
|
|
148
|
+
ORDER BY key
|
|
149
|
+
`).all(now);
|
|
150
|
+
else {
|
|
151
|
+
const pattern = JSON.stringify(prefix).slice(0, -1) + ",%";
|
|
152
|
+
const exactKey = JSON.stringify(prefix);
|
|
153
|
+
results = this.#db.prepare(`
|
|
154
|
+
SELECT key, value
|
|
155
|
+
FROM "${this.#tableName}"
|
|
156
|
+
WHERE (key LIKE ? ESCAPE '\\' OR key = ?)
|
|
157
|
+
AND (expires IS NULL OR expires > ?)
|
|
158
|
+
ORDER BY key
|
|
159
|
+
`).all(pattern, exactKey, now);
|
|
160
|
+
}
|
|
161
|
+
for (const row of results) yield {
|
|
162
|
+
key: this.#decodeKey(row.key),
|
|
163
|
+
value: this.#decodeValue(row.value)
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
137
167
|
* Creates the table used by the key–value store if it does not already exist.
|
|
138
168
|
* Does nothing if the table already exists.
|
|
139
169
|
*/
|
package/dist/kv.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
|
|
1
|
+
import { DatabaseSync } from "./dist/sqlite.node.cjs";
|
|
2
|
+
import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
|
|
3
3
|
|
|
4
4
|
//#region src/kv.d.ts
|
|
5
5
|
/**
|
|
@@ -12,12 +12,12 @@ interface SqliteKvStoreOptions {
|
|
|
12
12
|
* `"fedify_kv"` by default.
|
|
13
13
|
* @default `"fedify_kv"`
|
|
14
14
|
*/
|
|
15
|
-
tableName?: string;
|
|
15
|
+
readonly tableName?: string;
|
|
16
16
|
/**
|
|
17
17
|
* Whether the table has been initialized. `false` by default.
|
|
18
18
|
* @default `false`
|
|
19
19
|
*/
|
|
20
|
-
initialized?: boolean;
|
|
20
|
+
readonly initialized?: boolean;
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
23
|
* A key–value store that uses SQLite as the underlying storage.
|
|
@@ -37,14 +37,14 @@ interface SqliteKvStoreOptions {
|
|
|
37
37
|
*/
|
|
38
38
|
declare class SqliteKvStore implements KvStore {
|
|
39
39
|
#private;
|
|
40
|
-
readonly db:
|
|
40
|
+
readonly db: DatabaseSync;
|
|
41
41
|
readonly options: SqliteKvStoreOptions;
|
|
42
42
|
/**
|
|
43
43
|
* Creates a new SQLite key–value store.
|
|
44
44
|
* @param db The SQLite database to use. Supports `node:sqlite` and `bun:sqlite`.
|
|
45
45
|
* @param options The options for the key–value store.
|
|
46
46
|
*/
|
|
47
|
-
constructor(db:
|
|
47
|
+
constructor(db: DatabaseSync, options?: SqliteKvStoreOptions);
|
|
48
48
|
/**
|
|
49
49
|
* {@inheritDoc KvStore.get}
|
|
50
50
|
*/
|
|
@@ -61,6 +61,11 @@ declare class SqliteKvStore implements KvStore {
|
|
|
61
61
|
* {@inheritDoc KvStore.cas}
|
|
62
62
|
*/
|
|
63
63
|
cas(key: KvKey, expectedValue: unknown, newValue: unknown, options?: KvStoreSetOptions): Promise<boolean>;
|
|
64
|
+
/**
|
|
65
|
+
* {@inheritDoc KvStore.list}
|
|
66
|
+
* @since 1.10.0
|
|
67
|
+
*/
|
|
68
|
+
list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
|
|
64
69
|
/**
|
|
65
70
|
* Creates the table used by the key–value store if it does not already exist.
|
|
66
71
|
* Does nothing if the table already exists.
|
package/dist/kv.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Temporal } from "@js-temporal/polyfill";
|
|
2
2
|
import { PlatformDatabase } from "#sqlite";
|
|
3
|
-
import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
|
|
3
|
+
import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
|
|
4
4
|
|
|
5
5
|
//#region src/kv.d.ts
|
|
6
6
|
/**
|
|
@@ -13,12 +13,12 @@ interface SqliteKvStoreOptions {
|
|
|
13
13
|
* `"fedify_kv"` by default.
|
|
14
14
|
* @default `"fedify_kv"`
|
|
15
15
|
*/
|
|
16
|
-
tableName?: string;
|
|
16
|
+
readonly tableName?: string;
|
|
17
17
|
/**
|
|
18
18
|
* Whether the table has been initialized. `false` by default.
|
|
19
19
|
* @default `false`
|
|
20
20
|
*/
|
|
21
|
-
initialized?: boolean;
|
|
21
|
+
readonly initialized?: boolean;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* A key–value store that uses SQLite as the underlying storage.
|
|
@@ -62,6 +62,11 @@ declare class SqliteKvStore implements KvStore {
|
|
|
62
62
|
* {@inheritDoc KvStore.cas}
|
|
63
63
|
*/
|
|
64
64
|
cas(key: KvKey, expectedValue: unknown, newValue: unknown, options?: KvStoreSetOptions): Promise<boolean>;
|
|
65
|
+
/**
|
|
66
|
+
* {@inheritDoc KvStore.list}
|
|
67
|
+
* @since 1.10.0
|
|
68
|
+
*/
|
|
69
|
+
list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
|
|
65
70
|
/**
|
|
66
71
|
* Creates the table used by the key–value store if it does not already exist.
|
|
67
72
|
* Does nothing if the table already exists.
|
package/dist/kv.js
CHANGED
|
@@ -133,6 +133,36 @@ var SqliteKvStore = class SqliteKvStore {
|
|
|
133
133
|
}
|
|
134
134
|
}
|
|
135
135
|
/**
|
|
136
|
+
* {@inheritDoc KvStore.list}
|
|
137
|
+
* @since 1.10.0
|
|
138
|
+
*/
|
|
139
|
+
async *list(prefix) {
|
|
140
|
+
this.initialize();
|
|
141
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
142
|
+
let results;
|
|
143
|
+
if (prefix == null || prefix.length === 0) results = this.#db.prepare(`
|
|
144
|
+
SELECT key, value
|
|
145
|
+
FROM "${this.#tableName}"
|
|
146
|
+
WHERE expires IS NULL OR expires > ?
|
|
147
|
+
ORDER BY key
|
|
148
|
+
`).all(now);
|
|
149
|
+
else {
|
|
150
|
+
const pattern = JSON.stringify(prefix).slice(0, -1) + ",%";
|
|
151
|
+
const exactKey = JSON.stringify(prefix);
|
|
152
|
+
results = this.#db.prepare(`
|
|
153
|
+
SELECT key, value
|
|
154
|
+
FROM "${this.#tableName}"
|
|
155
|
+
WHERE (key LIKE ? ESCAPE '\\' OR key = ?)
|
|
156
|
+
AND (expires IS NULL OR expires > ?)
|
|
157
|
+
ORDER BY key
|
|
158
|
+
`).all(pattern, exactKey, now);
|
|
159
|
+
}
|
|
160
|
+
for (const row of results) yield {
|
|
161
|
+
key: this.#decodeKey(row.key),
|
|
162
|
+
value: this.#decodeValue(row.value)
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
136
166
|
* Creates the table used by the key–value store if it does not already exist.
|
|
137
167
|
* Does nothing if the table already exists.
|
|
138
168
|
*/
|
package/dist/mod.cjs
CHANGED
|
@@ -2,5 +2,7 @@
|
|
|
2
2
|
const { Temporal } = require("@js-temporal/polyfill");
|
|
3
3
|
|
|
4
4
|
const require_kv = require('./kv.cjs');
|
|
5
|
+
const require_mq = require('./mq.cjs');
|
|
5
6
|
|
|
6
|
-
exports.SqliteKvStore = require_kv.SqliteKvStore;
|
|
7
|
+
exports.SqliteKvStore = require_kv.SqliteKvStore;
|
|
8
|
+
exports.SqliteMessageQueue = require_mq.SqliteMessageQueue;
|
package/dist/mod.d.cts
CHANGED
package/dist/mod.d.ts
CHANGED
package/dist/mod.js
CHANGED
package/dist/mq.cjs
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
|
|
2
|
+
const { Temporal } = require("@js-temporal/polyfill");
|
|
3
|
+
|
|
4
|
+
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
5
|
+
const __sqlite = require_rolldown_runtime.__toESM(require("#sqlite"));
|
|
6
|
+
const __logtape_logtape = require_rolldown_runtime.__toESM(require("@logtape/logtape"));
|
|
7
|
+
|
|
8
|
+
//#region src/mq.ts
|
|
9
|
+
const logger = (0, __logtape_logtape.getLogger)([
|
|
10
|
+
"fedify",
|
|
11
|
+
"sqlite",
|
|
12
|
+
"mq"
|
|
13
|
+
]);
|
|
14
|
+
var EnqueueEvent = class extends Event {
|
|
15
|
+
delayMs;
|
|
16
|
+
constructor(delayMs) {
|
|
17
|
+
super("enqueue");
|
|
18
|
+
this.delayMs = delayMs;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* A message queue that uses SQLite as the underlying storage.
|
|
23
|
+
*
|
|
24
|
+
* This implementation is designed for single-node deployments and uses
|
|
25
|
+
* polling to check for new messages. It is not suitable for high-throughput
|
|
26
|
+
* scenarios or distributed environments.
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```ts ignore
|
|
30
|
+
* import { createFederation } from "@fedify/fedify";
|
|
31
|
+
* import { SqliteMessageQueue } from "@fedify/sqlite";
|
|
32
|
+
* import { DatabaseSync } from "node:sqlite";
|
|
33
|
+
*
|
|
34
|
+
* const db = new DatabaseSync(":memory:");
|
|
35
|
+
* const federation = createFederation({
|
|
36
|
+
* // ...
|
|
37
|
+
* queue: new SqliteMessageQueue(db),
|
|
38
|
+
* });
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
var SqliteMessageQueue = class SqliteMessageQueue {
|
|
42
|
+
static #defaultTableName = "fedify_message";
|
|
43
|
+
static #tableNameRegex = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
|
|
44
|
+
static #notifyChannels = /* @__PURE__ */ new Map();
|
|
45
|
+
static #activeInstances = /* @__PURE__ */ new Map();
|
|
46
|
+
static #getNotifyChannel(tableName) {
|
|
47
|
+
let channel = SqliteMessageQueue.#notifyChannels.get(tableName);
|
|
48
|
+
if (channel == null) {
|
|
49
|
+
channel = new EventTarget();
|
|
50
|
+
SqliteMessageQueue.#notifyChannels.set(tableName, channel);
|
|
51
|
+
}
|
|
52
|
+
return channel;
|
|
53
|
+
}
|
|
54
|
+
#db;
|
|
55
|
+
#tableName;
|
|
56
|
+
#pollIntervalMs;
|
|
57
|
+
#instanceId;
|
|
58
|
+
#maxRetries;
|
|
59
|
+
#retryDelayMs;
|
|
60
|
+
#journalMode;
|
|
61
|
+
#initialized;
|
|
62
|
+
/**
|
|
63
|
+
* SQLite message queue does not provide native retry mechanisms.
|
|
64
|
+
*/
|
|
65
|
+
nativeRetrial = false;
|
|
66
|
+
/**
|
|
67
|
+
* Creates a new SQLite message queue.
|
|
68
|
+
* @param db The SQLite database to use. Supports `node:sqlite`, `bun:sqlite`.
|
|
69
|
+
* @param options The options for the message queue.
|
|
70
|
+
*/
|
|
71
|
+
constructor(db, options = {}) {
|
|
72
|
+
this.db = db;
|
|
73
|
+
this.options = options;
|
|
74
|
+
this.#db = new __sqlite.SqliteDatabase(db);
|
|
75
|
+
this.#initialized = options.initialized ?? false;
|
|
76
|
+
this.#tableName = options.tableName ?? SqliteMessageQueue.#defaultTableName;
|
|
77
|
+
this.#instanceId = crypto.randomUUID();
|
|
78
|
+
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
|
|
79
|
+
this.#maxRetries = options.maxRetries ?? 5;
|
|
80
|
+
this.#retryDelayMs = options.retryDelayMs ?? 100;
|
|
81
|
+
this.#journalMode = options.journalMode ?? "WAL";
|
|
82
|
+
if (!SqliteMessageQueue.#tableNameRegex.test(this.#tableName)) throw new Error(`Invalid table name for the message queue: ${this.#tableName}.\
|
|
83
|
+
Only letters, digits, and underscores are allowed.`);
|
|
84
|
+
this.#registerInstance();
|
|
85
|
+
}
|
|
86
|
+
#registerInstance() {
|
|
87
|
+
let instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
|
|
88
|
+
if (instances == null) {
|
|
89
|
+
instances = /* @__PURE__ */ new Set();
|
|
90
|
+
SqliteMessageQueue.#activeInstances.set(this.#tableName, instances);
|
|
91
|
+
}
|
|
92
|
+
instances.add(this.#instanceId);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* {@inheritDoc MessageQueue.enqueue}
|
|
96
|
+
*/
|
|
97
|
+
enqueue(message, options) {
|
|
98
|
+
this.initialize();
|
|
99
|
+
const id = crypto.randomUUID();
|
|
100
|
+
const encodedMessage = this.#encodeMessage(message);
|
|
101
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
102
|
+
const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
|
|
103
|
+
const scheduled = now + delay.total({ unit: "milliseconds" });
|
|
104
|
+
if (options?.delay) logger.debug("Enqueuing a message with a delay of {delay}...", {
|
|
105
|
+
delay,
|
|
106
|
+
message
|
|
107
|
+
});
|
|
108
|
+
else logger.debug("Enqueuing a message...", { message });
|
|
109
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
110
|
+
return this.#retryOnBusy(() => {
|
|
111
|
+
this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
|
|
112
|
+
VALUES (?, ?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled, orderingKey);
|
|
113
|
+
logger.debug("Enqueued a message.", {
|
|
114
|
+
message,
|
|
115
|
+
orderingKey
|
|
116
|
+
});
|
|
117
|
+
const delayMs = delay.total("millisecond");
|
|
118
|
+
SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* {@inheritDoc MessageQueue.enqueueMany}
|
|
123
|
+
*/
|
|
124
|
+
enqueueMany(messages, options) {
|
|
125
|
+
if (messages.length === 0) return Promise.resolve();
|
|
126
|
+
this.initialize();
|
|
127
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
128
|
+
const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
|
|
129
|
+
const scheduled = now + delay.total({ unit: "milliseconds" });
|
|
130
|
+
if (options?.delay) logger.debug("Enqueuing messages with a delay of {delay}...", {
|
|
131
|
+
delay,
|
|
132
|
+
messages
|
|
133
|
+
});
|
|
134
|
+
else logger.debug("Enqueuing messages...", { messages });
|
|
135
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
136
|
+
return this.#withTransactionRetries(() => {
|
|
137
|
+
const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
|
|
138
|
+
VALUES (?, ?, ?, ?, ?)`);
|
|
139
|
+
for (const message of messages) {
|
|
140
|
+
const id = crypto.randomUUID();
|
|
141
|
+
const encodedMessage = this.#encodeMessage(message);
|
|
142
|
+
stmt.run(id, encodedMessage, now, scheduled, orderingKey);
|
|
143
|
+
}
|
|
144
|
+
logger.debug("Enqueued messages.", { messages });
|
|
145
|
+
const delayMs = delay.total("millisecond");
|
|
146
|
+
SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
|
|
147
|
+
}).catch((error) => {
|
|
148
|
+
logger.error("Failed to enqueue messages to table {tableName}: {error}", {
|
|
149
|
+
tableName: this.#tableName,
|
|
150
|
+
messageCount: messages.length,
|
|
151
|
+
error
|
|
152
|
+
});
|
|
153
|
+
throw error;
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* {@inheritDoc MessageQueue.listen}
|
|
158
|
+
*/
|
|
159
|
+
async listen(handler, options) {
|
|
160
|
+
this.initialize();
|
|
161
|
+
const { signal } = options ?? {};
|
|
162
|
+
logger.debug("Starting to listen for messages on table {tableName}...", { tableName: this.#tableName });
|
|
163
|
+
const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
|
|
164
|
+
const timeouts = /* @__PURE__ */ new Set();
|
|
165
|
+
const lockTableName = `${this.#tableName}_locks`;
|
|
166
|
+
const poll = async () => {
|
|
167
|
+
while (signal == null || !signal.aborted) {
|
|
168
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
169
|
+
const result = await this.#withTransactionRetries(() => {
|
|
170
|
+
return this.#db.prepare(`DELETE FROM "${this.#tableName}"
|
|
171
|
+
WHERE id = (
|
|
172
|
+
SELECT id FROM "${this.#tableName}"
|
|
173
|
+
WHERE scheduled <= ?
|
|
174
|
+
AND (ordering_key IS NULL
|
|
175
|
+
OR ordering_key NOT IN (SELECT ordering_key FROM "${lockTableName}"))
|
|
176
|
+
ORDER BY scheduled
|
|
177
|
+
LIMIT 1
|
|
178
|
+
)
|
|
179
|
+
RETURNING id, message, ordering_key`).get(now);
|
|
180
|
+
});
|
|
181
|
+
if (result) {
|
|
182
|
+
const message = this.#decodeMessage(result.message);
|
|
183
|
+
const orderingKey = result.ordering_key;
|
|
184
|
+
if (orderingKey != null) await this.#retryOnBusy(() => {
|
|
185
|
+
this.#db.prepare(`INSERT OR IGNORE INTO "${lockTableName}" (ordering_key) VALUES (?)`).run(orderingKey);
|
|
186
|
+
});
|
|
187
|
+
logger.debug("Processing message {id}...", {
|
|
188
|
+
id: result.id,
|
|
189
|
+
message,
|
|
190
|
+
orderingKey
|
|
191
|
+
});
|
|
192
|
+
try {
|
|
193
|
+
await handler(message);
|
|
194
|
+
logger.debug("Processed message {id}.", { id: result.id });
|
|
195
|
+
} catch (error) {
|
|
196
|
+
logger.error("Failed to process message {id} from table {tableName}: {error}", {
|
|
197
|
+
id: result.id,
|
|
198
|
+
tableName: this.#tableName,
|
|
199
|
+
message,
|
|
200
|
+
error
|
|
201
|
+
});
|
|
202
|
+
} finally {
|
|
203
|
+
if (orderingKey != null) await this.#retryOnBusy(() => {
|
|
204
|
+
this.#db.prepare(`DELETE FROM "${lockTableName}" WHERE ordering_key = ?`).run(orderingKey);
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
};
|
|
212
|
+
const onEnqueue = (event) => {
|
|
213
|
+
const delayMs = event.delayMs;
|
|
214
|
+
if (delayMs < 1) poll();
|
|
215
|
+
else timeouts.add(setTimeout(poll, delayMs));
|
|
216
|
+
};
|
|
217
|
+
channel.addEventListener("enqueue", onEnqueue);
|
|
218
|
+
signal?.addEventListener("abort", () => {
|
|
219
|
+
channel.removeEventListener("enqueue", onEnqueue);
|
|
220
|
+
for (const timeout of timeouts) clearTimeout(timeout);
|
|
221
|
+
});
|
|
222
|
+
await poll();
|
|
223
|
+
while (signal == null || !signal.aborted) {
|
|
224
|
+
let timeout;
|
|
225
|
+
await new Promise((resolve) => {
|
|
226
|
+
const onAbort = () => {
|
|
227
|
+
signal?.removeEventListener("abort", onAbort);
|
|
228
|
+
resolve(void 0);
|
|
229
|
+
};
|
|
230
|
+
signal?.addEventListener("abort", onAbort);
|
|
231
|
+
timeout = setTimeout(() => {
|
|
232
|
+
signal?.removeEventListener("abort", onAbort);
|
|
233
|
+
resolve(0);
|
|
234
|
+
}, this.#pollIntervalMs);
|
|
235
|
+
timeouts.add(timeout);
|
|
236
|
+
});
|
|
237
|
+
if (timeout != null) timeouts.delete(timeout);
|
|
238
|
+
await poll();
|
|
239
|
+
}
|
|
240
|
+
logger.debug("Stopped listening for messages on table {tableName}.", { tableName: this.#tableName });
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Creates the message queue table if it does not already exist.
|
|
244
|
+
* Does nothing if the table already exists.
|
|
245
|
+
*
|
|
246
|
+
* This method also configures the SQLite journal mode for better concurrency.
|
|
247
|
+
* WAL (Write-Ahead Logging) mode is enabled by default to improve
|
|
248
|
+
* concurrent access in multi-process environments.
|
|
249
|
+
*/
|
|
250
|
+
initialize() {
|
|
251
|
+
if (this.#initialized) return;
|
|
252
|
+
logger.debug("Initializing the message queue table {tableName}...", { tableName: this.#tableName });
|
|
253
|
+
this.#db.exec(`PRAGMA journal_mode=${this.#journalMode}`);
|
|
254
|
+
this.#withTransaction(() => {
|
|
255
|
+
this.#db.exec(`
|
|
256
|
+
CREATE TABLE IF NOT EXISTS "${this.#tableName}" (
|
|
257
|
+
id TEXT PRIMARY KEY,
|
|
258
|
+
message TEXT NOT NULL,
|
|
259
|
+
created INTEGER NOT NULL,
|
|
260
|
+
scheduled INTEGER NOT NULL,
|
|
261
|
+
ordering_key TEXT
|
|
262
|
+
)
|
|
263
|
+
`);
|
|
264
|
+
this.#db.exec(`
|
|
265
|
+
CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
|
|
266
|
+
ON "${this.#tableName}" (scheduled)
|
|
267
|
+
`);
|
|
268
|
+
this.#db.exec(`
|
|
269
|
+
CREATE TABLE IF NOT EXISTS "${this.#tableName}_locks" (
|
|
270
|
+
ordering_key TEXT PRIMARY KEY
|
|
271
|
+
)
|
|
272
|
+
`);
|
|
273
|
+
});
|
|
274
|
+
this.#initialized = true;
|
|
275
|
+
logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Drops the tables used by the message queue. Does nothing if the tables
|
|
279
|
+
* do not exist.
|
|
280
|
+
*/
|
|
281
|
+
drop() {
|
|
282
|
+
this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
|
|
283
|
+
this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}_locks"`);
|
|
284
|
+
this.#initialized = false;
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Closes the database connection.
|
|
288
|
+
*/
|
|
289
|
+
[Symbol.dispose]() {
|
|
290
|
+
try {
|
|
291
|
+
this.#db.close();
|
|
292
|
+
this.#unregisterInstance();
|
|
293
|
+
} catch (error) {
|
|
294
|
+
logger.error("Failed to close the database connection for table {tableName}: {error}", {
|
|
295
|
+
tableName: this.#tableName,
|
|
296
|
+
error
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
#unregisterInstance() {
|
|
301
|
+
const instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
|
|
302
|
+
if (instances == null) return;
|
|
303
|
+
instances.delete(this.#instanceId);
|
|
304
|
+
if (instances.size === 0) {
|
|
305
|
+
SqliteMessageQueue.#activeInstances.delete(this.#tableName);
|
|
306
|
+
SqliteMessageQueue.#notifyChannels.delete(this.#tableName);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Checks if an error is a SQLITE_BUSY error or transaction conflict.
|
|
311
|
+
* Handles different error formats from node:sqlite and bun:sqlite.
|
|
312
|
+
*/
|
|
313
|
+
#isBusyError(error) {
|
|
314
|
+
if (!(error instanceof Error)) return false;
|
|
315
|
+
if (error.message.includes("SQLITE_BUSY") || error.message.includes("database is locked") || error.message.includes("transaction within a transaction")) return true;
|
|
316
|
+
const errorWithCode = error;
|
|
317
|
+
if (errorWithCode.code === "SQLITE_BUSY") return true;
|
|
318
|
+
const errorWithErrno = error;
|
|
319
|
+
if (errorWithErrno.errno === 5) return true;
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Retries a database operation with exponential backoff on SQLITE_BUSY errors.
|
|
324
|
+
*/
|
|
325
|
+
async #retryOnBusy(operation) {
|
|
326
|
+
let lastError;
|
|
327
|
+
for (let attempt = 0; attempt <= this.#maxRetries; attempt++) try {
|
|
328
|
+
return operation();
|
|
329
|
+
} catch (error) {
|
|
330
|
+
lastError = error;
|
|
331
|
+
if (!this.#isBusyError(error)) {
|
|
332
|
+
logger.error("Database operation failed on table {tableName}: {error}", {
|
|
333
|
+
tableName: this.#tableName,
|
|
334
|
+
error
|
|
335
|
+
});
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
338
|
+
if (attempt === this.#maxRetries) {
|
|
339
|
+
logger.error("Max retries ({maxRetries}) reached for SQLITE_BUSY error on table {tableName}.", {
|
|
340
|
+
maxRetries: this.#maxRetries,
|
|
341
|
+
tableName: this.#tableName,
|
|
342
|
+
error
|
|
343
|
+
});
|
|
344
|
+
throw error;
|
|
345
|
+
}
|
|
346
|
+
const delayMs = this.#retryDelayMs * Math.pow(2, attempt);
|
|
347
|
+
logger.debug("SQLITE_BUSY error on table {tableName}, retrying in {delayMs}ms (attempt {attempt}/{maxRetries})...", {
|
|
348
|
+
tableName: this.#tableName,
|
|
349
|
+
delayMs,
|
|
350
|
+
attempt: attempt + 1,
|
|
351
|
+
maxRetries: this.#maxRetries
|
|
352
|
+
});
|
|
353
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
354
|
+
}
|
|
355
|
+
throw lastError;
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Executes a database operation within a transaction.
|
|
359
|
+
* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
|
|
360
|
+
*/
|
|
361
|
+
#withTransaction(operation) {
|
|
362
|
+
let transactionStarted = false;
|
|
363
|
+
try {
|
|
364
|
+
this.#db.exec("BEGIN IMMEDIATE");
|
|
365
|
+
transactionStarted = true;
|
|
366
|
+
const result = operation();
|
|
367
|
+
this.#db.exec("COMMIT");
|
|
368
|
+
return result;
|
|
369
|
+
} catch (error) {
|
|
370
|
+
if (transactionStarted) try {
|
|
371
|
+
this.#db.exec("ROLLBACK");
|
|
372
|
+
} catch {}
|
|
373
|
+
throw error;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Executes a database operation within a transaction with retry logic.
|
|
378
|
+
* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
|
|
379
|
+
* Retries on SQLITE_BUSY errors with exponential backoff.
|
|
380
|
+
*/
|
|
381
|
+
async #withTransactionRetries(operation) {
|
|
382
|
+
return await this.#retryOnBusy(() => this.#withTransaction(operation));
|
|
383
|
+
}
|
|
384
|
+
#encodeMessage(message) {
|
|
385
|
+
return JSON.stringify(message);
|
|
386
|
+
}
|
|
387
|
+
#decodeMessage(message) {
|
|
388
|
+
return JSON.parse(message);
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
//#endregion
|
|
393
|
+
exports.SqliteMessageQueue = SqliteMessageQueue;
|