@lordbex/thelounge 4.4.3-blowfish → 4.5.0-blowfish-pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -7
- package/dist/defaults/config.js +31 -2
- package/dist/package.json +94 -91
- package/dist/server/client.js +188 -194
- package/dist/server/clientManager.js +75 -72
- package/dist/server/command-line/index.js +44 -43
- package/dist/server/command-line/install.js +37 -70
- package/dist/server/command-line/outdated.js +12 -17
- package/dist/server/command-line/start.js +25 -26
- package/dist/server/command-line/storage.js +26 -31
- package/dist/server/command-line/uninstall.js +16 -23
- package/dist/server/command-line/upgrade.js +20 -26
- package/dist/server/command-line/users/add.js +33 -40
- package/dist/server/command-line/users/edit.js +18 -24
- package/dist/server/command-line/users/index.js +12 -16
- package/dist/server/command-line/users/list.js +11 -39
- package/dist/server/command-line/users/remove.js +16 -22
- package/dist/server/command-line/users/reset.js +34 -35
- package/dist/server/command-line/utils.js +231 -87
- package/dist/server/config.js +61 -52
- package/dist/server/helper.js +29 -28
- package/dist/server/identification.js +39 -34
- package/dist/server/index.js +1 -3
- package/dist/server/log.js +19 -16
- package/dist/server/models/chan.js +36 -33
- package/dist/server/models/msg.js +15 -19
- package/dist/server/models/network.js +102 -104
- package/dist/server/models/prefix.js +4 -7
- package/dist/server/models/user.js +5 -10
- package/dist/server/path-helper.js +8 -0
- package/dist/server/plugins/auth/ldap.js +177 -112
- package/dist/server/plugins/auth/local.js +10 -15
- package/dist/server/plugins/auth.js +6 -35
- package/dist/server/plugins/changelog.js +30 -27
- package/dist/server/plugins/clientCertificate.js +33 -37
- package/dist/server/plugins/dev-server.js +15 -21
- package/dist/server/plugins/inputs/action.js +9 -14
- package/dist/server/plugins/inputs/away.js +1 -3
- package/dist/server/plugins/inputs/ban.js +9 -14
- package/dist/server/plugins/inputs/blow.js +9 -14
- package/dist/server/plugins/inputs/connect.js +5 -10
- package/dist/server/plugins/inputs/ctcp.js +7 -12
- package/dist/server/plugins/inputs/disconnect.js +1 -3
- package/dist/server/plugins/inputs/ignore.js +23 -29
- package/dist/server/plugins/inputs/ignorelist.js +12 -18
- package/dist/server/plugins/inputs/index.js +8 -34
- package/dist/server/plugins/inputs/invite.js +7 -12
- package/dist/server/plugins/inputs/kick.js +7 -12
- package/dist/server/plugins/inputs/kill.js +1 -3
- package/dist/server/plugins/inputs/list.js +1 -3
- package/dist/server/plugins/inputs/mode.js +10 -15
- package/dist/server/plugins/inputs/msg.js +13 -18
- package/dist/server/plugins/inputs/mute.js +9 -15
- package/dist/server/plugins/inputs/nick.js +9 -14
- package/dist/server/plugins/inputs/notice.js +5 -7
- package/dist/server/plugins/inputs/part.js +11 -16
- package/dist/server/plugins/inputs/quit.js +7 -13
- package/dist/server/plugins/inputs/rainbow.js +55 -0
- package/dist/server/plugins/inputs/raw.js +1 -3
- package/dist/server/plugins/inputs/rejoin.js +7 -12
- package/dist/server/plugins/inputs/topic.js +7 -12
- package/dist/server/plugins/inputs/whois.js +1 -3
- package/dist/server/plugins/irc-events/away.js +14 -20
- package/dist/server/plugins/irc-events/cap.js +16 -22
- package/dist/server/plugins/irc-events/chghost.js +14 -13
- package/dist/server/plugins/irc-events/connection.js +61 -63
- package/dist/server/plugins/irc-events/ctcp.js +22 -28
- package/dist/server/plugins/irc-events/error.js +20 -26
- package/dist/server/plugins/irc-events/help.js +7 -13
- package/dist/server/plugins/irc-events/info.js +7 -13
- package/dist/server/plugins/irc-events/invite.js +7 -13
- package/dist/server/plugins/irc-events/join.js +30 -27
- package/dist/server/plugins/irc-events/kick.js +21 -17
- package/dist/server/plugins/irc-events/link.js +122 -109
- package/dist/server/plugins/irc-events/list.js +23 -26
- package/dist/server/plugins/irc-events/message.js +46 -52
- package/dist/server/plugins/irc-events/mode.js +66 -63
- package/dist/server/plugins/irc-events/modelist.js +29 -35
- package/dist/server/plugins/irc-events/motd.js +10 -16
- package/dist/server/plugins/irc-events/names.js +3 -6
- package/dist/server/plugins/irc-events/nick.js +26 -23
- package/dist/server/plugins/irc-events/part.js +19 -15
- package/dist/server/plugins/irc-events/quit.js +17 -14
- package/dist/server/plugins/irc-events/sasl.js +9 -15
- package/dist/server/plugins/irc-events/spgroups.js +38 -0
- package/dist/server/plugins/irc-events/spjoin.js +52 -0
- package/dist/server/plugins/irc-events/topic.js +12 -18
- package/dist/server/plugins/irc-events/unhandled.js +12 -12
- package/dist/server/plugins/irc-events/welcome.js +7 -13
- package/dist/server/plugins/irc-events/whois.js +20 -24
- package/dist/server/plugins/massEventAggregator.js +214 -0
- package/dist/server/plugins/messageStorage/sqlite.js +322 -141
- package/dist/server/plugins/messageStorage/text.js +21 -26
- package/dist/server/plugins/packages/index.js +105 -74
- package/dist/server/plugins/packages/publicClient.js +7 -16
- package/dist/server/plugins/packages/themes.js +11 -16
- package/dist/server/plugins/storage.js +28 -33
- package/dist/server/plugins/sts.js +12 -17
- package/dist/server/plugins/uploader.js +40 -43
- package/dist/server/plugins/webpush.js +23 -51
- package/dist/server/server.js +318 -271
- package/dist/server/storageCleaner.js +29 -37
- package/dist/server/utils/fish.js +349 -389
- package/dist/shared/irc.js +3 -6
- package/dist/shared/linkify.js +7 -14
- package/dist/shared/types/chan.js +6 -9
- package/dist/shared/types/changelog.js +1 -2
- package/dist/shared/types/config.js +1 -2
- package/dist/shared/types/mention.js +1 -2
- package/dist/shared/types/msg.js +3 -5
- package/dist/shared/types/network.js +1 -2
- package/dist/shared/types/storage.js +1 -2
- package/dist/shared/types/user.js +1 -2
- package/index.js +14 -10
- package/package.json +94 -91
- package/public/css/style.css +9 -6
- package/public/css/style.css.map +1 -1
- package/public/fonts/font-awesome/fa-brands-400.ttf +0 -0
- package/public/fonts/font-awesome/fa-brands-400.woff2 +0 -0
- package/public/fonts/font-awesome/fa-duotone-900.ttf +0 -0
- package/public/fonts/font-awesome/fa-duotone-900.woff2 +0 -0
- package/public/fonts/font-awesome/fa-light-300.ttf +0 -0
- package/public/fonts/font-awesome/fa-light-300.woff2 +0 -0
- package/public/fonts/font-awesome/fa-regular-400.ttf +0 -0
- package/public/fonts/font-awesome/fa-regular-400.woff2 +0 -0
- package/public/fonts/font-awesome/fa-solid-900.ttf +0 -0
- package/public/fonts/font-awesome/fa-solid-900.woff2 +0 -0
- package/public/fonts/font-awesome/fa-thin-100.ttf +0 -0
- package/public/fonts/font-awesome/fa-thin-100.woff2 +0 -0
- package/public/fonts/font-awesome/fa-v4compatibility.ttf +0 -0
- package/public/fonts/font-awesome/fa-v4compatibility.woff2 +0 -0
- package/public/js/bundle.js +1 -1
- package/public/js/bundle.js.map +1 -1
- package/public/js/bundle.vendor.js +1 -1
- package/public/js/bundle.vendor.js.LICENSE.txt +24 -6
- package/public/js/bundle.vendor.js.map +1 -1
- package/public/js/loading-error-handlers.js +1 -1
- package/public/service-worker.js +1 -1
- package/public/themes/default.css +1 -1
- package/public/themes/morning.css +1 -1
- package/dist/webpack.config.js +0 -224
|
@@ -1,40 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
|
-
const config_1 = __importDefault(require("../../config"));
|
|
11
|
-
const msg_1 = __importDefault(require("../../models/msg"));
|
|
12
|
-
const helper_1 = __importDefault(require("../../helper"));
|
|
13
|
-
// TODO; type
|
|
14
|
-
let sqlite3;
|
|
15
|
-
try {
|
|
16
|
-
sqlite3 = require("sqlite3");
|
|
17
|
-
}
|
|
18
|
-
catch (e) {
|
|
19
|
-
config_1.default.values.messageStorage = config_1.default.values.messageStorage.filter((item) => item !== "sqlite");
|
|
20
|
-
log_1.default.error("Unable to load sqlite3 module. See https://github.com/mapbox/node-sqlite3/wiki/Binaries");
|
|
21
|
-
}
|
|
22
|
-
exports.currentSchemaVersion = 1703322560448; // use `new Date().getTime()`
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import log from "../../log.js";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import fs from "fs/promises";
|
|
5
|
+
import Config from "../../config.js";
|
|
6
|
+
import Msg from "../../models/msg.js";
|
|
7
|
+
import Helper from "../../helper.js";
|
|
8
|
+
export const currentSchemaVersion = 1703322560448; // use `new Date().getTime()`
|
|
23
9
|
// Desired schema, adapt to the newest version and add migrations to the array below
|
|
24
10
|
const schema = [
|
|
25
11
|
"CREATE TABLE options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))",
|
|
26
12
|
"CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)",
|
|
27
13
|
`CREATE TABLE migrations (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
14
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
15
|
+
version INTEGER NOT NULL UNIQUE,
|
|
16
|
+
rollback_forbidden INTEGER DEFAULT 0 NOT NULL
|
|
17
|
+
)`,
|
|
32
18
|
`CREATE TABLE rollback_steps (
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
19
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
20
|
+
migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
|
|
21
|
+
step INTEGER NOT NULL,
|
|
22
|
+
statement TEXT NOT NULL
|
|
23
|
+
)`,
|
|
38
24
|
"CREATE INDEX network_channel ON messages (network, channel)",
|
|
39
25
|
"CREATE INDEX time ON messages (time)",
|
|
40
26
|
"CREATE INDEX msg_type_idx on messages (type)", // needed for efficient storageCleaner queries
|
|
@@ -42,7 +28,7 @@ const schema = [
|
|
|
42
28
|
// the migrations will be executed in an exclusive transaction as a whole
|
|
43
29
|
// add new migrations to the end, with the version being the new 'currentSchemaVersion'
|
|
44
30
|
// write a corresponding down migration into rollbacks
|
|
45
|
-
|
|
31
|
+
export const migrations = [
|
|
46
32
|
{
|
|
47
33
|
version: 1672236339873,
|
|
48
34
|
stmts: [
|
|
@@ -58,16 +44,16 @@ exports.migrations = [
|
|
|
58
44
|
version: 1679743888000,
|
|
59
45
|
stmts: [
|
|
60
46
|
`CREATE TABLE IF NOT EXISTS migrations (
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
47
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
48
|
+
version INTEGER NOT NULL UNIQUE,
|
|
49
|
+
rollback_forbidden INTEGER DEFAULT 0 NOT NULL
|
|
50
|
+
)`,
|
|
65
51
|
`CREATE TABLE IF NOT EXISTS rollback_steps (
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
52
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
53
|
+
migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
|
|
54
|
+
step INTEGER NOT NULL,
|
|
55
|
+
statement TEXT NOT NULL
|
|
56
|
+
)`,
|
|
71
57
|
],
|
|
72
58
|
},
|
|
73
59
|
{
|
|
@@ -77,7 +63,7 @@ exports.migrations = [
|
|
|
77
63
|
];
|
|
78
64
|
// down migrations need to restore the state of the prior version.
|
|
79
65
|
// rollback can be disallowed by adding rollback_forbidden: true to it
|
|
80
|
-
|
|
66
|
+
export const rollbacks = [
|
|
81
67
|
{
|
|
82
68
|
version: 1672236339873,
|
|
83
69
|
stmts: [], // changes aren't visible, left empty on purpose
|
|
@@ -105,31 +91,39 @@ class SqliteMessageStorage {
|
|
|
105
91
|
database;
|
|
106
92
|
initDone;
|
|
107
93
|
userName;
|
|
94
|
+
// Message batching for improved write performance
|
|
95
|
+
batchQueue = [];
|
|
96
|
+
batchSize = 50; // Flush after 50 messages
|
|
97
|
+
batchTimeout = 1000; // Flush after 1 second
|
|
98
|
+
batchTimer = null;
|
|
99
|
+
insertStmt = null;
|
|
108
100
|
constructor(userName) {
|
|
109
101
|
this.userName = userName;
|
|
110
102
|
this.isEnabled = false;
|
|
111
103
|
this.initDone = new Deferred();
|
|
112
104
|
}
|
|
113
105
|
async _enable(connection_string) {
|
|
114
|
-
this.database = new sqlite3.Database(connection_string);
|
|
115
106
|
try {
|
|
107
|
+
this.database = new Database(connection_string);
|
|
116
108
|
await this.run_pragmas(); // must be done outside of a transaction
|
|
117
109
|
await this.run_migrations();
|
|
110
|
+
// Prepare insert statement for batching
|
|
111
|
+
this.insertStmt = this.database.prepare("INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)");
|
|
118
112
|
}
|
|
119
113
|
catch (e) {
|
|
120
114
|
this.isEnabled = false;
|
|
121
|
-
throw
|
|
115
|
+
throw Helper.catch_to_error("Migration failed", e);
|
|
122
116
|
}
|
|
123
117
|
this.isEnabled = true;
|
|
124
118
|
}
|
|
125
119
|
async enable() {
|
|
126
|
-
const logsPath =
|
|
127
|
-
const sqlitePath =
|
|
120
|
+
const logsPath = Config.getUserLogsPath();
|
|
121
|
+
const sqlitePath = path.join(logsPath, `${this.userName}.sqlite3`);
|
|
128
122
|
try {
|
|
129
|
-
await
|
|
123
|
+
await fs.mkdir(logsPath, { recursive: true });
|
|
130
124
|
}
|
|
131
125
|
catch (e) {
|
|
132
|
-
throw
|
|
126
|
+
throw Helper.catch_to_error("Unable to create logs directory", e);
|
|
133
127
|
}
|
|
134
128
|
try {
|
|
135
129
|
await this._enable(sqlitePath);
|
|
@@ -142,7 +136,7 @@ class SqliteMessageStorage {
|
|
|
142
136
|
for (const stmt of schema) {
|
|
143
137
|
await this.serialize_run(stmt);
|
|
144
138
|
}
|
|
145
|
-
await this.serialize_run("INSERT INTO options (name, value) VALUES ('schema_version', ?)",
|
|
139
|
+
await this.serialize_run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", currentSchemaVersion.toString());
|
|
146
140
|
}
|
|
147
141
|
async current_version() {
|
|
148
142
|
const have_options = await this.serialize_get("select 1 from sqlite_master where type = 'table' and name = 'options'");
|
|
@@ -159,10 +153,10 @@ class SqliteMessageStorage {
|
|
|
159
153
|
return storedSchemaVersion;
|
|
160
154
|
}
|
|
161
155
|
async update_version_in_db() {
|
|
162
|
-
return this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'",
|
|
156
|
+
return this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'", currentSchemaVersion.toString());
|
|
163
157
|
}
|
|
164
158
|
async _run_migrations(dbVersion) {
|
|
165
|
-
|
|
159
|
+
log.info(`sqlite messages schema version is out of date (${dbVersion} < ${currentSchemaVersion}). Running migrations.`);
|
|
166
160
|
const to_execute = necessaryMigrations(dbVersion);
|
|
167
161
|
for (const stmt of to_execute.map((m) => m.stmts).flat()) {
|
|
168
162
|
await this.serialize_run(stmt);
|
|
@@ -174,10 +168,10 @@ class SqliteMessageStorage {
|
|
|
174
168
|
}
|
|
175
169
|
async run_migrations() {
|
|
176
170
|
const version = await this.current_version();
|
|
177
|
-
if (version >
|
|
178
|
-
throw `sqlite messages schema version is higher than expected (${version} > ${
|
|
171
|
+
if (version > currentSchemaVersion) {
|
|
172
|
+
throw new Error(`sqlite messages schema version is higher than expected (${version} > ${currentSchemaVersion}). Is NexusIRC out of date?`);
|
|
179
173
|
}
|
|
180
|
-
else if (version ===
|
|
174
|
+
else if (version === currentSchemaVersion) {
|
|
181
175
|
return; // nothing to do
|
|
182
176
|
}
|
|
183
177
|
await this.serialize_run("BEGIN EXCLUSIVE TRANSACTION");
|
|
@@ -205,37 +199,82 @@ class SqliteMessageStorage {
|
|
|
205
199
|
if (!this.isEnabled) {
|
|
206
200
|
return;
|
|
207
201
|
}
|
|
202
|
+
// Flush any pending batched messages
|
|
203
|
+
await this.flushBatch();
|
|
204
|
+
// Clear batch timer
|
|
205
|
+
if (this.batchTimer) {
|
|
206
|
+
clearTimeout(this.batchTimer);
|
|
207
|
+
this.batchTimer = null;
|
|
208
|
+
}
|
|
208
209
|
this.isEnabled = false;
|
|
209
|
-
|
|
210
|
-
this.database.close(
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
210
|
+
try {
|
|
211
|
+
this.database.close();
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
throw new Error(`Failed to close sqlite database: ${err instanceof Error ? err.message : String(err)}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Flush batched messages to database using a transaction
|
|
219
|
+
*/
|
|
220
|
+
async flushBatch() {
|
|
221
|
+
if (this.batchQueue.length === 0) {
|
|
222
|
+
return Promise.resolve();
|
|
223
|
+
}
|
|
224
|
+
if (!this.insertStmt) {
|
|
225
|
+
log.error("Cannot flush batch: insert statement not prepared");
|
|
226
|
+
return Promise.resolve();
|
|
227
|
+
}
|
|
228
|
+
const messages = this.batchQueue.splice(0); // Take all messages and clear queue
|
|
229
|
+
try {
|
|
230
|
+
// Use transaction for batch insert (much faster than individual inserts)
|
|
231
|
+
const transaction = this.database.transaction((msgs) => {
|
|
232
|
+
for (const msg of msgs) {
|
|
233
|
+
this.insertStmt.run(msg.network, msg.channel, msg.time, msg.type, msg.msg);
|
|
214
234
|
}
|
|
215
|
-
resolve();
|
|
216
235
|
});
|
|
217
|
-
|
|
236
|
+
transaction(messages);
|
|
237
|
+
}
|
|
238
|
+
catch (err) {
|
|
239
|
+
log.error(`Failed to flush message batch: ${err instanceof Error ? err.message : String(err)}`);
|
|
240
|
+
// Re-add messages to queue on failure
|
|
241
|
+
this.batchQueue.unshift(...messages);
|
|
242
|
+
throw err;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Schedule a batch flush
|
|
247
|
+
*/
|
|
248
|
+
scheduleBatchFlush() {
|
|
249
|
+
if (this.batchTimer) {
|
|
250
|
+
return; // Timer already scheduled
|
|
251
|
+
}
|
|
252
|
+
this.batchTimer = setTimeout(() => {
|
|
253
|
+
this.batchTimer = null;
|
|
254
|
+
this.flushBatch().catch((err) => log.error(`Batch flush error: ${err}`));
|
|
255
|
+
}, this.batchTimeout);
|
|
218
256
|
}
|
|
219
257
|
async fetch_rollbacks(since_version) {
|
|
220
258
|
const res = await this.serialize_fetchall(`select version, rollback_forbidden, statement
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
259
|
+
from rollback_steps
|
|
260
|
+
join migrations on migrations.id=rollback_steps.migration_id
|
|
261
|
+
where version > ?
|
|
262
|
+
order by version desc, step asc`, since_version);
|
|
225
263
|
const result = [];
|
|
226
264
|
// convert to Rollback[]
|
|
227
265
|
// requires ordering in the sql statement
|
|
228
266
|
for (const raw of res) {
|
|
229
267
|
const last = result.at(-1);
|
|
230
|
-
|
|
268
|
+
const r = raw;
|
|
269
|
+
if (!last || r.version !== last.version) {
|
|
231
270
|
result.push({
|
|
232
|
-
version:
|
|
233
|
-
rollback_forbidden: Boolean(
|
|
234
|
-
stmts: [
|
|
271
|
+
version: r.version,
|
|
272
|
+
rollback_forbidden: Boolean(r.rollback_forbidden),
|
|
273
|
+
stmts: [r.statement],
|
|
235
274
|
});
|
|
236
275
|
}
|
|
237
276
|
else {
|
|
238
|
-
last.stmts.push(
|
|
277
|
+
last.stmts.push(r.statement);
|
|
239
278
|
}
|
|
240
279
|
}
|
|
241
280
|
return result;
|
|
@@ -278,21 +317,21 @@ class SqliteMessageStorage {
|
|
|
278
317
|
return new_version;
|
|
279
318
|
}
|
|
280
319
|
async downgrade() {
|
|
281
|
-
const res = await this.downgrade_to(
|
|
320
|
+
const res = await this.downgrade_to(currentSchemaVersion);
|
|
282
321
|
return res;
|
|
283
322
|
}
|
|
284
323
|
async insert_rollback_since(version) {
|
|
285
324
|
const missing = newRollbacks(version);
|
|
286
325
|
for (const rollback of missing) {
|
|
287
326
|
const migration = await this.serialize_get(`insert into migrations
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
327
|
+
(version, rollback_forbidden)
|
|
328
|
+
values (?, ?)
|
|
329
|
+
returning id`, rollback.version, rollback.rollback_forbidden || 0);
|
|
291
330
|
for (const stmt of rollback.stmts) {
|
|
292
331
|
let step = 0;
|
|
293
332
|
await this.serialize_run(`insert into rollback_steps
|
|
294
|
-
|
|
295
|
-
|
|
333
|
+
(migration_id, step, statement)
|
|
334
|
+
values (?, ?, ?)`, migration.id, step, stmt);
|
|
296
335
|
step++;
|
|
297
336
|
}
|
|
298
337
|
}
|
|
@@ -303,15 +342,30 @@ class SqliteMessageStorage {
|
|
|
303
342
|
return;
|
|
304
343
|
}
|
|
305
344
|
const clonedMsg = Object.keys(msg).reduce((newMsg, prop) => {
|
|
306
|
-
// id is regenerated when messages are retrieved
|
|
307
345
|
// previews are not stored because storage is cleared on lounge restart
|
|
308
346
|
// type and time are stored in a separate column
|
|
309
|
-
|
|
347
|
+
// id IS now stored so it can be retrieved consistently
|
|
348
|
+
if (prop !== "previews" && prop !== "type" && prop !== "time") {
|
|
310
349
|
newMsg[prop] = msg[prop];
|
|
311
350
|
}
|
|
312
351
|
return newMsg;
|
|
313
352
|
}, {});
|
|
314
|
-
|
|
353
|
+
// Add to batch queue instead of immediate insert
|
|
354
|
+
this.batchQueue.push({
|
|
355
|
+
network: network.uuid,
|
|
356
|
+
channel: channel.name.toLowerCase(),
|
|
357
|
+
time: msg.time.getTime(),
|
|
358
|
+
type: msg.type,
|
|
359
|
+
msg: JSON.stringify(clonedMsg),
|
|
360
|
+
});
|
|
361
|
+
// Flush batch if it reaches the size limit
|
|
362
|
+
if (this.batchQueue.length >= this.batchSize) {
|
|
363
|
+
await this.flushBatch();
|
|
364
|
+
}
|
|
365
|
+
else {
|
|
366
|
+
// Schedule flush after timeout
|
|
367
|
+
this.scheduleBatchFlush();
|
|
368
|
+
}
|
|
315
369
|
}
|
|
316
370
|
async deleteChannel(network, channel) {
|
|
317
371
|
await this.initDone.promise;
|
|
@@ -322,20 +376,42 @@ class SqliteMessageStorage {
|
|
|
322
376
|
}
|
|
323
377
|
async getMessages(network, channel, nextID) {
|
|
324
378
|
await this.initDone.promise;
|
|
325
|
-
if (!this.isEnabled ||
|
|
379
|
+
if (!this.isEnabled || Config.values.maxHistory === 0) {
|
|
326
380
|
return [];
|
|
327
381
|
}
|
|
382
|
+
// Flush any pending batched writes before reading
|
|
383
|
+
await this.flushBatch();
|
|
328
384
|
// If unlimited history is specified, load 100k messages
|
|
329
|
-
const limit =
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
const
|
|
336
|
-
|
|
385
|
+
const limit = Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory;
|
|
386
|
+
// Select id from SQLite to use as the canonical message ID
|
|
387
|
+
const rows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", network.uuid, channel.name.toLowerCase(), limit);
|
|
388
|
+
// Track max ID so we can update client.idMsg
|
|
389
|
+
let maxId = 0;
|
|
390
|
+
const messages = rows.reverse().map((row) => {
|
|
391
|
+
const r = row;
|
|
392
|
+
const msg = JSON.parse(r.msg);
|
|
393
|
+
msg.time = r.time;
|
|
394
|
+
msg.type = r.type;
|
|
395
|
+
const newMsg = new Msg(msg);
|
|
396
|
+
// Use ID from stored JSON if available (new messages)
|
|
397
|
+
// Fall back to SQLite row ID for old messages without stored ID
|
|
398
|
+
if (typeof msg.id === "number") {
|
|
399
|
+
newMsg.id = msg.id;
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
newMsg.id = r.id;
|
|
403
|
+
}
|
|
404
|
+
if (newMsg.id > maxId) {
|
|
405
|
+
maxId = newMsg.id;
|
|
406
|
+
}
|
|
337
407
|
return newMsg;
|
|
338
408
|
});
|
|
409
|
+
// Ensure client.idMsg is higher than any loaded message ID
|
|
410
|
+
// by calling nextID until it surpasses maxId
|
|
411
|
+
while (nextID() <= maxId) {
|
|
412
|
+
// Keep incrementing until we're past the max
|
|
413
|
+
}
|
|
414
|
+
return messages;
|
|
339
415
|
}
|
|
340
416
|
async search(query) {
|
|
341
417
|
await this.initDone.promise;
|
|
@@ -343,9 +419,37 @@ class SqliteMessageStorage {
|
|
|
343
419
|
// this should never be hit as messageProvider is checked in client.search()
|
|
344
420
|
throw new Error("search called but sqlite provider not enabled. This is a programming error");
|
|
345
421
|
}
|
|
422
|
+
// Flush any pending batched writes before searching
|
|
423
|
+
await this.flushBatch();
|
|
424
|
+
const searchTermParts = query.searchTerm.split(" ");
|
|
425
|
+
let userFilter = null;
|
|
426
|
+
let dateEndFilter = null;
|
|
427
|
+
let dateStartFilter = null;
|
|
428
|
+
for (const part of [...searchTermParts]) {
|
|
429
|
+
if (part.startsWith("from:") && userFilter === null) {
|
|
430
|
+
userFilter = part.slice(5);
|
|
431
|
+
searchTermParts.splice(searchTermParts.indexOf(part), 1);
|
|
432
|
+
}
|
|
433
|
+
if (part.startsWith("datebefore:") && dateEndFilter === null) {
|
|
434
|
+
const dateStr = part.slice(11);
|
|
435
|
+
const date = new Date(dateStr);
|
|
436
|
+
if (!Number.isNaN(date.getTime())) {
|
|
437
|
+
dateEndFilter = date.getTime();
|
|
438
|
+
searchTermParts.splice(searchTermParts.indexOf(part), 1);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
if (part.startsWith("dateafter:") && dateStartFilter === null) {
|
|
442
|
+
const dateStr = part.slice(10);
|
|
443
|
+
const date = new Date(dateStr);
|
|
444
|
+
if (!Number.isNaN(date.getTime())) {
|
|
445
|
+
dateStartFilter = date.getTime();
|
|
446
|
+
searchTermParts.splice(searchTermParts.indexOf(part), 1);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
}
|
|
346
450
|
// Using the '@' character to escape '%' and '_' in patterns.
|
|
347
|
-
const escapedSearchTerm =
|
|
348
|
-
let select =
|
|
451
|
+
const escapedSearchTerm = searchTermParts.join(" ").replace(/([%_@])/g, "@$1");
|
|
452
|
+
let select = "SELECT id, msg, type, time, network, channel FROM messages WHERE type = 'message' AND json_extract(msg, '$.text') LIKE ? ESCAPE '@'";
|
|
349
453
|
const params = [`%${escapedSearchTerm}%`];
|
|
350
454
|
if (query.networkUuid) {
|
|
351
455
|
select += " AND network = ? ";
|
|
@@ -355,6 +459,18 @@ class SqliteMessageStorage {
|
|
|
355
459
|
select += " AND channel = ? ";
|
|
356
460
|
params.push(query.channelName.toLowerCase());
|
|
357
461
|
}
|
|
462
|
+
if (userFilter !== null) {
|
|
463
|
+
select += " AND LOWER(json_extract(msg, '$.from.nick')) = ? ";
|
|
464
|
+
params.push(userFilter.toLowerCase());
|
|
465
|
+
}
|
|
466
|
+
if (dateEndFilter !== null) {
|
|
467
|
+
select += " AND time <= ? ";
|
|
468
|
+
params.push(dateEndFilter);
|
|
469
|
+
}
|
|
470
|
+
if (dateStartFilter !== null) {
|
|
471
|
+
select += " AND time >= ? ";
|
|
472
|
+
params.push(dateStartFilter);
|
|
473
|
+
}
|
|
358
474
|
const maxResults = 100;
|
|
359
475
|
select += " ORDER BY time DESC LIMIT ? OFFSET ? ";
|
|
360
476
|
params.push(maxResults);
|
|
@@ -362,11 +478,13 @@ class SqliteMessageStorage {
|
|
|
362
478
|
const rows = await this.serialize_fetchall(select, ...params);
|
|
363
479
|
return {
|
|
364
480
|
...query,
|
|
365
|
-
results: parseSearchRowsToMessages(
|
|
481
|
+
results: parseSearchRowsToMessages(rows).reverse(),
|
|
366
482
|
};
|
|
367
483
|
}
|
|
368
484
|
async deleteMessages(req) {
|
|
369
485
|
await this.initDone.promise;
|
|
486
|
+
// Flush any pending batched writes before deleting
|
|
487
|
+
await this.flushBatch();
|
|
370
488
|
let sql = "delete from messages where id in (select id from messages where\n";
|
|
371
489
|
// We roughly get a timestamp from N days before.
|
|
372
490
|
// We don't adjust for daylight savings time or other weird time jumps
|
|
@@ -385,70 +503,133 @@ class SqliteMessageStorage {
|
|
|
385
503
|
sql += ")";
|
|
386
504
|
return this.serialize_run(sql);
|
|
387
505
|
}
|
|
506
|
+
/**
|
|
507
|
+
* Get last N messages for a channel (for initial load)
|
|
508
|
+
*/
|
|
509
|
+
async getLastMessages(networkUuid, channelName, limit) {
|
|
510
|
+
await this.initDone.promise;
|
|
511
|
+
if (!this.isEnabled) {
|
|
512
|
+
return [];
|
|
513
|
+
}
|
|
514
|
+
// Flush any pending batched writes before reading
|
|
515
|
+
await this.flushBatch();
|
|
516
|
+
const rows = await this.serialize_fetchall("SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), limit);
|
|
517
|
+
return rows.reverse().map((row) => {
|
|
518
|
+
const r = row;
|
|
519
|
+
const msg = JSON.parse(r.msg);
|
|
520
|
+
msg.time = r.time;
|
|
521
|
+
msg.type = r.type;
|
|
522
|
+
return new Msg(msg);
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Get messages before a specific timestamp (for lazy loading)
|
|
527
|
+
*/
|
|
528
|
+
async getMessagesBefore(networkUuid, channelName, beforeTime, limit) {
|
|
529
|
+
await this.initDone.promise;
|
|
530
|
+
if (!this.isEnabled) {
|
|
531
|
+
return [];
|
|
532
|
+
}
|
|
533
|
+
const rows = await this.serialize_fetchall("SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? AND time < ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), beforeTime, limit);
|
|
534
|
+
return rows.reverse().map((row) => {
|
|
535
|
+
const r = row;
|
|
536
|
+
const msg = JSON.parse(r.msg);
|
|
537
|
+
msg.time = r.time;
|
|
538
|
+
msg.type = r.type;
|
|
539
|
+
return new Msg(msg);
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Get messages around a specific timestamp (for jumping to search results)
|
|
544
|
+
* Returns messages before and after the target time
|
|
545
|
+
*/
|
|
546
|
+
async getMessagesAround(networkUuid, channelName, targetTime, limit = 200) {
|
|
547
|
+
await this.initDone.promise;
|
|
548
|
+
if (!this.isEnabled) {
|
|
549
|
+
return [];
|
|
550
|
+
}
|
|
551
|
+
const halfLimit = Math.floor(limit / 2);
|
|
552
|
+
// Get messages before the target time
|
|
553
|
+
const beforeRows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? AND time <= ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), targetTime, halfLimit);
|
|
554
|
+
// Get messages after the target time
|
|
555
|
+
const afterRows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? AND time > ? ORDER BY time ASC LIMIT ?", networkUuid, channelName.toLowerCase(), targetTime, halfLimit);
|
|
556
|
+
// Combine: before (reversed) + after
|
|
557
|
+
const allRows = [...beforeRows.reverse(), ...afterRows];
|
|
558
|
+
return allRows.map((row) => {
|
|
559
|
+
const r = row;
|
|
560
|
+
const msg = JSON.parse(r.msg);
|
|
561
|
+
msg.time = r.time;
|
|
562
|
+
msg.type = r.type;
|
|
563
|
+
// Use stored ID from JSON if available, fall back to SQLite row ID
|
|
564
|
+
if (typeof msg.id !== "number") {
|
|
565
|
+
msg.id = r.id;
|
|
566
|
+
}
|
|
567
|
+
return new Msg(msg);
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Get total message count for a channel
|
|
572
|
+
*/
|
|
573
|
+
async getMessageCount(networkUuid, channelName) {
|
|
574
|
+
await this.initDone.promise;
|
|
575
|
+
if (!this.isEnabled) {
|
|
576
|
+
return 0;
|
|
577
|
+
}
|
|
578
|
+
const row = await this.serialize_get("SELECT COUNT(*) as count FROM messages WHERE network = ? AND channel = ?", networkUuid, channelName.toLowerCase());
|
|
579
|
+
return row ? row.count : 0;
|
|
580
|
+
}
|
|
388
581
|
canProvideMessages() {
|
|
389
582
|
return this.isEnabled;
|
|
390
583
|
}
|
|
391
584
|
serialize_run(stmt, ...params) {
|
|
392
|
-
|
|
393
|
-
this.database.
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
resolve(this.changes); // number of affected rows, `this` is re-bound by sqlite3
|
|
400
|
-
});
|
|
401
|
-
});
|
|
402
|
-
});
|
|
585
|
+
try {
|
|
586
|
+
const result = this.database.prepare(stmt).run(...params);
|
|
587
|
+
return Promise.resolve(result.changes);
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
591
|
+
}
|
|
403
592
|
}
|
|
404
593
|
serialize_fetchall(stmt, ...params) {
|
|
405
|
-
|
|
406
|
-
this.database.
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
resolve(rows);
|
|
413
|
-
});
|
|
414
|
-
});
|
|
415
|
-
});
|
|
594
|
+
try {
|
|
595
|
+
const rows = this.database.prepare(stmt).all(...params);
|
|
596
|
+
return Promise.resolve(rows);
|
|
597
|
+
}
|
|
598
|
+
catch (err) {
|
|
599
|
+
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
600
|
+
}
|
|
416
601
|
}
|
|
417
602
|
serialize_get(stmt, ...params) {
|
|
418
|
-
|
|
419
|
-
this.database.
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
resolve(row);
|
|
426
|
-
});
|
|
427
|
-
});
|
|
428
|
-
});
|
|
603
|
+
try {
|
|
604
|
+
const row = this.database.prepare(stmt).get(...params);
|
|
605
|
+
return Promise.resolve(row);
|
|
606
|
+
}
|
|
607
|
+
catch (err) {
|
|
608
|
+
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
609
|
+
}
|
|
429
610
|
}
|
|
430
611
|
}
|
|
431
|
-
|
|
432
|
-
function parseSearchRowsToMessages(id, rows) {
|
|
612
|
+
function parseSearchRowsToMessages(rows) {
|
|
433
613
|
const messages = [];
|
|
434
614
|
for (const row of rows) {
|
|
435
|
-
const
|
|
436
|
-
msg
|
|
437
|
-
msg.
|
|
438
|
-
msg.
|
|
439
|
-
msg.
|
|
440
|
-
msg.
|
|
441
|
-
|
|
442
|
-
id
|
|
615
|
+
const r = row;
|
|
616
|
+
const msg = JSON.parse(r.msg);
|
|
617
|
+
msg.time = r.time;
|
|
618
|
+
msg.type = r.type;
|
|
619
|
+
msg.networkUuid = r.network;
|
|
620
|
+
msg.channelName = r.channel;
|
|
621
|
+
// Use stored ID from JSON if available, fall back to SQLite row ID
|
|
622
|
+
if (typeof msg.id !== "number") {
|
|
623
|
+
msg.id = r.id;
|
|
624
|
+
}
|
|
625
|
+
messages.push(new Msg(msg));
|
|
443
626
|
}
|
|
444
627
|
return messages;
|
|
445
628
|
}
|
|
446
|
-
function necessaryMigrations(since) {
|
|
447
|
-
return
|
|
629
|
+
export function necessaryMigrations(since) {
|
|
630
|
+
return migrations.filter((m) => m.version > since);
|
|
448
631
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
return exports.rollbacks.filter((r) => r.version > since);
|
|
632
|
+
export function newRollbacks(since) {
|
|
633
|
+
return rollbacks.filter((r) => r.version > since);
|
|
452
634
|
}
|
|
453
|
-
|
|
454
|
-
exports.default = SqliteMessageStorage;
|
|
635
|
+
export default SqliteMessageStorage;
|