@lordbex/thelounge 4.4.3-blowfish → 4.5.0-blowfish-pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/README.md +31 -7
  2. package/dist/defaults/config.js +31 -2
  3. package/dist/package.json +94 -91
  4. package/dist/server/client.js +188 -194
  5. package/dist/server/clientManager.js +75 -72
  6. package/dist/server/command-line/index.js +44 -43
  7. package/dist/server/command-line/install.js +37 -70
  8. package/dist/server/command-line/outdated.js +12 -17
  9. package/dist/server/command-line/start.js +25 -26
  10. package/dist/server/command-line/storage.js +26 -31
  11. package/dist/server/command-line/uninstall.js +16 -23
  12. package/dist/server/command-line/upgrade.js +20 -26
  13. package/dist/server/command-line/users/add.js +33 -40
  14. package/dist/server/command-line/users/edit.js +18 -24
  15. package/dist/server/command-line/users/index.js +12 -16
  16. package/dist/server/command-line/users/list.js +11 -39
  17. package/dist/server/command-line/users/remove.js +16 -22
  18. package/dist/server/command-line/users/reset.js +34 -35
  19. package/dist/server/command-line/utils.js +231 -87
  20. package/dist/server/config.js +61 -52
  21. package/dist/server/helper.js +29 -28
  22. package/dist/server/identification.js +39 -34
  23. package/dist/server/index.js +1 -3
  24. package/dist/server/log.js +19 -16
  25. package/dist/server/models/chan.js +36 -33
  26. package/dist/server/models/msg.js +15 -19
  27. package/dist/server/models/network.js +102 -104
  28. package/dist/server/models/prefix.js +4 -7
  29. package/dist/server/models/user.js +5 -10
  30. package/dist/server/path-helper.js +8 -0
  31. package/dist/server/plugins/auth/ldap.js +177 -112
  32. package/dist/server/plugins/auth/local.js +10 -15
  33. package/dist/server/plugins/auth.js +6 -35
  34. package/dist/server/plugins/changelog.js +30 -27
  35. package/dist/server/plugins/clientCertificate.js +33 -37
  36. package/dist/server/plugins/dev-server.js +15 -21
  37. package/dist/server/plugins/inputs/action.js +9 -14
  38. package/dist/server/plugins/inputs/away.js +1 -3
  39. package/dist/server/plugins/inputs/ban.js +9 -14
  40. package/dist/server/plugins/inputs/blow.js +9 -14
  41. package/dist/server/plugins/inputs/connect.js +5 -10
  42. package/dist/server/plugins/inputs/ctcp.js +7 -12
  43. package/dist/server/plugins/inputs/disconnect.js +1 -3
  44. package/dist/server/plugins/inputs/ignore.js +23 -29
  45. package/dist/server/plugins/inputs/ignorelist.js +12 -18
  46. package/dist/server/plugins/inputs/index.js +8 -34
  47. package/dist/server/plugins/inputs/invite.js +7 -12
  48. package/dist/server/plugins/inputs/kick.js +7 -12
  49. package/dist/server/plugins/inputs/kill.js +1 -3
  50. package/dist/server/plugins/inputs/list.js +1 -3
  51. package/dist/server/plugins/inputs/mode.js +10 -15
  52. package/dist/server/plugins/inputs/msg.js +13 -18
  53. package/dist/server/plugins/inputs/mute.js +9 -15
  54. package/dist/server/plugins/inputs/nick.js +9 -14
  55. package/dist/server/plugins/inputs/notice.js +5 -7
  56. package/dist/server/plugins/inputs/part.js +11 -16
  57. package/dist/server/plugins/inputs/quit.js +7 -13
  58. package/dist/server/plugins/inputs/rainbow.js +55 -0
  59. package/dist/server/plugins/inputs/raw.js +1 -3
  60. package/dist/server/plugins/inputs/rejoin.js +7 -12
  61. package/dist/server/plugins/inputs/topic.js +7 -12
  62. package/dist/server/plugins/inputs/whois.js +1 -3
  63. package/dist/server/plugins/irc-events/away.js +14 -20
  64. package/dist/server/plugins/irc-events/cap.js +16 -22
  65. package/dist/server/plugins/irc-events/chghost.js +14 -13
  66. package/dist/server/plugins/irc-events/connection.js +61 -63
  67. package/dist/server/plugins/irc-events/ctcp.js +22 -28
  68. package/dist/server/plugins/irc-events/error.js +20 -26
  69. package/dist/server/plugins/irc-events/help.js +7 -13
  70. package/dist/server/plugins/irc-events/info.js +7 -13
  71. package/dist/server/plugins/irc-events/invite.js +7 -13
  72. package/dist/server/plugins/irc-events/join.js +30 -27
  73. package/dist/server/plugins/irc-events/kick.js +21 -17
  74. package/dist/server/plugins/irc-events/link.js +122 -109
  75. package/dist/server/plugins/irc-events/list.js +23 -26
  76. package/dist/server/plugins/irc-events/message.js +46 -52
  77. package/dist/server/plugins/irc-events/mode.js +66 -63
  78. package/dist/server/plugins/irc-events/modelist.js +29 -35
  79. package/dist/server/plugins/irc-events/motd.js +10 -16
  80. package/dist/server/plugins/irc-events/names.js +3 -6
  81. package/dist/server/plugins/irc-events/nick.js +26 -23
  82. package/dist/server/plugins/irc-events/part.js +19 -15
  83. package/dist/server/plugins/irc-events/quit.js +17 -14
  84. package/dist/server/plugins/irc-events/sasl.js +9 -15
  85. package/dist/server/plugins/irc-events/spgroups.js +38 -0
  86. package/dist/server/plugins/irc-events/spjoin.js +52 -0
  87. package/dist/server/plugins/irc-events/topic.js +12 -18
  88. package/dist/server/plugins/irc-events/unhandled.js +12 -12
  89. package/dist/server/plugins/irc-events/welcome.js +7 -13
  90. package/dist/server/plugins/irc-events/whois.js +20 -24
  91. package/dist/server/plugins/massEventAggregator.js +214 -0
  92. package/dist/server/plugins/messageStorage/sqlite.js +322 -141
  93. package/dist/server/plugins/messageStorage/text.js +21 -26
  94. package/dist/server/plugins/packages/index.js +105 -74
  95. package/dist/server/plugins/packages/publicClient.js +7 -16
  96. package/dist/server/plugins/packages/themes.js +11 -16
  97. package/dist/server/plugins/storage.js +28 -33
  98. package/dist/server/plugins/sts.js +12 -17
  99. package/dist/server/plugins/uploader.js +40 -43
  100. package/dist/server/plugins/webpush.js +23 -51
  101. package/dist/server/server.js +318 -271
  102. package/dist/server/storageCleaner.js +29 -37
  103. package/dist/server/utils/fish.js +349 -389
  104. package/dist/shared/irc.js +3 -6
  105. package/dist/shared/linkify.js +7 -14
  106. package/dist/shared/types/chan.js +6 -9
  107. package/dist/shared/types/changelog.js +1 -2
  108. package/dist/shared/types/config.js +1 -2
  109. package/dist/shared/types/mention.js +1 -2
  110. package/dist/shared/types/msg.js +3 -5
  111. package/dist/shared/types/network.js +1 -2
  112. package/dist/shared/types/storage.js +1 -2
  113. package/dist/shared/types/user.js +1 -2
  114. package/index.js +14 -10
  115. package/package.json +94 -91
  116. package/public/css/style.css +9 -6
  117. package/public/css/style.css.map +1 -1
  118. package/public/fonts/font-awesome/fa-brands-400.ttf +0 -0
  119. package/public/fonts/font-awesome/fa-brands-400.woff2 +0 -0
  120. package/public/fonts/font-awesome/fa-duotone-900.ttf +0 -0
  121. package/public/fonts/font-awesome/fa-duotone-900.woff2 +0 -0
  122. package/public/fonts/font-awesome/fa-light-300.ttf +0 -0
  123. package/public/fonts/font-awesome/fa-light-300.woff2 +0 -0
  124. package/public/fonts/font-awesome/fa-regular-400.ttf +0 -0
  125. package/public/fonts/font-awesome/fa-regular-400.woff2 +0 -0
  126. package/public/fonts/font-awesome/fa-solid-900.ttf +0 -0
  127. package/public/fonts/font-awesome/fa-solid-900.woff2 +0 -0
  128. package/public/fonts/font-awesome/fa-thin-100.ttf +0 -0
  129. package/public/fonts/font-awesome/fa-thin-100.woff2 +0 -0
  130. package/public/fonts/font-awesome/fa-v4compatibility.ttf +0 -0
  131. package/public/fonts/font-awesome/fa-v4compatibility.woff2 +0 -0
  132. package/public/js/bundle.js +1 -1
  133. package/public/js/bundle.js.map +1 -1
  134. package/public/js/bundle.vendor.js +1 -1
  135. package/public/js/bundle.vendor.js.LICENSE.txt +24 -6
  136. package/public/js/bundle.vendor.js.map +1 -1
  137. package/public/js/loading-error-handlers.js +1 -1
  138. package/public/service-worker.js +1 -1
  139. package/public/themes/default.css +1 -1
  140. package/public/themes/morning.css +1 -1
  141. package/dist/webpack.config.js +0 -224
@@ -1,40 +1,26 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.newRollbacks = exports.necessaryMigrations = exports.rollbacks = exports.migrations = exports.currentSchemaVersion = void 0;
7
- const log_1 = __importDefault(require("../../log"));
8
- const path_1 = __importDefault(require("path"));
9
- const promises_1 = __importDefault(require("fs/promises"));
10
- const config_1 = __importDefault(require("../../config"));
11
- const msg_1 = __importDefault(require("../../models/msg"));
12
- const helper_1 = __importDefault(require("../../helper"));
13
- // TODO; type
14
- let sqlite3;
15
- try {
16
- sqlite3 = require("sqlite3");
17
- }
18
- catch (e) {
19
- config_1.default.values.messageStorage = config_1.default.values.messageStorage.filter((item) => item !== "sqlite");
20
- log_1.default.error("Unable to load sqlite3 module. See https://github.com/mapbox/node-sqlite3/wiki/Binaries");
21
- }
22
- exports.currentSchemaVersion = 1703322560448; // use `new Date().getTime()`
1
+ import Database from "better-sqlite3";
2
+ import log from "../../log.js";
3
+ import path from "path";
4
+ import fs from "fs/promises";
5
+ import Config from "../../config.js";
6
+ import Msg from "../../models/msg.js";
7
+ import Helper from "../../helper.js";
8
+ export const currentSchemaVersion = 1703322560448; // use `new Date().getTime()`
23
9
  // Desired schema, adapt to the newest version and add migrations to the array below
24
10
  const schema = [
25
11
  "CREATE TABLE options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))",
26
12
  "CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)",
27
13
  `CREATE TABLE migrations (
28
- id INTEGER PRIMARY KEY AUTOINCREMENT,
29
- version INTEGER NOT NULL UNIQUE,
30
- rollback_forbidden INTEGER DEFAULT 0 NOT NULL
31
- )`,
14
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
15
+ version INTEGER NOT NULL UNIQUE,
16
+ rollback_forbidden INTEGER DEFAULT 0 NOT NULL
17
+ )`,
32
18
  `CREATE TABLE rollback_steps (
33
- id INTEGER PRIMARY KEY AUTOINCREMENT,
34
- migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
35
- step INTEGER NOT NULL,
36
- statement TEXT NOT NULL
37
- )`,
19
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
20
+ migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
21
+ step INTEGER NOT NULL,
22
+ statement TEXT NOT NULL
23
+ )`,
38
24
  "CREATE INDEX network_channel ON messages (network, channel)",
39
25
  "CREATE INDEX time ON messages (time)",
40
26
  "CREATE INDEX msg_type_idx on messages (type)", // needed for efficient storageCleaner queries
@@ -42,7 +28,7 @@ const schema = [
42
28
  // the migrations will be executed in an exclusive transaction as a whole
43
29
  // add new migrations to the end, with the version being the new 'currentSchemaVersion'
44
30
  // write a corresponding down migration into rollbacks
45
- exports.migrations = [
31
+ export const migrations = [
46
32
  {
47
33
  version: 1672236339873,
48
34
  stmts: [
@@ -58,16 +44,16 @@ exports.migrations = [
58
44
  version: 1679743888000,
59
45
  stmts: [
60
46
  `CREATE TABLE IF NOT EXISTS migrations (
61
- id INTEGER PRIMARY KEY AUTOINCREMENT,
62
- version INTEGER NOT NULL UNIQUE,
63
- rollback_forbidden INTEGER DEFAULT 0 NOT NULL
64
- )`,
47
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
48
+ version INTEGER NOT NULL UNIQUE,
49
+ rollback_forbidden INTEGER DEFAULT 0 NOT NULL
50
+ )`,
65
51
  `CREATE TABLE IF NOT EXISTS rollback_steps (
66
- id INTEGER PRIMARY KEY AUTOINCREMENT,
67
- migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
68
- step INTEGER NOT NULL,
69
- statement TEXT NOT NULL
70
- )`,
52
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
53
+ migration_id INTEGER NOT NULL REFERENCES migrations ON DELETE CASCADE,
54
+ step INTEGER NOT NULL,
55
+ statement TEXT NOT NULL
56
+ )`,
71
57
  ],
72
58
  },
73
59
  {
@@ -77,7 +63,7 @@ exports.migrations = [
77
63
  ];
78
64
  // down migrations need to restore the state of the prior version.
79
65
  // rollback can be disallowed by adding rollback_forbidden: true to it
80
- exports.rollbacks = [
66
+ export const rollbacks = [
81
67
  {
82
68
  version: 1672236339873,
83
69
  stmts: [], // changes aren't visible, left empty on purpose
@@ -105,31 +91,39 @@ class SqliteMessageStorage {
105
91
  database;
106
92
  initDone;
107
93
  userName;
94
+ // Message batching for improved write performance
95
+ batchQueue = [];
96
+ batchSize = 50; // Flush after 50 messages
97
+ batchTimeout = 1000; // Flush after 1 second
98
+ batchTimer = null;
99
+ insertStmt = null;
108
100
  constructor(userName) {
109
101
  this.userName = userName;
110
102
  this.isEnabled = false;
111
103
  this.initDone = new Deferred();
112
104
  }
113
105
  async _enable(connection_string) {
114
- this.database = new sqlite3.Database(connection_string);
115
106
  try {
107
+ this.database = new Database(connection_string);
116
108
  await this.run_pragmas(); // must be done outside of a transaction
117
109
  await this.run_migrations();
110
+ // Prepare insert statement for batching
111
+ this.insertStmt = this.database.prepare("INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)");
118
112
  }
119
113
  catch (e) {
120
114
  this.isEnabled = false;
121
- throw helper_1.default.catch_to_error("Migration failed", e);
115
+ throw Helper.catch_to_error("Migration failed", e);
122
116
  }
123
117
  this.isEnabled = true;
124
118
  }
125
119
  async enable() {
126
- const logsPath = config_1.default.getUserLogsPath();
127
- const sqlitePath = path_1.default.join(logsPath, `${this.userName}.sqlite3`);
120
+ const logsPath = Config.getUserLogsPath();
121
+ const sqlitePath = path.join(logsPath, `${this.userName}.sqlite3`);
128
122
  try {
129
- await promises_1.default.mkdir(logsPath, { recursive: true });
123
+ await fs.mkdir(logsPath, { recursive: true });
130
124
  }
131
125
  catch (e) {
132
- throw helper_1.default.catch_to_error("Unable to create logs directory", e);
126
+ throw Helper.catch_to_error("Unable to create logs directory", e);
133
127
  }
134
128
  try {
135
129
  await this._enable(sqlitePath);
@@ -142,7 +136,7 @@ class SqliteMessageStorage {
142
136
  for (const stmt of schema) {
143
137
  await this.serialize_run(stmt);
144
138
  }
145
- await this.serialize_run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", exports.currentSchemaVersion.toString());
139
+ await this.serialize_run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", currentSchemaVersion.toString());
146
140
  }
147
141
  async current_version() {
148
142
  const have_options = await this.serialize_get("select 1 from sqlite_master where type = 'table' and name = 'options'");
@@ -159,10 +153,10 @@ class SqliteMessageStorage {
159
153
  return storedSchemaVersion;
160
154
  }
161
155
  async update_version_in_db() {
162
- return this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'", exports.currentSchemaVersion.toString());
156
+ return this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'", currentSchemaVersion.toString());
163
157
  }
164
158
  async _run_migrations(dbVersion) {
165
- log_1.default.info(`sqlite messages schema version is out of date (${dbVersion} < ${exports.currentSchemaVersion}). Running migrations.`);
159
+ log.info(`sqlite messages schema version is out of date (${dbVersion} < ${currentSchemaVersion}). Running migrations.`);
166
160
  const to_execute = necessaryMigrations(dbVersion);
167
161
  for (const stmt of to_execute.map((m) => m.stmts).flat()) {
168
162
  await this.serialize_run(stmt);
@@ -174,10 +168,10 @@ class SqliteMessageStorage {
174
168
  }
175
169
  async run_migrations() {
176
170
  const version = await this.current_version();
177
- if (version > exports.currentSchemaVersion) {
178
- throw `sqlite messages schema version is higher than expected (${version} > ${exports.currentSchemaVersion}). Is The Lounge out of date?`;
171
+ if (version > currentSchemaVersion) {
172
+ throw new Error(`sqlite messages schema version is higher than expected (${version} > ${currentSchemaVersion}). Is NexusIRC out of date?`);
179
173
  }
180
- else if (version === exports.currentSchemaVersion) {
174
+ else if (version === currentSchemaVersion) {
181
175
  return; // nothing to do
182
176
  }
183
177
  await this.serialize_run("BEGIN EXCLUSIVE TRANSACTION");
@@ -205,37 +199,82 @@ class SqliteMessageStorage {
205
199
  if (!this.isEnabled) {
206
200
  return;
207
201
  }
202
+ // Flush any pending batched messages
203
+ await this.flushBatch();
204
+ // Clear batch timer
205
+ if (this.batchTimer) {
206
+ clearTimeout(this.batchTimer);
207
+ this.batchTimer = null;
208
+ }
208
209
  this.isEnabled = false;
209
- return new Promise((resolve, reject) => {
210
- this.database.close((err) => {
211
- if (err) {
212
- reject(`Failed to close sqlite database: ${err.message}`);
213
- return;
210
+ try {
211
+ this.database.close();
212
+ }
213
+ catch (err) {
214
+ throw new Error(`Failed to close sqlite database: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ }
217
+ /**
218
+ * Flush batched messages to database using a transaction
219
+ */
220
+ async flushBatch() {
221
+ if (this.batchQueue.length === 0) {
222
+ return Promise.resolve();
223
+ }
224
+ if (!this.insertStmt) {
225
+ log.error("Cannot flush batch: insert statement not prepared");
226
+ return Promise.resolve();
227
+ }
228
+ const messages = this.batchQueue.splice(0); // Take all messages and clear queue
229
+ try {
230
+ // Use transaction for batch insert (much faster than individual inserts)
231
+ const transaction = this.database.transaction((msgs) => {
232
+ for (const msg of msgs) {
233
+ this.insertStmt.run(msg.network, msg.channel, msg.time, msg.type, msg.msg);
214
234
  }
215
- resolve();
216
235
  });
217
- });
236
+ transaction(messages);
237
+ }
238
+ catch (err) {
239
+ log.error(`Failed to flush message batch: ${err instanceof Error ? err.message : String(err)}`);
240
+ // Re-add messages to queue on failure
241
+ this.batchQueue.unshift(...messages);
242
+ throw err;
243
+ }
244
+ }
245
+ /**
246
+ * Schedule a batch flush
247
+ */
248
+ scheduleBatchFlush() {
249
+ if (this.batchTimer) {
250
+ return; // Timer already scheduled
251
+ }
252
+ this.batchTimer = setTimeout(() => {
253
+ this.batchTimer = null;
254
+ this.flushBatch().catch((err) => log.error(`Batch flush error: ${err}`));
255
+ }, this.batchTimeout);
218
256
  }
219
257
  async fetch_rollbacks(since_version) {
220
258
  const res = await this.serialize_fetchall(`select version, rollback_forbidden, statement
221
- from rollback_steps
222
- join migrations on migrations.id=rollback_steps.migration_id
223
- where version > ?
224
- order by version desc, step asc`, since_version);
259
+ from rollback_steps
260
+ join migrations on migrations.id=rollback_steps.migration_id
261
+ where version > ?
262
+ order by version desc, step asc`, since_version);
225
263
  const result = [];
226
264
  // convert to Rollback[]
227
265
  // requires ordering in the sql statement
228
266
  for (const raw of res) {
229
267
  const last = result.at(-1);
230
- if (!last || raw.version !== last.version) {
268
+ const r = raw;
269
+ if (!last || r.version !== last.version) {
231
270
  result.push({
232
- version: raw.version,
233
- rollback_forbidden: Boolean(raw.rollback_forbidden),
234
- stmts: [raw.statement],
271
+ version: r.version,
272
+ rollback_forbidden: Boolean(r.rollback_forbidden),
273
+ stmts: [r.statement],
235
274
  });
236
275
  }
237
276
  else {
238
- last.stmts.push(raw.statement);
277
+ last.stmts.push(r.statement);
239
278
  }
240
279
  }
241
280
  return result;
@@ -278,21 +317,21 @@ class SqliteMessageStorage {
278
317
  return new_version;
279
318
  }
280
319
  async downgrade() {
281
- const res = await this.downgrade_to(exports.currentSchemaVersion);
320
+ const res = await this.downgrade_to(currentSchemaVersion);
282
321
  return res;
283
322
  }
284
323
  async insert_rollback_since(version) {
285
324
  const missing = newRollbacks(version);
286
325
  for (const rollback of missing) {
287
326
  const migration = await this.serialize_get(`insert into migrations
288
- (version, rollback_forbidden)
289
- values (?, ?)
290
- returning id`, rollback.version, rollback.rollback_forbidden || 0);
327
+ (version, rollback_forbidden)
328
+ values (?, ?)
329
+ returning id`, rollback.version, rollback.rollback_forbidden || 0);
291
330
  for (const stmt of rollback.stmts) {
292
331
  let step = 0;
293
332
  await this.serialize_run(`insert into rollback_steps
294
- (migration_id, step, statement)
295
- values (?, ?, ?)`, migration.id, step, stmt);
333
+ (migration_id, step, statement)
334
+ values (?, ?, ?)`, migration.id, step, stmt);
296
335
  step++;
297
336
  }
298
337
  }
@@ -303,15 +342,30 @@ class SqliteMessageStorage {
303
342
  return;
304
343
  }
305
344
  const clonedMsg = Object.keys(msg).reduce((newMsg, prop) => {
306
- // id is regenerated when messages are retrieved
307
345
  // previews are not stored because storage is cleared on lounge restart
308
346
  // type and time are stored in a separate column
309
- if (prop !== "id" && prop !== "previews" && prop !== "type" && prop !== "time") {
347
+ // id IS now stored so it can be retrieved consistently
348
+ if (prop !== "previews" && prop !== "type" && prop !== "time") {
310
349
  newMsg[prop] = msg[prop];
311
350
  }
312
351
  return newMsg;
313
352
  }, {});
314
- await this.serialize_run("INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)", network.uuid, channel.name.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg));
353
+ // Add to batch queue instead of immediate insert
354
+ this.batchQueue.push({
355
+ network: network.uuid,
356
+ channel: channel.name.toLowerCase(),
357
+ time: msg.time.getTime(),
358
+ type: msg.type,
359
+ msg: JSON.stringify(clonedMsg),
360
+ });
361
+ // Flush batch if it reaches the size limit
362
+ if (this.batchQueue.length >= this.batchSize) {
363
+ await this.flushBatch();
364
+ }
365
+ else {
366
+ // Schedule flush after timeout
367
+ this.scheduleBatchFlush();
368
+ }
315
369
  }
316
370
  async deleteChannel(network, channel) {
317
371
  await this.initDone.promise;
@@ -322,20 +376,42 @@ class SqliteMessageStorage {
322
376
  }
323
377
  async getMessages(network, channel, nextID) {
324
378
  await this.initDone.promise;
325
- if (!this.isEnabled || config_1.default.values.maxHistory === 0) {
379
+ if (!this.isEnabled || Config.values.maxHistory === 0) {
326
380
  return [];
327
381
  }
382
+ // Flush any pending batched writes before reading
383
+ await this.flushBatch();
328
384
  // If unlimited history is specified, load 100k messages
329
- const limit = config_1.default.values.maxHistory < 0 ? 100000 : config_1.default.values.maxHistory;
330
- const rows = await this.serialize_fetchall("SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", network.uuid, channel.name.toLowerCase(), limit);
331
- return rows.reverse().map((row) => {
332
- const msg = JSON.parse(row.msg);
333
- msg.time = row.time;
334
- msg.type = row.type;
335
- const newMsg = new msg_1.default(msg);
336
- newMsg.id = nextID();
385
+ const limit = Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory;
386
+ // Select id from SQLite to use as the canonical message ID
387
+ const rows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", network.uuid, channel.name.toLowerCase(), limit);
388
+ // Track max ID so we can update client.idMsg
389
+ let maxId = 0;
390
+ const messages = rows.reverse().map((row) => {
391
+ const r = row;
392
+ const msg = JSON.parse(r.msg);
393
+ msg.time = r.time;
394
+ msg.type = r.type;
395
+ const newMsg = new Msg(msg);
396
+ // Use ID from stored JSON if available (new messages)
397
+ // Fall back to SQLite row ID for old messages without stored ID
398
+ if (typeof msg.id === "number") {
399
+ newMsg.id = msg.id;
400
+ }
401
+ else {
402
+ newMsg.id = r.id;
403
+ }
404
+ if (newMsg.id > maxId) {
405
+ maxId = newMsg.id;
406
+ }
337
407
  return newMsg;
338
408
  });
409
+ // Ensure client.idMsg is higher than any loaded message ID
410
+ // by calling nextID until it surpasses maxId
411
+ while (nextID() <= maxId) {
412
+ // Keep incrementing until we're past the max
413
+ }
414
+ return messages;
339
415
  }
340
416
  async search(query) {
341
417
  await this.initDone.promise;
@@ -343,9 +419,37 @@ class SqliteMessageStorage {
343
419
  // this should never be hit as messageProvider is checked in client.search()
344
420
  throw new Error("search called but sqlite provider not enabled. This is a programming error");
345
421
  }
422
+ // Flush any pending batched writes before searching
423
+ await this.flushBatch();
424
+ const searchTermParts = query.searchTerm.split(" ");
425
+ let userFilter = null;
426
+ let dateEndFilter = null;
427
+ let dateStartFilter = null;
428
+ for (const part of [...searchTermParts]) {
429
+ if (part.startsWith("from:") && userFilter === null) {
430
+ userFilter = part.slice(5);
431
+ searchTermParts.splice(searchTermParts.indexOf(part), 1);
432
+ }
433
+ if (part.startsWith("datebefore:") && dateEndFilter === null) {
434
+ const dateStr = part.slice(11);
435
+ const date = new Date(dateStr);
436
+ if (!Number.isNaN(date.getTime())) {
437
+ dateEndFilter = date.getTime();
438
+ searchTermParts.splice(searchTermParts.indexOf(part), 1);
439
+ }
440
+ }
441
+ if (part.startsWith("dateafter:") && dateStartFilter === null) {
442
+ const dateStr = part.slice(10);
443
+ const date = new Date(dateStr);
444
+ if (!Number.isNaN(date.getTime())) {
445
+ dateStartFilter = date.getTime();
446
+ searchTermParts.splice(searchTermParts.indexOf(part), 1);
447
+ }
448
+ }
449
+ }
346
450
  // Using the '@' character to escape '%' and '_' in patterns.
347
- const escapedSearchTerm = query.searchTerm.replace(/([%_@])/g, "@$1");
348
- let select = 'SELECT msg, type, time, network, channel FROM messages WHERE type = "message" AND json_extract(msg, "$.text") LIKE ? ESCAPE \'@\'';
451
+ const escapedSearchTerm = searchTermParts.join(" ").replace(/([%_@])/g, "@$1");
452
+ let select = "SELECT id, msg, type, time, network, channel FROM messages WHERE type = 'message' AND json_extract(msg, '$.text') LIKE ? ESCAPE '@'";
349
453
  const params = [`%${escapedSearchTerm}%`];
350
454
  if (query.networkUuid) {
351
455
  select += " AND network = ? ";
@@ -355,6 +459,18 @@ class SqliteMessageStorage {
355
459
  select += " AND channel = ? ";
356
460
  params.push(query.channelName.toLowerCase());
357
461
  }
462
+ if (userFilter !== null) {
463
+ select += " AND LOWER(json_extract(msg, '$.from.nick')) = ? ";
464
+ params.push(userFilter.toLowerCase());
465
+ }
466
+ if (dateEndFilter !== null) {
467
+ select += " AND time <= ? ";
468
+ params.push(dateEndFilter);
469
+ }
470
+ if (dateStartFilter !== null) {
471
+ select += " AND time >= ? ";
472
+ params.push(dateStartFilter);
473
+ }
358
474
  const maxResults = 100;
359
475
  select += " ORDER BY time DESC LIMIT ? OFFSET ? ";
360
476
  params.push(maxResults);
@@ -362,11 +478,13 @@ class SqliteMessageStorage {
362
478
  const rows = await this.serialize_fetchall(select, ...params);
363
479
  return {
364
480
  ...query,
365
- results: parseSearchRowsToMessages(query.offset, rows).reverse(),
481
+ results: parseSearchRowsToMessages(rows).reverse(),
366
482
  };
367
483
  }
368
484
  async deleteMessages(req) {
369
485
  await this.initDone.promise;
486
+ // Flush any pending batched writes before deleting
487
+ await this.flushBatch();
370
488
  let sql = "delete from messages where id in (select id from messages where\n";
371
489
  // We roughly get a timestamp from N days before.
372
490
  // We don't adjust for daylight savings time or other weird time jumps
@@ -385,70 +503,133 @@ class SqliteMessageStorage {
385
503
  sql += ")";
386
504
  return this.serialize_run(sql);
387
505
  }
506
+ /**
507
+ * Get last N messages for a channel (for initial load)
508
+ */
509
+ async getLastMessages(networkUuid, channelName, limit) {
510
+ await this.initDone.promise;
511
+ if (!this.isEnabled) {
512
+ return [];
513
+ }
514
+ // Flush any pending batched writes before reading
515
+ await this.flushBatch();
516
+ const rows = await this.serialize_fetchall("SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), limit);
517
+ return rows.reverse().map((row) => {
518
+ const r = row;
519
+ const msg = JSON.parse(r.msg);
520
+ msg.time = r.time;
521
+ msg.type = r.type;
522
+ return new Msg(msg);
523
+ });
524
+ }
525
+ /**
526
+ * Get messages before a specific timestamp (for lazy loading)
527
+ */
528
+ async getMessagesBefore(networkUuid, channelName, beforeTime, limit) {
529
+ await this.initDone.promise;
530
+ if (!this.isEnabled) {
531
+ return [];
532
+ }
533
+ const rows = await this.serialize_fetchall("SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? AND time < ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), beforeTime, limit);
534
+ return rows.reverse().map((row) => {
535
+ const r = row;
536
+ const msg = JSON.parse(r.msg);
537
+ msg.time = r.time;
538
+ msg.type = r.type;
539
+ return new Msg(msg);
540
+ });
541
+ }
542
+ /**
543
+ * Get messages around a specific timestamp (for jumping to search results)
544
+ * Returns messages before and after the target time
545
+ */
546
+ async getMessagesAround(networkUuid, channelName, targetTime, limit = 200) {
547
+ await this.initDone.promise;
548
+ if (!this.isEnabled) {
549
+ return [];
550
+ }
551
+ const halfLimit = Math.floor(limit / 2);
552
+ // Get messages before the target time
553
+ const beforeRows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? AND time <= ? ORDER BY time DESC LIMIT ?", networkUuid, channelName.toLowerCase(), targetTime, halfLimit);
554
+ // Get messages after the target time
555
+ const afterRows = await this.serialize_fetchall("SELECT id, msg, type, time FROM messages WHERE network = ? AND channel = ? AND time > ? ORDER BY time ASC LIMIT ?", networkUuid, channelName.toLowerCase(), targetTime, halfLimit);
556
+ // Combine: before (reversed) + after
557
+ const allRows = [...beforeRows.reverse(), ...afterRows];
558
+ return allRows.map((row) => {
559
+ const r = row;
560
+ const msg = JSON.parse(r.msg);
561
+ msg.time = r.time;
562
+ msg.type = r.type;
563
+ // Use stored ID from JSON if available, fall back to SQLite row ID
564
+ if (typeof msg.id !== "number") {
565
+ msg.id = r.id;
566
+ }
567
+ return new Msg(msg);
568
+ });
569
+ }
570
+ /**
571
+ * Get total message count for a channel
572
+ */
573
+ async getMessageCount(networkUuid, channelName) {
574
+ await this.initDone.promise;
575
+ if (!this.isEnabled) {
576
+ return 0;
577
+ }
578
+ const row = await this.serialize_get("SELECT COUNT(*) as count FROM messages WHERE network = ? AND channel = ?", networkUuid, channelName.toLowerCase());
579
+ return row ? row.count : 0;
580
+ }
388
581
  canProvideMessages() {
389
582
  return this.isEnabled;
390
583
  }
391
584
  serialize_run(stmt, ...params) {
392
- return new Promise((resolve, reject) => {
393
- this.database.serialize(() => {
394
- this.database.run(stmt, params, function (err) {
395
- if (err) {
396
- reject(err);
397
- return;
398
- }
399
- resolve(this.changes); // number of affected rows, `this` is re-bound by sqlite3
400
- });
401
- });
402
- });
585
+ try {
586
+ const result = this.database.prepare(stmt).run(...params);
587
+ return Promise.resolve(result.changes);
588
+ }
589
+ catch (err) {
590
+ return Promise.reject(err instanceof Error ? err : new Error(String(err)));
591
+ }
403
592
  }
404
593
  serialize_fetchall(stmt, ...params) {
405
- return new Promise((resolve, reject) => {
406
- this.database.serialize(() => {
407
- this.database.all(stmt, params, (err, rows) => {
408
- if (err) {
409
- reject(err);
410
- return;
411
- }
412
- resolve(rows);
413
- });
414
- });
415
- });
594
+ try {
595
+ const rows = this.database.prepare(stmt).all(...params);
596
+ return Promise.resolve(rows);
597
+ }
598
+ catch (err) {
599
+ return Promise.reject(err instanceof Error ? err : new Error(String(err)));
600
+ }
416
601
  }
417
602
  serialize_get(stmt, ...params) {
418
- return new Promise((resolve, reject) => {
419
- this.database.serialize(() => {
420
- this.database.get(stmt, params, (err, row) => {
421
- if (err) {
422
- reject(err);
423
- return;
424
- }
425
- resolve(row);
426
- });
427
- });
428
- });
603
+ try {
604
+ const row = this.database.prepare(stmt).get(...params);
605
+ return Promise.resolve(row);
606
+ }
607
+ catch (err) {
608
+ return Promise.reject(err instanceof Error ? err : new Error(String(err)));
609
+ }
429
610
  }
430
611
  }
431
- // TODO: type any
432
- function parseSearchRowsToMessages(id, rows) {
612
+ function parseSearchRowsToMessages(rows) {
433
613
  const messages = [];
434
614
  for (const row of rows) {
435
- const msg = JSON.parse(row.msg);
436
- msg.time = row.time;
437
- msg.type = row.type;
438
- msg.networkUuid = row.network;
439
- msg.channelName = row.channel;
440
- msg.id = id;
441
- messages.push(new msg_1.default(msg));
442
- id += 1;
615
+ const r = row;
616
+ const msg = JSON.parse(r.msg);
617
+ msg.time = r.time;
618
+ msg.type = r.type;
619
+ msg.networkUuid = r.network;
620
+ msg.channelName = r.channel;
621
+ // Use stored ID from JSON if available, fall back to SQLite row ID
622
+ if (typeof msg.id !== "number") {
623
+ msg.id = r.id;
624
+ }
625
+ messages.push(new Msg(msg));
443
626
  }
444
627
  return messages;
445
628
  }
446
- function necessaryMigrations(since) {
447
- return exports.migrations.filter((m) => m.version > since);
629
+ export function necessaryMigrations(since) {
630
+ return migrations.filter((m) => m.version > since);
448
631
  }
449
- exports.necessaryMigrations = necessaryMigrations;
450
- function newRollbacks(since) {
451
- return exports.rollbacks.filter((r) => r.version > since);
632
+ export function newRollbacks(since) {
633
+ return rollbacks.filter((r) => r.version > since);
452
634
  }
453
- exports.newRollbacks = newRollbacks;
454
- exports.default = SqliteMessageStorage;
635
+ export default SqliteMessageStorage;