noormme 1.2.6 → 1.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -133,18 +133,22 @@ class Cortex {
133
133
  this.executionLock = true;
134
134
  console.log('[Cortex] Initiating Autonomous Soul-Searching Loop v2 (Deep Hardening Pass)...');
135
135
  try {
136
+ // Phase 1: System Health & Diagnostic (Strict Transaction)
136
137
  await this.db.transaction().execute(async (trx) => {
137
- // 1. Audit health & Run self-tests
138
138
  await this.#runIsolated('Audit', () => this.governor.performAudit(trx));
139
139
  await this.#runIsolated('Self-Tests', () => this.tests.runAllProbes(trx));
140
- // 2. Run background rituals (optimization, compression)
141
- await this.#runIsolated('Rituals', () => this.rituals.runPendingRituals(trx));
142
- // 3. Learn from actions & Prune dead data
140
+ });
141
+ // Phase 2: Autonomous Rituals (Individual Transactional Isolation)
142
+ // We run these outside the main transaction to prevent long-running ritual locks
143
+ // from blocking the entire database.
144
+ await this.#runIsolated('Rituals', () => this.rituals.runPendingRituals());
145
+ // Phase 3: Background Maintenance (Unified Maintenance Transaction)
146
+ await this.db.transaction().execute(async (trx) => {
143
147
  await this.#runIsolated('Action Refinement', () => this.refiner.refineActions(trx));
144
148
  await this.#runIsolated('Zombie Pruning', () => this.ablation.pruneZombies(30, trx));
145
149
  await this.#runIsolated('Ablation Monitoring', () => this.ablation.monitorAblationPerformance(trx));
146
150
  });
147
- // These are often long-running or have their own internal transaction boundaries
151
+ // Phase 4: Long-Running Evolutionary Cycles (Internal transaction boundaries)
148
152
  await this.#runIsolated('Strategy Mutation', () => this.strategy.mutateStrategy());
149
153
  await this.#runIsolated('Evolution Pulse', () => this.evolutionRitual.execute());
150
154
  await this.#runIsolated('Knowledge Broadcast', () => this.hive.broadcastKnowledge());
@@ -16,7 +16,8 @@ export declare class RitualOrchestrator {
16
16
  */
17
17
  scheduleRitual(name: string, type: AgentRitual['type'], frequency: AgentRitual['frequency'], definition?: string, metadata?: Record<string, any>): Promise<AgentRitual>;
18
18
  /**
19
- * Run all pending rituals that are due
19
+ * Run all pending rituals that are due.
20
+ * PRODUCTION HARDENING: Separate Discovery/Locking from Execution to minimize lock duration.
20
21
  */
21
22
  runPendingRituals(trxOrDb?: any): Promise<number>;
22
23
  /**
@@ -37,7 +37,8 @@ class RitualOrchestrator {
37
37
  return this.parseRitual(ritual);
38
38
  }
39
39
  /**
40
- * Run all pending rituals that are due
40
+ * Run all pending rituals that are due.
41
+ * PRODUCTION HARDENING: Separate Discovery/Locking from Execution to minimize lock duration.
41
42
  */
42
43
  async runPendingRituals(trxOrDb = this.db) {
43
44
  const now = new Date();
@@ -56,7 +57,7 @@ class RitualOrchestrator {
56
57
  if (due.length === 0)
57
58
  return [];
58
59
  for (const ritual of due) {
59
- // Production Hardening: Distributed Lock
60
+ // Distributed Lock to prevent other nodes/instances from picking this up
60
61
  await trx
61
62
  .updateTable(this.ritualsTable)
62
63
  .set({ locked_until: lockTimeout })
@@ -65,15 +66,18 @@ class RitualOrchestrator {
65
66
  }
66
67
  return due;
67
68
  };
69
+ // DISCOVERY PHASE: Short transaction to find and lock pending work
68
70
  const pending = (trxOrDb !== this.db)
69
71
  ? await runner(trxOrDb)
70
72
  : await this.db.transaction().execute(runner);
71
73
  if (pending.length === 0)
72
74
  return 0;
73
- console.log(`[RitualOrchestrator] Found ${pending.length} pending rituals due. Locking for execution...`);
75
+ console.log(`[RitualOrchestrator] Found ${pending.length} pending rituals due. Executing in isolation...`);
76
+ // EXECUTION PHASE: Run rituals outside the discovery transaction.
77
+ // Each ritual will handle its own internal DB calls / transactions.
74
78
  for (const ritual of pending) {
75
- // Execute with the same trxOrDb context
76
- await this.executeRitual(ritual, trxOrDb);
79
+ // Use this.db directly to ensure we don't hold the parent transaction lock
80
+ await this.executeRitual(ritual, this.db);
77
81
  }
78
82
  return pending.length;
79
83
  }
@@ -1,16 +1,17 @@
1
1
  import type { Kysely } from '../../kysely.js';
2
2
  import { Logger } from '../../logging/logger.js';
3
3
  export interface SQLiteOptimizationConfig {
4
- enableAutoPragma: boolean;
5
- enableAutoIndexing: boolean;
6
- enablePerformanceTuning: boolean;
7
- enableBackupRecommendations: boolean;
8
- slowQueryThreshold: number;
9
- autoVacuumMode: 'NONE' | 'FULL' | 'INCREMENTAL';
10
- journalMode: 'DELETE' | 'TRUNCATE' | 'PERSIST' | 'MEMORY' | 'WAL' | 'OFF';
11
- synchronous: 'OFF' | 'NORMAL' | 'FULL' | 'EXTRA';
12
- cacheSize: number;
13
- tempStore: 'DEFAULT' | 'FILE' | 'MEMORY';
4
+ enableAutoPragma?: boolean;
5
+ enableAutoIndexing?: boolean;
6
+ enablePerformanceTuning?: boolean;
7
+ enableBackupRecommendations?: boolean;
8
+ slowQueryThreshold?: number;
9
+ autoVacuumMode?: 'NONE' | 'FULL' | 'INCREMENTAL';
10
+ journalMode?: 'DELETE' | 'TRUNCATE' | 'PERSIST' | 'MEMORY' | 'WAL' | 'OFF';
11
+ synchronous?: 'OFF' | 'NORMAL' | 'FULL' | 'EXTRA';
12
+ busyTimeout?: number;
13
+ cacheSize?: number;
14
+ tempStore?: 'DEFAULT' | 'FILE' | 'MEMORY';
14
15
  }
15
16
  export interface SQLiteOptimizationResult {
16
17
  appliedOptimizations: string[];
@@ -46,7 +47,7 @@ export declare class SQLiteAutoOptimizer {
46
47
  /**
47
48
  * Get default optimization configuration
48
49
  */
49
- getDefaultConfig(): SQLiteOptimizationConfig;
50
+ getDefaultConfig(): Required<SQLiteOptimizationConfig>;
50
51
  /**
51
52
  * Analyze current SQLite configuration and performance
52
53
  */
@@ -32,6 +32,7 @@ class SQLiteAutoOptimizer {
32
32
  autoVacuumMode: 'INCREMENTAL',
33
33
  journalMode: 'WAL',
34
34
  synchronous: 'NORMAL',
35
+ busyTimeout: 5000,
35
36
  cacheSize: -64000, // 64MB cache
36
37
  tempStore: 'MEMORY',
37
38
  };
@@ -83,7 +84,7 @@ class SQLiteAutoOptimizer {
83
84
  /**
84
85
  * Apply automatic optimizations based on configuration
85
86
  */
86
- async optimizeDatabase(db, config = this.getDefaultConfig()) {
87
+ async optimizeDatabase(db, config) {
87
88
  const result = {
88
89
  appliedOptimizations: [],
89
90
  recommendations: [],
@@ -91,15 +92,17 @@ class SQLiteAutoOptimizer {
91
92
  warnings: [],
92
93
  };
93
94
  try {
95
+ // Merge provided config with defaults
96
+ const finalConfig = { ...this.getDefaultConfig(), ...config };
94
97
  // Analyze current state
95
98
  const metrics = await this.analyzeDatabase(db);
96
99
  // Apply pragma optimizations
97
- if (config.enableAutoPragma) {
98
- await this.applyPragmaOptimizations(db, config, metrics, result);
100
+ if (finalConfig.enableAutoPragma) {
101
+ await this.applyPragmaOptimizations(db, finalConfig, metrics, result);
99
102
  }
100
103
  // Apply performance tuning
101
- if (config.enablePerformanceTuning) {
102
- await this.applyPerformanceTuning(db, config, metrics, result);
104
+ if (finalConfig.enablePerformanceTuning) {
105
+ await this.applyPerformanceTuning(db, finalConfig, metrics, result);
103
106
  }
104
107
  // Generate recommendations
105
108
  await this.generateRecommendations(db, metrics, result);
@@ -179,6 +182,17 @@ class SQLiteAutoOptimizer {
179
182
  result.warnings.push('Failed to set temp store');
180
183
  }
181
184
  }
185
+ // Set busy timeout
186
+ if (config.busyTimeout) {
187
+ try {
188
+ await (0, sql_js_1.sql) `PRAGMA busy_timeout = ${sql_js_1.sql.lit(config.busyTimeout)}`.execute(db);
189
+ optimizations.push(`Set busy timeout to ${config.busyTimeout}ms`);
190
+ result.performanceImpact = 'low';
191
+ }
192
+ catch (error) {
193
+ result.warnings.push('Failed to set busy timeout');
194
+ }
195
+ }
182
196
  result.appliedOptimizations.push(...optimizations);
183
197
  }
184
198
  /**
@@ -11,6 +11,12 @@ export interface SqliteDialectConfig {
11
11
  * https://github.com/JoshuaWise/better-sqlite3/blob/master/docs/api.md#new-databasepath-options
12
12
  */
13
13
  database: SqliteDatabase | (() => Promise<SqliteDatabase>);
14
+ /**
15
+ * Maximum number of concurrent connections to the database.
16
+ * Requires `database` to be a factory function `() => Promise<SqliteDatabase>`.
17
+ * Defaults to 1.
18
+ */
19
+ poolSize?: number;
14
20
  /**
15
21
  * Called once when the first query is executed.
16
22
  *
@@ -34,7 +34,7 @@ export declare class SqliteDriver implements Driver {
34
34
  /**
35
35
  * Releases a connection back to the pool.
36
36
  */
37
- releaseConnection(): Promise<void>;
37
+ releaseConnection(connection: DatabaseConnection): Promise<void>;
38
38
  /**
39
39
  * Destroys the driver and releases all resources.
40
40
  */
@@ -5,40 +5,103 @@ const savepoint_parser_js_1 = require("../../parser/savepoint-parser.js");
5
5
  const compiled_query_js_1 = require("../../query-compiler/compiled-query.js");
6
6
  const object_utils_js_1 = require("../../util/object-utils.js");
7
7
  const query_id_js_1 = require("../../util/query-id.js");
8
+ // Global WriteMutex registry.
9
+ // This serializes all writes to a specific SQLite file across multiple connections
10
+ // in the SAME Node.js process, preventing synchronous C-level event loop blocking.
11
+ const globalWriteMutexRegistry = new Map();
8
12
  class SqliteDriver {
9
13
  #config;
10
- #connectionMutex = new ConnectionMutex();
11
- #db;
12
- #connection;
14
+ #writeMutex;
15
+ #dbPath;
16
+ #connections = [];
17
+ #freeConnections = [];
18
+ #waiters = [];
19
+ #initialized = false;
13
20
  constructor(config) {
14
21
  this.#config = (0, object_utils_js_1.freeze)({ ...config });
15
22
  }
16
23
  async init() {
17
- this.#db = (0, object_utils_js_1.isFunction)(this.#config.database)
24
+ const poolSize = this.#config.poolSize || 1;
25
+ // Retrieve database name/path (fallback to memory if not accessible)
26
+ // We instantiate the first DB to get the path before building the pool
27
+ const firstDb = (0, object_utils_js_1.isFunction)(this.#config.database)
18
28
  ? await this.#config.database()
19
29
  : this.#config.database;
20
- this.#connection = new SqliteConnection(this.#db);
30
+ this.#dbPath = firstDb.name || ':memory:';
31
+ // Assign global write mutex keyed by path, with reference counting
32
+ const entry = globalWriteMutexRegistry.get(this.#dbPath);
33
+ if (entry) {
34
+ entry.refCount++;
35
+ this.#writeMutex = entry.mutex;
36
+ }
37
+ else {
38
+ const mutex = new ConnectionMutex();
39
+ globalWriteMutexRegistry.set(this.#dbPath, { mutex, refCount: 1 });
40
+ this.#writeMutex = mutex;
41
+ }
42
+ // Build the connection pool
43
+ const conn1 = new SqliteConnection(firstDb, this.#writeMutex);
44
+ this.#connections.push(conn1);
45
+ this.#freeConnections.push(conn1);
46
+ await this.#setupConnection(conn1);
47
+ // If using a factory function, we can create a real pool.
48
+ if (poolSize > 1 && (0, object_utils_js_1.isFunction)(this.#config.database)) {
49
+ for (let i = 1; i < poolSize; i++) {
50
+ const db = await this.#config.database();
51
+ const conn = new SqliteConnection(db, this.#writeMutex);
52
+ this.#connections.push(conn);
53
+ this.#freeConnections.push(conn);
54
+ await this.#setupConnection(conn);
55
+ }
56
+ }
57
+ this.#initialized = true;
58
+ }
59
+ async #setupConnection(conn) {
60
+ // Set baseline PRAGMAs for concurrency and performance
61
+ await conn.executeQuery(compiled_query_js_1.CompiledQuery.raw('pragma busy_timeout = 5000'));
62
+ await conn.executeQuery(compiled_query_js_1.CompiledQuery.raw('pragma journal_mode = WAL'));
63
+ await conn.executeQuery(compiled_query_js_1.CompiledQuery.raw('pragma synchronous = NORMAL'));
64
+ await conn.executeQuery(compiled_query_js_1.CompiledQuery.raw('pragma journal_size_limit = 67108864'));
65
+ await conn.executeQuery(compiled_query_js_1.CompiledQuery.raw('pragma temp_store = MEMORY'));
21
66
  if (this.#config.onCreateConnection) {
22
- await this.#config.onCreateConnection(this.#connection);
67
+ await this.#config.onCreateConnection(conn);
23
68
  }
24
69
  }
25
70
  async acquireConnection() {
26
- if (!this.#connection) {
27
- throw new Error('driver has already been destroyed');
71
+ if (!this.#initialized) {
72
+ throw new Error('driver has not been initialized or has already been destroyed');
73
+ }
74
+ if (this.#freeConnections.length > 0) {
75
+ return this.#freeConnections.pop();
28
76
  }
29
- // SQLite only has one single connection. We use a mutex here to wait
30
- // until the single connection has been released.
31
- await this.#connectionMutex.lock();
32
- return this.#connection;
77
+ // Wait for a connection to become available
78
+ return new Promise((resolve) => {
79
+ this.#waiters.push(resolve);
80
+ });
33
81
  }
34
82
  async beginTransaction(connection) {
35
- await connection.executeQuery(compiled_query_js_1.CompiledQuery.raw('begin'));
83
+ const sqliteConn = connection;
84
+ // Acquire the JS-level WriteMutex before executing BEGIN IMMEDIATE.
85
+ // This serializes all explicit transactions at the JS level.
86
+ await this.#writeMutex.lock();
87
+ sqliteConn.hasWriteMutex = true;
88
+ await connection.executeQuery(compiled_query_js_1.CompiledQuery.raw('begin immediate'));
36
89
  }
37
90
  async commitTransaction(connection) {
38
91
  await connection.executeQuery(compiled_query_js_1.CompiledQuery.raw('commit'));
92
+ const sqliteConn = connection;
93
+ if (sqliteConn.hasWriteMutex) {
94
+ sqliteConn.hasWriteMutex = false;
95
+ this.#writeMutex.unlock();
96
+ }
39
97
  }
40
98
  async rollbackTransaction(connection) {
41
99
  await connection.executeQuery(compiled_query_js_1.CompiledQuery.raw('rollback'));
100
+ const sqliteConn = connection;
101
+ if (sqliteConn.hasWriteMutex) {
102
+ sqliteConn.hasWriteMutex = false;
103
+ this.#writeMutex.unlock();
104
+ }
42
105
  }
43
106
  async savepoint(connection, savepointName, compileQuery) {
44
107
  await connection.executeQuery(compileQuery((0, savepoint_parser_js_1.parseSavepointCommand)('savepoint', savepointName), (0, query_id_js_1.createQueryId)()));
@@ -49,24 +112,52 @@ class SqliteDriver {
49
112
  async releaseSavepoint(connection, savepointName, compileQuery) {
50
113
  await connection.executeQuery(compileQuery((0, savepoint_parser_js_1.parseSavepointCommand)('release', savepointName), (0, query_id_js_1.createQueryId)()));
51
114
  }
52
- async releaseConnection() {
53
- this.#connectionMutex.unlock();
115
+ async releaseConnection(connection) {
116
+ const sqliteConn = connection;
117
+ // Safety check: ensure mutex is unlocked if connection is released abruptly
118
+ if (sqliteConn.hasWriteMutex) {
119
+ sqliteConn.hasWriteMutex = false;
120
+ this.#writeMutex.unlock();
121
+ }
122
+ if (this.#waiters.length > 0) {
123
+ const resolve = this.#waiters.shift();
124
+ resolve(sqliteConn);
125
+ }
126
+ else {
127
+ this.#freeConnections.push(sqliteConn);
128
+ }
54
129
  }
55
130
  async destroy() {
56
- if (this.#db) {
57
- this.#db.close();
58
- this.#db = undefined;
59
- this.#connection = undefined;
131
+ this.#initialized = false;
132
+ for (const conn of this.#connections) {
133
+ conn.close();
134
+ }
135
+ this.#connections = [];
136
+ this.#freeConnections = [];
137
+ this.#waiters = [];
138
+ // Update reference counting and clean up global registry
139
+ const entry = globalWriteMutexRegistry.get(this.#dbPath);
140
+ if (entry) {
141
+ entry.refCount--;
142
+ if (entry.refCount <= 0) {
143
+ globalWriteMutexRegistry.delete(this.#dbPath);
144
+ }
60
145
  }
61
146
  }
62
147
  }
63
148
  exports.SqliteDriver = SqliteDriver;
64
149
  class SqliteConnection {
65
- #db;
66
- constructor(db) {
67
- this.#db = db;
150
+ db;
151
+ writeMutex;
152
+ hasWriteMutex = false;
153
+ constructor(db, writeMutex) {
154
+ this.db = db;
155
+ this.writeMutex = writeMutex;
68
156
  }
69
- executeQuery(compiledQuery) {
157
+ close() {
158
+ this.db.close();
159
+ }
160
+ async executeQuery(compiledQuery) {
70
161
  const { sql, parameters } = compiledQuery;
71
162
  // Convert parameters to SQLite-compatible types
72
163
  const sqliteParameters = parameters.map((param) => {
@@ -84,20 +175,40 @@ class SqliteConnection {
84
175
  }
85
176
  return param;
86
177
  });
87
- const stmt = this.#db.prepare(sql);
178
+ const stmt = this.db.prepare(sql);
179
+ // If it's a read query, execute immediately concurrently
88
180
  if (stmt.reader) {
89
- return Promise.resolve({
181
+ return {
90
182
  rows: stmt.all(sqliteParameters),
91
- });
92
- }
93
- const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
94
- return Promise.resolve({
95
- numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
96
- insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
97
- ? BigInt(lastInsertRowid)
98
- : undefined,
99
- rows: [],
100
- });
183
+ };
184
+ }
185
+ // It's a write query!
186
+ if (!this.hasWriteMutex) {
187
+ // Not in an explicit transaction, so we must acquire the WriteMutex temporarily
188
+ await this.writeMutex.lock();
189
+ try {
190
+ const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
191
+ return {
192
+ numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
193
+ insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
194
+ ? BigInt(lastInsertRowid) : undefined,
195
+ rows: [],
196
+ };
197
+ }
198
+ finally {
199
+ this.writeMutex.unlock();
200
+ }
201
+ }
202
+ else {
203
+ // Already holding the WriteMutex via explicit transaction
204
+ const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
205
+ return {
206
+ numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
207
+ insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
208
+ ? BigInt(lastInsertRowid) : undefined,
209
+ rows: [],
210
+ };
211
+ }
101
212
  }
102
213
  async *streamQuery(compiledQuery, _chunkSize) {
103
214
  const { sql, parameters, query } = compiledQuery;
@@ -117,7 +228,7 @@ class SqliteConnection {
117
228
  }
118
229
  return param;
119
230
  });
120
- const stmt = this.#db.prepare(sql);
231
+ const stmt = this.db.prepare(sql);
121
232
  if (stmt.reader) {
122
233
  const iter = stmt.iterate(sqliteParameters);
123
234
  for (const row of iter) {
@@ -679,7 +679,7 @@ class NOORMME {
679
679
  switch (dialect) {
680
680
  case 'sqlite':
681
681
  return new sqlite_dialect_js_1.SqliteDialect({
682
- database: new better_sqlite3_1.default(connection.database),
682
+ database: new better_sqlite3_1.default(connection.database, { timeout: 5000 }),
683
683
  });
684
684
  case 'postgresql':
685
685
  return new postgresql_dialect_js_1.PostgresDialect({
@@ -131,18 +131,22 @@ export class Cortex {
131
131
  this.executionLock = true;
132
132
  console.log('[Cortex] Initiating Autonomous Soul-Searching Loop v2 (Deep Hardening Pass)...');
133
133
  try {
134
+ // Phase 1: System Health & Diagnostic (Strict Transaction)
134
135
  await this.db.transaction().execute(async (trx) => {
135
- // 1. Audit health & Run self-tests
136
136
  await this.#runIsolated('Audit', () => this.governor.performAudit(trx));
137
137
  await this.#runIsolated('Self-Tests', () => this.tests.runAllProbes(trx));
138
- // 2. Run background rituals (optimization, compression)
139
- await this.#runIsolated('Rituals', () => this.rituals.runPendingRituals(trx));
140
- // 3. Learn from actions & Prune dead data
138
+ });
139
+ // Phase 2: Autonomous Rituals (Individual Transactional Isolation)
140
+ // We run these outside the main transaction to prevent long-running ritual locks
141
+ // from blocking the entire database.
142
+ await this.#runIsolated('Rituals', () => this.rituals.runPendingRituals());
143
+ // Phase 3: Background Maintenance (Unified Maintenance Transaction)
144
+ await this.db.transaction().execute(async (trx) => {
141
145
  await this.#runIsolated('Action Refinement', () => this.refiner.refineActions(trx));
142
146
  await this.#runIsolated('Zombie Pruning', () => this.ablation.pruneZombies(30, trx));
143
147
  await this.#runIsolated('Ablation Monitoring', () => this.ablation.monitorAblationPerformance(trx));
144
148
  });
145
- // These are often long-running or have their own internal transaction boundaries
149
+ // Phase 4: Long-Running Evolutionary Cycles (Internal transaction boundaries)
146
150
  await this.#runIsolated('Strategy Mutation', () => this.strategy.mutateStrategy());
147
151
  await this.#runIsolated('Evolution Pulse', () => this.evolutionRitual.execute());
148
152
  await this.#runIsolated('Knowledge Broadcast', () => this.hive.broadcastKnowledge());
@@ -16,7 +16,8 @@ export declare class RitualOrchestrator {
16
16
  */
17
17
  scheduleRitual(name: string, type: AgentRitual['type'], frequency: AgentRitual['frequency'], definition?: string, metadata?: Record<string, any>): Promise<AgentRitual>;
18
18
  /**
19
- * Run all pending rituals that are due
19
+ * Run all pending rituals that are due.
20
+ * PRODUCTION HARDENING: Separate Discovery/Locking from Execution to minimize lock duration.
20
21
  */
21
22
  runPendingRituals(trxOrDb?: any): Promise<number>;
22
23
  /**
@@ -35,7 +35,8 @@ export class RitualOrchestrator {
35
35
  return this.parseRitual(ritual);
36
36
  }
37
37
  /**
38
- * Run all pending rituals that are due
38
+ * Run all pending rituals that are due.
39
+ * PRODUCTION HARDENING: Separate Discovery/Locking from Execution to minimize lock duration.
39
40
  */
40
41
  async runPendingRituals(trxOrDb = this.db) {
41
42
  const now = new Date();
@@ -54,7 +55,7 @@ export class RitualOrchestrator {
54
55
  if (due.length === 0)
55
56
  return [];
56
57
  for (const ritual of due) {
57
- // Production Hardening: Distributed Lock
58
+ // Distributed Lock to prevent other nodes/instances from picking this up
58
59
  await trx
59
60
  .updateTable(this.ritualsTable)
60
61
  .set({ locked_until: lockTimeout })
@@ -63,15 +64,18 @@ export class RitualOrchestrator {
63
64
  }
64
65
  return due;
65
66
  };
67
+ // DISCOVERY PHASE: Short transaction to find and lock pending work
66
68
  const pending = (trxOrDb !== this.db)
67
69
  ? await runner(trxOrDb)
68
70
  : await this.db.transaction().execute(runner);
69
71
  if (pending.length === 0)
70
72
  return 0;
71
- console.log(`[RitualOrchestrator] Found ${pending.length} pending rituals due. Locking for execution...`);
73
+ console.log(`[RitualOrchestrator] Found ${pending.length} pending rituals due. Executing in isolation...`);
74
+ // EXECUTION PHASE: Run rituals outside the discovery transaction.
75
+ // Each ritual will handle its own internal DB calls / transactions.
72
76
  for (const ritual of pending) {
73
- // Execute with the same trxOrDb context
74
- await this.executeRitual(ritual, trxOrDb);
77
+ // Use this.db directly to ensure we don't hold the parent transaction lock
78
+ await this.executeRitual(ritual, this.db);
75
79
  }
76
80
  return pending.length;
77
81
  }
@@ -1,16 +1,17 @@
1
1
  import type { Kysely } from '../../kysely.js';
2
2
  import { Logger } from '../../logging/logger.js';
3
3
  export interface SQLiteOptimizationConfig {
4
- enableAutoPragma: boolean;
5
- enableAutoIndexing: boolean;
6
- enablePerformanceTuning: boolean;
7
- enableBackupRecommendations: boolean;
8
- slowQueryThreshold: number;
9
- autoVacuumMode: 'NONE' | 'FULL' | 'INCREMENTAL';
10
- journalMode: 'DELETE' | 'TRUNCATE' | 'PERSIST' | 'MEMORY' | 'WAL' | 'OFF';
11
- synchronous: 'OFF' | 'NORMAL' | 'FULL' | 'EXTRA';
12
- cacheSize: number;
13
- tempStore: 'DEFAULT' | 'FILE' | 'MEMORY';
4
+ enableAutoPragma?: boolean;
5
+ enableAutoIndexing?: boolean;
6
+ enablePerformanceTuning?: boolean;
7
+ enableBackupRecommendations?: boolean;
8
+ slowQueryThreshold?: number;
9
+ autoVacuumMode?: 'NONE' | 'FULL' | 'INCREMENTAL';
10
+ journalMode?: 'DELETE' | 'TRUNCATE' | 'PERSIST' | 'MEMORY' | 'WAL' | 'OFF';
11
+ synchronous?: 'OFF' | 'NORMAL' | 'FULL' | 'EXTRA';
12
+ busyTimeout?: number;
13
+ cacheSize?: number;
14
+ tempStore?: 'DEFAULT' | 'FILE' | 'MEMORY';
14
15
  }
15
16
  export interface SQLiteOptimizationResult {
16
17
  appliedOptimizations: string[];
@@ -46,7 +47,7 @@ export declare class SQLiteAutoOptimizer {
46
47
  /**
47
48
  * Get default optimization configuration
48
49
  */
49
- getDefaultConfig(): SQLiteOptimizationConfig;
50
+ getDefaultConfig(): Required<SQLiteOptimizationConfig>;
50
51
  /**
51
52
  * Analyze current SQLite configuration and performance
52
53
  */
@@ -30,6 +30,7 @@ export class SQLiteAutoOptimizer {
30
30
  autoVacuumMode: 'INCREMENTAL',
31
31
  journalMode: 'WAL',
32
32
  synchronous: 'NORMAL',
33
+ busyTimeout: 5000,
33
34
  cacheSize: -64000, // 64MB cache
34
35
  tempStore: 'MEMORY',
35
36
  };
@@ -81,7 +82,7 @@ export class SQLiteAutoOptimizer {
81
82
  /**
82
83
  * Apply automatic optimizations based on configuration
83
84
  */
84
- async optimizeDatabase(db, config = this.getDefaultConfig()) {
85
+ async optimizeDatabase(db, config) {
85
86
  const result = {
86
87
  appliedOptimizations: [],
87
88
  recommendations: [],
@@ -89,15 +90,17 @@ export class SQLiteAutoOptimizer {
89
90
  warnings: [],
90
91
  };
91
92
  try {
93
+ // Merge provided config with defaults
94
+ const finalConfig = { ...this.getDefaultConfig(), ...config };
92
95
  // Analyze current state
93
96
  const metrics = await this.analyzeDatabase(db);
94
97
  // Apply pragma optimizations
95
- if (config.enableAutoPragma) {
96
- await this.applyPragmaOptimizations(db, config, metrics, result);
98
+ if (finalConfig.enableAutoPragma) {
99
+ await this.applyPragmaOptimizations(db, finalConfig, metrics, result);
97
100
  }
98
101
  // Apply performance tuning
99
- if (config.enablePerformanceTuning) {
100
- await this.applyPerformanceTuning(db, config, metrics, result);
102
+ if (finalConfig.enablePerformanceTuning) {
103
+ await this.applyPerformanceTuning(db, finalConfig, metrics, result);
101
104
  }
102
105
  // Generate recommendations
103
106
  await this.generateRecommendations(db, metrics, result);
@@ -177,6 +180,17 @@ export class SQLiteAutoOptimizer {
177
180
  result.warnings.push('Failed to set temp store');
178
181
  }
179
182
  }
183
+ // Set busy timeout
184
+ if (config.busyTimeout) {
185
+ try {
186
+ await sql `PRAGMA busy_timeout = ${sql.lit(config.busyTimeout)}`.execute(db);
187
+ optimizations.push(`Set busy timeout to ${config.busyTimeout}ms`);
188
+ result.performanceImpact = 'low';
189
+ }
190
+ catch (error) {
191
+ result.warnings.push('Failed to set busy timeout');
192
+ }
193
+ }
180
194
  result.appliedOptimizations.push(...optimizations);
181
195
  }
182
196
  /**
@@ -11,6 +11,12 @@ export interface SqliteDialectConfig {
11
11
  * https://github.com/JoshuaWise/better-sqlite3/blob/master/docs/api.md#new-databasepath-options
12
12
  */
13
13
  database: SqliteDatabase | (() => Promise<SqliteDatabase>);
14
+ /**
15
+ * Maximum number of concurrent connections to the database.
16
+ * Requires `database` to be a factory function `() => Promise<SqliteDatabase>`.
17
+ * Defaults to 1.
18
+ */
19
+ poolSize?: number;
14
20
  /**
15
21
  * Called once when the first query is executed.
16
22
  *
@@ -34,7 +34,7 @@ export declare class SqliteDriver implements Driver {
34
34
  /**
35
35
  * Releases a connection back to the pool.
36
36
  */
37
- releaseConnection(): Promise<void>;
37
+ releaseConnection(connection: DatabaseConnection): Promise<void>;
38
38
  /**
39
39
  * Destroys the driver and releases all resources.
40
40
  */
@@ -3,40 +3,103 @@ import { parseSavepointCommand } from '../../parser/savepoint-parser.js';
3
3
  import { CompiledQuery } from '../../query-compiler/compiled-query.js';
4
4
  import { freeze, isFunction } from '../../util/object-utils.js';
5
5
  import { createQueryId } from '../../util/query-id.js';
6
+ // Global WriteMutex registry.
7
+ // This serializes all writes to a specific SQLite file across multiple connections
8
+ // in the SAME Node.js process, preventing synchronous C-level event loop blocking.
9
+ const globalWriteMutexRegistry = new Map();
6
10
  export class SqliteDriver {
7
11
  #config;
8
- #connectionMutex = new ConnectionMutex();
9
- #db;
10
- #connection;
12
+ #writeMutex;
13
+ #dbPath;
14
+ #connections = [];
15
+ #freeConnections = [];
16
+ #waiters = [];
17
+ #initialized = false;
11
18
  constructor(config) {
12
19
  this.#config = freeze({ ...config });
13
20
  }
14
21
  async init() {
15
- this.#db = isFunction(this.#config.database)
22
+ const poolSize = this.#config.poolSize || 1;
23
+ // Retrieve database name/path (fallback to memory if not accessible)
24
+ // We instantiate the first DB to get the path before building the pool
25
+ const firstDb = isFunction(this.#config.database)
16
26
  ? await this.#config.database()
17
27
  : this.#config.database;
18
- this.#connection = new SqliteConnection(this.#db);
28
+ this.#dbPath = firstDb.name || ':memory:';
29
+ // Assign global write mutex keyed by path, with reference counting
30
+ const entry = globalWriteMutexRegistry.get(this.#dbPath);
31
+ if (entry) {
32
+ entry.refCount++;
33
+ this.#writeMutex = entry.mutex;
34
+ }
35
+ else {
36
+ const mutex = new ConnectionMutex();
37
+ globalWriteMutexRegistry.set(this.#dbPath, { mutex, refCount: 1 });
38
+ this.#writeMutex = mutex;
39
+ }
40
+ // Build the connection pool
41
+ const conn1 = new SqliteConnection(firstDb, this.#writeMutex);
42
+ this.#connections.push(conn1);
43
+ this.#freeConnections.push(conn1);
44
+ await this.#setupConnection(conn1);
45
+ // If using a factory function, we can create a real pool.
46
+ if (poolSize > 1 && isFunction(this.#config.database)) {
47
+ for (let i = 1; i < poolSize; i++) {
48
+ const db = await this.#config.database();
49
+ const conn = new SqliteConnection(db, this.#writeMutex);
50
+ this.#connections.push(conn);
51
+ this.#freeConnections.push(conn);
52
+ await this.#setupConnection(conn);
53
+ }
54
+ }
55
+ this.#initialized = true;
56
+ }
57
+ async #setupConnection(conn) {
58
+ // Set baseline PRAGMAs for concurrency and performance
59
+ await conn.executeQuery(CompiledQuery.raw('pragma busy_timeout = 5000'));
60
+ await conn.executeQuery(CompiledQuery.raw('pragma journal_mode = WAL'));
61
+ await conn.executeQuery(CompiledQuery.raw('pragma synchronous = NORMAL'));
62
+ await conn.executeQuery(CompiledQuery.raw('pragma journal_size_limit = 67108864'));
63
+ await conn.executeQuery(CompiledQuery.raw('pragma temp_store = MEMORY'));
19
64
  if (this.#config.onCreateConnection) {
20
- await this.#config.onCreateConnection(this.#connection);
65
+ await this.#config.onCreateConnection(conn);
21
66
  }
22
67
  }
23
68
  async acquireConnection() {
24
- if (!this.#connection) {
25
- throw new Error('driver has already been destroyed');
69
+ if (!this.#initialized) {
70
+ throw new Error('driver has not been initialized or has already been destroyed');
71
+ }
72
+ if (this.#freeConnections.length > 0) {
73
+ return this.#freeConnections.pop();
26
74
  }
27
- // SQLite only has one single connection. We use a mutex here to wait
28
- // until the single connection has been released.
29
- await this.#connectionMutex.lock();
30
- return this.#connection;
75
+ // Wait for a connection to become available
76
+ return new Promise((resolve) => {
77
+ this.#waiters.push(resolve);
78
+ });
31
79
  }
32
80
  async beginTransaction(connection) {
33
- await connection.executeQuery(CompiledQuery.raw('begin'));
81
+ const sqliteConn = connection;
82
+ // Acquire the JS-level WriteMutex before executing BEGIN IMMEDIATE.
83
+ // This serializes all explicit transactions at the JS level.
84
+ await this.#writeMutex.lock();
85
+ sqliteConn.hasWriteMutex = true;
86
+ await connection.executeQuery(CompiledQuery.raw('begin immediate'));
34
87
  }
35
88
  async commitTransaction(connection) {
36
89
  await connection.executeQuery(CompiledQuery.raw('commit'));
90
+ const sqliteConn = connection;
91
+ if (sqliteConn.hasWriteMutex) {
92
+ sqliteConn.hasWriteMutex = false;
93
+ this.#writeMutex.unlock();
94
+ }
37
95
  }
38
96
  async rollbackTransaction(connection) {
39
97
  await connection.executeQuery(CompiledQuery.raw('rollback'));
98
+ const sqliteConn = connection;
99
+ if (sqliteConn.hasWriteMutex) {
100
+ sqliteConn.hasWriteMutex = false;
101
+ this.#writeMutex.unlock();
102
+ }
40
103
  }
41
104
  async savepoint(connection, savepointName, compileQuery) {
42
105
  await connection.executeQuery(compileQuery(parseSavepointCommand('savepoint', savepointName), createQueryId()));
@@ -47,23 +110,51 @@ export class SqliteDriver {
47
110
  async releaseSavepoint(connection, savepointName, compileQuery) {
48
111
  await connection.executeQuery(compileQuery(parseSavepointCommand('release', savepointName), createQueryId()));
49
112
  }
50
- async releaseConnection() {
51
- this.#connectionMutex.unlock();
113
+ async releaseConnection(connection) {
114
+ const sqliteConn = connection;
115
+ // Safety check: ensure mutex is unlocked if connection is released abruptly
116
+ if (sqliteConn.hasWriteMutex) {
117
+ sqliteConn.hasWriteMutex = false;
118
+ this.#writeMutex.unlock();
119
+ }
120
+ if (this.#waiters.length > 0) {
121
+ const resolve = this.#waiters.shift();
122
+ resolve(sqliteConn);
123
+ }
124
+ else {
125
+ this.#freeConnections.push(sqliteConn);
126
+ }
52
127
  }
53
128
  async destroy() {
54
- if (this.#db) {
55
- this.#db.close();
56
- this.#db = undefined;
57
- this.#connection = undefined;
129
+ this.#initialized = false;
130
+ for (const conn of this.#connections) {
131
+ conn.close();
132
+ }
133
+ this.#connections = [];
134
+ this.#freeConnections = [];
135
+ this.#waiters = [];
136
+ // Update reference counting and clean up global registry
137
+ const entry = globalWriteMutexRegistry.get(this.#dbPath);
138
+ if (entry) {
139
+ entry.refCount--;
140
+ if (entry.refCount <= 0) {
141
+ globalWriteMutexRegistry.delete(this.#dbPath);
142
+ }
58
143
  }
59
144
  }
60
145
  }
61
146
  class SqliteConnection {
62
- #db;
63
- constructor(db) {
64
- this.#db = db;
147
+ db;
148
+ writeMutex;
149
+ hasWriteMutex = false;
150
+ constructor(db, writeMutex) {
151
+ this.db = db;
152
+ this.writeMutex = writeMutex;
65
153
  }
66
- executeQuery(compiledQuery) {
154
+ close() {
155
+ this.db.close();
156
+ }
157
+ async executeQuery(compiledQuery) {
67
158
  const { sql, parameters } = compiledQuery;
68
159
  // Convert parameters to SQLite-compatible types
69
160
  const sqliteParameters = parameters.map((param) => {
@@ -81,20 +172,40 @@ class SqliteConnection {
81
172
  }
82
173
  return param;
83
174
  });
84
- const stmt = this.#db.prepare(sql);
175
+ const stmt = this.db.prepare(sql);
176
+ // If it's a read query, execute immediately concurrently
85
177
  if (stmt.reader) {
86
- return Promise.resolve({
178
+ return {
87
179
  rows: stmt.all(sqliteParameters),
88
- });
89
- }
90
- const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
91
- return Promise.resolve({
92
- numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
93
- insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
94
- ? BigInt(lastInsertRowid)
95
- : undefined,
96
- rows: [],
97
- });
180
+ };
181
+ }
182
+ // It's a write query!
183
+ if (!this.hasWriteMutex) {
184
+ // Not in an explicit transaction, so we must acquire the WriteMutex temporarily
185
+ await this.writeMutex.lock();
186
+ try {
187
+ const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
188
+ return {
189
+ numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
190
+ insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
191
+ ? BigInt(lastInsertRowid) : undefined,
192
+ rows: [],
193
+ };
194
+ }
195
+ finally {
196
+ this.writeMutex.unlock();
197
+ }
198
+ }
199
+ else {
200
+ // Already holding the WriteMutex via explicit transaction
201
+ const { changes, lastInsertRowid } = stmt.run(sqliteParameters);
202
+ return {
203
+ numAffectedRows: changes !== undefined && changes !== null ? BigInt(changes) : undefined,
204
+ insertId: lastInsertRowid !== undefined && lastInsertRowid !== null
205
+ ? BigInt(lastInsertRowid) : undefined,
206
+ rows: [],
207
+ };
208
+ }
98
209
  }
99
210
  async *streamQuery(compiledQuery, _chunkSize) {
100
211
  const { sql, parameters, query } = compiledQuery;
@@ -114,7 +225,7 @@ class SqliteConnection {
114
225
  }
115
226
  return param;
116
227
  });
117
- const stmt = this.#db.prepare(sql);
228
+ const stmt = this.db.prepare(sql);
118
229
  if (stmt.reader) {
119
230
  const iter = stmt.iterate(sqliteParameters);
120
231
  for (const row of iter) {
@@ -674,7 +674,7 @@ export class NOORMME {
674
674
  switch (dialect) {
675
675
  case 'sqlite':
676
676
  return new SqliteDialect({
677
- database: new Database(connection.database),
677
+ database: new Database(connection.database, { timeout: 5000 }),
678
678
  });
679
679
  case 'postgresql':
680
680
  return new PostgresDialect({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "noormme",
3
- "version": "1.2.6",
3
+ "version": "1.2.7",
4
4
  "description": "NOORMME - The Agentic Data Engine. High-fidelity persistence and cognitive governance for Autonomous AI Agents.",
5
5
  "repository": {
6
6
  "type": "git",