bunsane 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. package/CHANGELOG.md +445 -318
  2. package/config/cache.config.ts +35 -1
  3. package/core/App.ts +24 -1064
  4. package/core/ArcheType.ts +78 -2110
  5. package/core/BatchLoader.ts +56 -32
  6. package/core/Entity.ts +85 -1043
  7. package/core/EntityHookManager.ts +52 -754
  8. package/core/Logger.ts +10 -0
  9. package/core/RequestContext.ts +64 -6
  10. package/core/RequestLoaders.ts +187 -36
  11. package/core/SchedulerManager.ts +28 -600
  12. package/core/app/bootstrap.ts +133 -0
  13. package/core/app/cors.ts +85 -0
  14. package/core/app/graphqlSetup.ts +56 -0
  15. package/core/app/healthEndpoints.ts +31 -0
  16. package/core/app/metricsCollector.ts +27 -0
  17. package/core/app/preparedStatementWarmup.ts +15 -0
  18. package/core/app/processHandlers.ts +43 -0
  19. package/core/app/requestRouter.ts +310 -0
  20. package/core/app/restRegistry.ts +80 -0
  21. package/core/app/shutdown.ts +97 -0
  22. package/core/app/studioRouter.ts +83 -0
  23. package/core/archetype/customTypes.ts +100 -0
  24. package/core/archetype/decorators.ts +171 -0
  25. package/core/archetype/fieldResolvers.ts +666 -0
  26. package/core/archetype/helpers.ts +29 -0
  27. package/core/archetype/relationLoader.ts +161 -0
  28. package/core/archetype/schemaBuilder.ts +141 -0
  29. package/core/archetype/weaver.ts +218 -0
  30. package/core/archetype/zodSchemaBuilder.ts +527 -0
  31. package/core/cache/CacheManager.ts +173 -267
  32. package/core/cache/CompressionUtils.ts +34 -3
  33. package/core/cache/MemoryCache.ts +40 -37
  34. package/core/cache/RedisCache.ts +4 -4
  35. package/core/cache/health.ts +30 -0
  36. package/core/cache/invalidation.ts +96 -0
  37. package/core/cache/strategies/writeInvalidate.ts +111 -0
  38. package/core/cache/strategies/writeThrough.ts +233 -0
  39. package/core/components/BaseComponent.ts +16 -8
  40. package/core/components/ComponentRegistry.ts +28 -0
  41. package/core/decorators/IndexedField.ts +1 -1
  42. package/core/entity/cacheStrategies.ts +97 -0
  43. package/core/entity/componentAccess.ts +364 -0
  44. package/core/entity/finders.ts +202 -0
  45. package/core/entity/pendingOps.ts +72 -0
  46. package/core/entity/saveEntity.ts +377 -0
  47. package/core/hooks/dispatcher.ts +439 -0
  48. package/core/hooks/guards.ts +155 -0
  49. package/core/hooks/registry.ts +247 -0
  50. package/core/metadata/definitions/Component.ts +1 -1
  51. package/core/metadata/index.ts +15 -4
  52. package/core/middleware/AccessLog.ts +8 -1
  53. package/core/middleware/RateLimit.ts +102 -105
  54. package/core/middleware/RequestId.ts +2 -9
  55. package/core/middleware/SecurityHeaders.ts +2 -11
  56. package/core/middleware/headers.ts +28 -0
  57. package/core/remote/OutboxWorker.ts +213 -183
  58. package/core/remote/RemoteManager.ts +401 -400
  59. package/core/remote/types.ts +153 -151
  60. package/core/requestScope.ts +34 -0
  61. package/core/scheduler/cronEvaluator.ts +174 -0
  62. package/core/scheduler/lifecycleHooks.ts +21 -0
  63. package/core/scheduler/lockCoordinator.ts +27 -0
  64. package/core/scheduler/metrics.ts +14 -0
  65. package/core/scheduler/taskRunner.ts +420 -0
  66. package/database/DatabaseHelper.ts +128 -101
  67. package/database/IndexingStrategy.ts +72 -2
  68. package/database/PreparedStatementCache.ts +20 -5
  69. package/database/cancellable.ts +35 -0
  70. package/database/index.ts +15 -3
  71. package/database/instrumentedDb.ts +141 -0
  72. package/endpoints/archetypes.ts +2 -8
  73. package/endpoints/tables.ts +6 -1
  74. package/gql/index.ts +1 -1
  75. package/gql/visitors/ResolverGeneratorVisitor.ts +25 -4
  76. package/package.json +22 -1
  77. package/query/CTENode.ts +5 -3
  78. package/query/ComponentInclusionNode.ts +240 -13
  79. package/query/OrNode.ts +6 -5
  80. package/query/Query.ts +203 -59
  81. package/query/QueryContext.ts +6 -0
  82. package/query/QueryDAG.ts +7 -2
  83. package/query/membershipSource.ts +66 -0
  84. package/storage/LocalStorageProvider.ts +8 -3
  85. package/studio/dist/assets/index-BMZ67Npg.js +254 -0
  86. package/studio/dist/assets/index-BpbuYz9g.css +1 -0
  87. package/studio/{index.html → dist/index.html} +3 -2
  88. package/swagger/generator.ts +11 -1
  89. package/upload/UploadManager.ts +8 -6
  90. package/utils/uuid.ts +40 -10
  91. package/.claude/settings.local.json +0 -47
  92. package/.prettierrc +0 -4
  93. package/.serena/memories/architectural-decision-no-dependency-injection.md +0 -76
  94. package/.serena/memories/architecture.md +0 -154
  95. package/.serena/memories/cache-interface-refactoring-2026-01-24.md +0 -165
  96. package/.serena/memories/code_style_and_conventions.md +0 -76
  97. package/.serena/memories/project_overview.md +0 -43
  98. package/.serena/memories/schema-dsl-plan.md +0 -107
  99. package/.serena/memories/suggested_commands.md +0 -80
  100. package/.serena/memories/typescript-compilation-status.md +0 -54
  101. package/.serena/project.yml +0 -114
  102. package/BunSane.jpg +0 -0
  103. package/CLAUDE.md +0 -198
  104. package/TODO.md +0 -2
  105. package/bun.lock +0 -302
  106. package/bunfig.toml +0 -10
  107. package/docs/SCALABILITY_PLAN.md +0 -175
  108. package/studio/bun.lock +0 -482
  109. package/studio/package.json +0 -39
  110. package/studio/postcss.config.js +0 -6
  111. package/studio/src/components/DataTable.tsx +0 -211
  112. package/studio/src/components/Layout.tsx +0 -13
  113. package/studio/src/components/PageContainer.tsx +0 -9
  114. package/studio/src/components/PageHeader.tsx +0 -13
  115. package/studio/src/components/SearchBar.tsx +0 -57
  116. package/studio/src/components/Sidebar.tsx +0 -294
  117. package/studio/src/components/ui/button.tsx +0 -56
  118. package/studio/src/components/ui/checkbox.tsx +0 -26
  119. package/studio/src/components/ui/input.tsx +0 -25
  120. package/studio/src/hooks/useDataTable.ts +0 -131
  121. package/studio/src/index.css +0 -36
  122. package/studio/src/lib/api.ts +0 -186
  123. package/studio/src/lib/utils.ts +0 -13
  124. package/studio/src/main.tsx +0 -17
  125. package/studio/src/pages/ArcheType.tsx +0 -239
  126. package/studio/src/pages/Components.tsx +0 -124
  127. package/studio/src/pages/EntityInspector.tsx +0 -302
  128. package/studio/src/pages/QueryRunner.tsx +0 -246
  129. package/studio/src/pages/Table.tsx +0 -94
  130. package/studio/src/pages/Welcome.tsx +0 -241
  131. package/studio/src/routes.tsx +0 -45
  132. package/studio/src/store/archeTypeSettings.ts +0 -30
  133. package/studio/src/store/studio.ts +0 -65
  134. package/studio/src/utils/columnHelpers.tsx +0 -114
  135. package/studio/studio-instructions.md +0 -81
  136. package/studio/tailwind.config.js +0 -77
  137. package/studio/utils.ts +0 -54
  138. package/studio/vite.config.js +0 -19
  139. package/tests/benchmark/BENCHMARK_DATABASES_PLAN.md +0 -338
  140. package/tests/benchmark/bunfig.toml +0 -9
  141. package/tests/benchmark/fixtures/EcommerceComponents.ts +0 -283
  142. package/tests/benchmark/fixtures/EcommerceDataGenerators.ts +0 -301
  143. package/tests/benchmark/fixtures/RelationTracker.ts +0 -159
  144. package/tests/benchmark/fixtures/index.ts +0 -6
  145. package/tests/benchmark/index.ts +0 -22
  146. package/tests/benchmark/noop-preload.ts +0 -3
  147. package/tests/benchmark/query-lateral-benchmark.test.ts +0 -372
  148. package/tests/benchmark/runners/BenchmarkLoader.ts +0 -132
  149. package/tests/benchmark/runners/index.ts +0 -4
  150. package/tests/benchmark/scenarios/query-benchmarks.test.ts +0 -465
  151. package/tests/benchmark/scripts/generate-db.ts +0 -344
  152. package/tests/benchmark/scripts/run-benchmarks.ts +0 -97
  153. package/tests/e2e/http.test.ts +0 -130
  154. package/tests/fixtures/archetypes/TestUserArchetype.ts +0 -21
  155. package/tests/fixtures/components/TestOrder.ts +0 -23
  156. package/tests/fixtures/components/TestProduct.ts +0 -23
  157. package/tests/fixtures/components/TestUser.ts +0 -20
  158. package/tests/fixtures/components/index.ts +0 -6
  159. package/tests/graphql/SchemaGeneration.test.ts +0 -90
  160. package/tests/graphql/builders/ResolverBuilder.test.ts +0 -223
  161. package/tests/graphql/builders/TypeDefBuilder.test.ts +0 -153
  162. package/tests/helpers/MockRedisClient.ts +0 -113
  163. package/tests/helpers/MockRedisStreamServer.ts +0 -448
  164. package/tests/integration/archetype/ArcheType.persistence.test.ts +0 -241
  165. package/tests/integration/cache/CacheInvalidation.test.ts +0 -259
  166. package/tests/integration/entity/Entity.persistence.test.ts +0 -333
  167. package/tests/integration/entity/Entity.saveTimeout.test.ts +0 -110
  168. package/tests/integration/query/Query.complexAnalysis.test.ts +0 -557
  169. package/tests/integration/query/Query.edgeCases.test.ts +0 -595
  170. package/tests/integration/query/Query.exec.test.ts +0 -576
  171. package/tests/integration/query/Query.explainAnalyze.test.ts +0 -233
  172. package/tests/integration/query/Query.jsonbArray.test.ts +0 -214
  173. package/tests/integration/remote/dlq.test.ts +0 -175
  174. package/tests/integration/remote/event-dispatch.test.ts +0 -114
  175. package/tests/integration/remote/outbox.test.ts +0 -130
  176. package/tests/integration/remote/rpc.test.ts +0 -177
  177. package/tests/pglite-setup.ts +0 -62
  178. package/tests/setup.ts +0 -164
  179. package/tests/stress/BenchmarkRunner.ts +0 -203
  180. package/tests/stress/DataSeeder.ts +0 -190
  181. package/tests/stress/StressTestReporter.ts +0 -229
  182. package/tests/stress/cursor-perf-test.ts +0 -171
  183. package/tests/stress/fixtures/RealisticComponents.ts +0 -235
  184. package/tests/stress/fixtures/StressTestComponents.ts +0 -58
  185. package/tests/stress/index.ts +0 -7
  186. package/tests/stress/scenarios/query-benchmarks.test.ts +0 -285
  187. package/tests/stress/scenarios/realistic-scenarios.test.ts +0 -1081
  188. package/tests/stress/scenarios/timeout-investigation.test.ts +0 -522
  189. package/tests/unit/BatchLoader.test.ts +0 -196
  190. package/tests/unit/archetype/ArcheType.test.ts +0 -107
  191. package/tests/unit/cache/CacheManager.test.ts +0 -367
  192. package/tests/unit/cache/MemoryCache.test.ts +0 -260
  193. package/tests/unit/cache/RedisCache.test.ts +0 -411
  194. package/tests/unit/entity/Entity.components.test.ts +0 -317
  195. package/tests/unit/entity/Entity.drainSideEffects.test.ts +0 -51
  196. package/tests/unit/entity/Entity.reload.test.ts +0 -63
  197. package/tests/unit/entity/Entity.requireComponents.test.ts +0 -72
  198. package/tests/unit/entity/Entity.test.ts +0 -345
  199. package/tests/unit/gql/depthLimit.test.ts +0 -203
  200. package/tests/unit/gql/operationMiddleware.test.ts +0 -293
  201. package/tests/unit/health/Health.test.ts +0 -129
  202. package/tests/unit/middleware/AccessLog.test.ts +0 -37
  203. package/tests/unit/middleware/Middleware.test.ts +0 -98
  204. package/tests/unit/middleware/RequestId.test.ts +0 -54
  205. package/tests/unit/middleware/SecurityHeaders.test.ts +0 -66
  206. package/tests/unit/query/FilterBuilder.test.ts +0 -111
  207. package/tests/unit/query/JsonbArrayBuilder.test.ts +0 -178
  208. package/tests/unit/query/Query.emptyString.test.ts +0 -69
  209. package/tests/unit/query/Query.test.ts +0 -310
  210. package/tests/unit/remote/CircuitBreaker.test.ts +0 -159
  211. package/tests/unit/remote/RemoteError.test.ts +0 -55
  212. package/tests/unit/remote/decorators.test.ts +0 -195
  213. package/tests/unit/remote/metrics.test.ts +0 -115
  214. package/tests/unit/remote/mockRedisStreamServer.test.ts +0 -104
  215. package/tests/unit/scheduler/DistributedLock.test.ts +0 -274
  216. package/tests/unit/scheduler/SchedulerManager.timeBased.test.ts +0 -95
  217. package/tests/unit/schema/schema-integration.test.ts +0 -426
  218. package/tests/unit/schema/schema.test.ts +0 -580
  219. package/tests/unit/storage/S3StorageProvider.test.ts +0 -567
  220. package/tests/unit/upload/RestUpload.test.ts +0 -267
  221. package/tests/unit/validateEnv.test.ts +0 -82
  222. package/tests/utils/entity-tracker.ts +0 -57
  223. package/tests/utils/index.ts +0 -13
  224. package/tests/utils/test-context.ts +0 -149
@@ -2,6 +2,7 @@ import db from "./index";
2
2
  import { logger as MainLogger } from "../core/Logger";
3
3
  import { getMetadataStorage } from "../core/metadata";
4
4
  import { ensureMultipleJSONBPathIndexes } from "./IndexingStrategy";
5
+ import { getMembershipTable } from "../query/membershipSource";
5
6
  const logger = MainLogger.child({ scope: "DatabaseHelper" });
6
7
 
7
8
  const BUNSANE_RELATION_TYPED_COLUMN = process.env.BUNSANE_RELATION_TYPED_COLUMN === 'true' || false;
@@ -49,7 +50,7 @@ export const GetSchema = async () => {
49
50
 
50
51
  export const HasValidBaseTable = async (): Promise<boolean> => {
51
52
  const tables = await GetSchema();
52
- const neededTables = ["entities", "components", "entity_components"];
53
+ const neededTables = ["entities", "components"];
53
54
  return neededTables.every(t => tables.includes(t));
54
55
  }
55
56
 
@@ -73,13 +74,9 @@ export const PrepareDatabase = async () => {
73
74
  logger.error(`Failed to create component table: ${error}`);
74
75
  throw error;
75
76
  }
76
- try {
77
- await CreateEntityComponentTable();
78
- await PopulateComponentIds();
79
- } catch (error) {
80
- logger.error(`Failed to create entity component table: ${error}`);
81
- throw error;
82
- }
77
+ // `entity_components` is no longer created or written. `components`
78
+ // (UNIQUE(entity_id, type_id)) is the single source of membership truth
79
+ // as of Phase 3 of docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md.
83
80
  }
84
81
 
85
82
  export const GetDatabaseDataSize = async () => {
@@ -131,27 +128,17 @@ export const CreateComponentTable = async () => {
131
128
 
132
129
  // If the table exists but has a different partitioning strategy, we need to recreate it
133
130
  if (tableExists.length > 0 && existingStrategy !== partitionStrategy) {
134
- logger.info(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
131
+ await assertComponentDataSafeToDrop(existingStrategy, partitionStrategy);
132
+ logger.warn(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
135
133
 
136
134
  // Drop the existing table and all its partitions
137
135
  await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
138
136
 
139
137
  // Also clean up any orphaned partition tables
140
- const orphanedPartitions = await db.unsafe(`
141
- SELECT tablename
142
- FROM pg_tables
143
- WHERE tablename LIKE 'components_%'
144
- AND schemaname = 'public'
145
- `);
146
-
147
- for (const partition of orphanedPartitions) {
148
- await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
149
- }
138
+ await dropOrphanedPartitionTables();
150
139
  }
151
140
 
152
141
  if (partitionStrategy === 'hash') {
153
- // Clean up any existing LIST partition tables before creating HASH partitions
154
- await cleanupOldListPartitions();
155
142
  await CreateHashPartitionedComponentTable();
156
143
  } else {
157
144
  // Original LIST partitioning
@@ -178,41 +165,64 @@ export const CreateComponentTable = async () => {
178
165
  }
179
166
  }
180
167
 
181
- const cleanupOldListPartitions = async () => {
168
+ /**
169
+ * Pure decision: may the components table be dropped for a partition strategy
170
+ * switch? Returns null when safe, otherwise the refusal error message.
171
+ */
172
+ export const partitionRecreateRefusal = (
173
+ hasData: boolean,
174
+ forceFlag: string | undefined,
175
+ existingStrategy: string | null,
176
+ requestedStrategy: string
177
+ ): string | null => {
178
+ if (!hasData) return null;
179
+ if (forceFlag === 'true') return null;
180
+ return (
181
+ `Refusing to recreate 'components' table: partition strategy changed from '${existingStrategy}' to '${requestedStrategy}' but the table contains data. ` +
182
+ `Recreating would permanently delete all component data. Options: ` +
183
+ `(1) set BUNSANE_PARTITION_STRATEGY back to '${existingStrategy}' to keep the current layout; ` +
184
+ `(2) back up component data, then restart once with BUNSANE_FORCE_PARTITION_RECREATE=true to drop and recreate (DESTRUCTIVE); ` +
185
+ `(3) migrate the data manually to the new layout before switching.`
186
+ );
187
+ }
188
+
189
+ export const assertComponentDataSafeToDrop = async (existingStrategy: string | null, requestedStrategy: string) => {
190
+ let hasData = false;
182
191
  try {
183
- logger.info(`Cleaning up old LIST partition tables before creating HASH partitions`);
184
-
185
- // Get all existing LIST partition tables
186
- const existingPartitions = await db.unsafe(`
187
- SELECT tablename
188
- FROM pg_tables
189
- WHERE tablename LIKE 'components_%'
190
- AND schemaname = 'public'
191
- AND tablename != 'components'
192
- `);
193
-
194
- for (const row of existingPartitions) {
195
- const tableName = row.tablename;
196
- logger.trace(`Dropping old LIST partition table: ${tableName}`);
197
- await db.unsafe(`DROP TABLE IF EXISTS ${tableName} CASCADE`);
198
- }
199
-
200
- // Drop the main components table if it exists (to recreate with HASH partitioning)
201
- const mainTableExists = await db.unsafe(`
202
- SELECT 1 FROM information_schema.tables
203
- WHERE table_name = 'components'
204
- AND table_schema = 'public'
205
- `);
206
-
207
- if (mainTableExists.length > 0) {
208
- logger.trace(`Dropping existing components table for HASH partitioning`);
209
- await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
210
- }
211
-
212
- logger.info(`Cleaned up ${existingPartitions.length} old partition tables`);
192
+ const rows = await db.unsafe(`SELECT 1 FROM components LIMIT 1`);
193
+ hasData = rows.length > 0;
213
194
  } catch (error) {
214
- logger.warn(`Could not clean up old LIST partitions: ${error}`);
215
- // Continue anyway - the table creation might still work
195
+ logger.warn(`Could not check components table for data before recreate: ${error}`);
196
+ }
197
+
198
+ const refusal = partitionRecreateRefusal(
199
+ hasData,
200
+ process.env.BUNSANE_FORCE_PARTITION_RECREATE,
201
+ existingStrategy,
202
+ requestedStrategy
203
+ );
204
+ if (refusal) throw new Error(refusal);
205
+
206
+ if (hasData) {
207
+ logger.warn(`BUNSANE_FORCE_PARTITION_RECREATE=true — dropping components table WITH DATA to switch partition strategy from '${existingStrategy}' to '${requestedStrategy}'. This permanently deletes all component data.`);
208
+ }
209
+ }
210
+
211
+ const dropOrphanedPartitionTables = async () => {
212
+ const orphanedPartitions = await db.unsafe(`
213
+ SELECT tablename
214
+ FROM pg_tables
215
+ WHERE tablename LIKE 'components_%'
216
+ AND schemaname = 'public'
217
+ AND tablename != 'components'
218
+ `);
219
+
220
+ for (const partition of orphanedPartitions) {
221
+ await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
222
+ }
223
+
224
+ if (orphanedPartitions.length > 0) {
225
+ logger.info(`Cleaned up ${orphanedPartitions.length} orphaned partition tables`);
216
226
  }
217
227
  }
218
228
 
@@ -471,58 +481,75 @@ export const CreateEntityComponentTable = async () => {
471
481
  }
472
482
  }
473
483
 
484
+ /**
485
+ * Rollback/repair tool. Backfills the legacy `entity_components` mirror from
486
+ * `components` (the single source of truth as of Phase 3). Only needed if you
487
+ * intend to downgrade to a build that still reads `entity_components`
488
+ * (BUNSANE_MEMBERSHIP_SOURCE=legacy).
489
+ *
490
+ * The framework no longer creates `entity_components` on boot. If the table is
491
+ * absent this throws a clear error: create it first via
492
+ * `CreateEntityComponentTable()`, then re-run this.
493
+ *
494
+ * Intended for a freshly-created/empty table: ON CONFLICT DO NOTHING skips
495
+ * pre-existing rows, so `deleted_at` drift on them is not reconciled.
496
+ */
474
497
  export const PopulateComponentIds = async () => {
498
+ const tableExists = await db.unsafe(`
499
+ SELECT 1 FROM information_schema.tables
500
+ WHERE table_name = 'entity_components'
501
+ AND table_schema = 'public'
502
+ `);
503
+ if (tableExists.length === 0) {
504
+ throw new Error(
505
+ `Cannot populate entity_components: the table does not exist. ` +
506
+ `It is no longer created since Phase 3 of the entity_components removal. ` +
507
+ `If you need the legacy mirror (e.g. for a downgrade), run ` +
508
+ `CreateEntityComponentTable() first, then re-run PopulateComponentIds().`
509
+ );
510
+ }
475
511
  try {
476
- // Populate component_id for existing rows that don't have it set
477
- await db`UPDATE entity_components
478
- SET component_id = c.id
479
- FROM components c
480
- WHERE entity_components.entity_id = c.entity_id
481
- AND entity_components.type_id = c.type_id
512
+ // Backfill membership rows from components, then set component_id for
513
+ // any rows still missing it.
514
+ await db`INSERT INTO entity_components (entity_id, type_id, component_id, deleted_at)
515
+ SELECT c.entity_id, c.type_id, c.id, c.deleted_at
516
+ FROM components c
517
+ ON CONFLICT (entity_id, type_id) DO NOTHING`;
518
+ await db`UPDATE entity_components
519
+ SET component_id = c.id
520
+ FROM components c
521
+ WHERE entity_components.entity_id = c.entity_id
522
+ AND entity_components.type_id = c.type_id
482
523
  AND entity_components.component_id IS NULL`;
483
-
484
- logger.info(`Populated component_id for existing entity_components rows`);
524
+
525
+ logger.info(`Backfilled entity_components from components`);
485
526
  } catch (error) {
486
- logger.warn(`Could not populate component_id for existing rows: ${error}`);
527
+ logger.warn(`Could not backfill entity_components: ${error}`);
528
+ throw error;
487
529
  }
488
530
  }
489
531
 
490
532
  export const EnsureDatabaseMigrations = async () => {
491
533
  logger.trace(`Checking for database migrations...`);
492
-
493
- try {
494
- // First, ensure the table exists and has the basic structure
495
- await CreateEntityComponentTable();
496
-
497
- // Check if entity_components table has component_id column
498
- const columnCheck = await db`SELECT column_name FROM information_schema.columns
499
- WHERE table_name = 'entity_components'
500
- AND column_name = 'component_id'
501
- AND table_schema = 'public'`;
502
-
503
- if (columnCheck.length === 0) {
504
- logger.info(`entity_components table missing component_id column, adding it...`);
505
- // Add the column
506
- await db`ALTER TABLE entity_components ADD COLUMN component_id UUID`;
507
- logger.info(`Added component_id column to entity_components table`);
508
-
509
- // Wait a bit for the column to be available
510
- await new Promise(resolve => setTimeout(resolve, 500));
511
-
512
- // Populate existing data
513
- await PopulateComponentIds();
514
- } else {
515
- logger.trace(`entity_components table already has component_id column`);
516
- }
517
- } catch (error) {
518
- logger.error(`Failed during database migration: ${error}`);
519
- // Try to add the column anyway in case the check failed
520
- try {
521
- await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
522
- logger.info(`Attempted to add component_id column as fallback`);
523
- } catch (fallbackError) {
524
- logger.error(`Fallback column addition also failed: ${fallbackError}`);
525
- }
534
+
535
+ // `entity_components` is no longer created, migrated, or written (Phase 3
536
+ // of docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md). Any pre-existing table is
537
+ // left in place, untouched — never auto-dropped. Membership now lives
538
+ // solely in `components` (UNIQUE(entity_id, type_id)). Run
539
+ // `PopulateComponentIds()` manually to backfill the legacy table if a
540
+ // downgrade is ever required.
541
+ const orphanCheck = await db.unsafe(`
542
+ SELECT 1 FROM information_schema.tables
543
+ WHERE table_name = 'entity_components'
544
+ AND table_schema = 'public'
545
+ `);
546
+ if (orphanCheck.length > 0) {
547
+ logger.info(
548
+ `[entity_components] Orphaned table detected. ` +
549
+ `This table is no longer used by the framework (see docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md). ` +
550
+ `Verify your upgrade succeeded (run a smoke-test query against the 'components' table), ` +
551
+ `then drop the orphan manually: DROP TABLE entity_components;`
552
+ );
526
553
  }
527
554
  }
528
555
 
@@ -621,10 +648,10 @@ export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 1
621
648
  await db.unsafe(`ANALYZE ${tempTableName}`);
622
649
 
623
650
  // Run benchmark query
624
- const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
625
- SELECT DISTINCT ec.entity_id as id
626
- FROM entity_components ec
627
- WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1)
651
+ const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
652
+ SELECT DISTINCT ec.entity_id as id
653
+ FROM ${getMembershipTable()} ec
654
+ WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1)
628
655
  AND ec.deleted_at IS NULL`);
629
656
 
630
657
  const plan = explainResult[0]['QUERY PLAN'] ? JSON.parse(explainResult[0]['QUERY PLAN']) : explainResult[0];
@@ -11,7 +11,7 @@ const validateIdentifier = (str: string, maxLength: number = 64): string => {
11
11
  return str;
12
12
  };
13
13
 
14
- export type IndexType = 'gin' | 'btree' | 'hash' | 'numeric';
14
+ export type IndexType = 'gin' | 'btree' | 'hash' | 'numeric' | 'fulltext';
15
15
 
16
16
  export interface IndexDefinition {
17
17
  tableName: string;
@@ -140,8 +140,9 @@ export const ensureMultipleJSONBPathIndexes = async (
140
140
  ): Promise<void> => {
141
141
  for (const def of indexDefinitions) {
142
142
  if (def.indexType === 'numeric') {
143
- // Use numeric index for range queries
144
143
  await ensureNumericIndex(def.tableName, def.field);
144
+ } else if (def.indexType === 'fulltext') {
145
+ await ensureFullTextIndex(def.tableName, def.field);
145
146
  } else {
146
147
  await ensureJSONBPathIndex(
147
148
  def.tableName,
@@ -241,6 +242,75 @@ export const ensureNumericIndex = async (
241
242
  }
242
243
  };
243
244
 
245
+ /**
246
+ * Creates a GIN index on a JSONB field for full-text search using to_tsvector.
247
+ *
248
+ * The index expression MUST match FullTextSearchBuilder's vectorSql exactly:
249
+ * to_tsvector('<language>', <alias>.data->'<field>')
250
+ * so PostgreSQL can use this index for those predicates.
251
+ *
252
+ * @param tableName The table name to create index on
253
+ * @param field The JSONB field path containing text for full-text search
254
+ * @param language PostgreSQL text search language (default 'english')
255
+ */
256
+ export const ensureFullTextIndex = async (
257
+ tableName: string,
258
+ field: string,
259
+ language: string = 'english'
260
+ ): Promise<void> => {
261
+ tableName = validateIdentifier(tableName);
262
+ field = validateIdentifier(field);
263
+
264
+ const indexName = `idx_${tableName}_${field}_fts`;
265
+
266
+ try {
267
+ logger.trace(`Ensuring full-text GIN index ${indexName} on ${tableName} for field ${field} (language: ${language})`);
268
+
269
+ const existingIndexes = await db.unsafe(`
270
+ SELECT indexname
271
+ FROM pg_indexes
272
+ WHERE tablename = '${tableName}' AND indexname = '${indexName}'
273
+ `);
274
+
275
+ if (existingIndexes.length > 0) {
276
+ logger.trace(`Index ${indexName} already exists`);
277
+ return;
278
+ }
279
+
280
+ const partitionCheck = await db.unsafe(`
281
+ SELECT relkind
282
+ FROM pg_class
283
+ WHERE relname = '${tableName}' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
284
+ `);
285
+
286
+ const isPartitioned = partitionCheck.length > 0 && partitionCheck[0].relkind === 'p';
287
+ const useConcurrently = !isPartitioned && !process.env.USE_PGLITE;
288
+
289
+ // Expression matches FullTextSearchBuilder.vectorSql: to_tsvector('<lang>', data->'<field>')
290
+ // The -> operator (not ->>) returns jsonb; PostgreSQL casts jsonb text to tsvector input.
291
+ const indexSQL = `CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName} ON ${tableName} USING GIN (to_tsvector('${language}', data->'${field}'))`;
292
+
293
+ logger.trace(`Creating full-text index with SQL: ${indexSQL}`);
294
+ await db.unsafe(indexSQL);
295
+ logger.info(`Created full-text GIN index ${indexName} on ${tableName}${useConcurrently ? ' (concurrently)' : ' (blocking)'}`);
296
+
297
+ } catch (error: any) {
298
+ if (error.message && (
299
+ error.message.includes('already exists') ||
300
+ error.code === '42P07'
301
+ )) {
302
+ logger.trace(`Index ${indexName} already exists (confirmed by error), skipping creation`);
303
+ return;
304
+ }
305
+ if (error.code === '40P01' || (error.message && error.message.includes('deadlock'))) {
306
+ logger.warn(`Deadlock detected while creating index ${indexName}, skipping`);
307
+ return;
308
+ }
309
+ logger.error(`Failed to create full-text index on ${tableName} for field ${field}: ${error}`);
310
+ throw error;
311
+ }
312
+ };
313
+
244
314
  /**
245
315
  * Creates a composite index on multiple JSONB fields for efficient combined filter queries.
246
316
  * Useful for queries like `WHERE status = 'active' AND age >= 21`
@@ -1,4 +1,5 @@
1
1
  import { logger } from "../core/Logger";
2
+ import { timedUnsafe, type PerRequestCounters } from "./instrumentedDb";
2
3
 
3
4
  export interface CacheEntry {
4
5
  sql: string;
@@ -19,8 +20,14 @@ export interface CacheStats {
19
20
  }
20
21
 
21
22
  /**
22
- * LRU Cache for prepared statements to eliminate PostgreSQL planning overhead
23
- * for repeated query patterns in the Bunsane Query system.
23
+ * @deprecated No longer used on the query hot path. This cache never called
24
+ * a prepare API it stored `{ sql, _isPrepared: true }` placeholders and
25
+ * every "hit" still executed `db.unsafe(sql, params)`, so PostgreSQL
26
+ * re-planned regardless. Bun SQL auto-prepares parameterized statements per
27
+ * connection (prepare:true default), providing real server-side plan reuse
28
+ * at the driver layer. The class is kept so `Query.getCacheStats()`,
29
+ * `/metrics`, and registry invalidation call sites remain stable; it now
30
+ * reports an idle cache.
24
31
  */
25
32
  export class PreparedStatementCache {
26
33
  private cache: Map<string, CacheEntry> = new Map();
@@ -108,15 +115,23 @@ export class PreparedStatementCache {
108
115
  }
109
116
 
110
117
  /**
111
- * Execute a prepared statement with parameters
118
+ * Execute a prepared statement with parameters. Routes through
119
+ * `timedUnsafe` so the call is timed and (when a signal is supplied)
120
+ * cancellable via Bun's `Query.cancel()` on abort.
112
121
  */
113
- public async execute(statement: any, params: any[], db: any): Promise<any[]> {
122
+ public async execute(
123
+ statement: any,
124
+ params: any[],
125
+ db: any,
126
+ signal?: AbortSignal,
127
+ perRequest?: PerRequestCounters,
128
+ ): Promise<any[]> {
114
129
  // Empty-string params are legitimate for text-field filters
115
130
  // (`c.data->>'field' = ''`). UUID-typed params never reach this
116
131
  // point empty — callers (Query.findById etc.) guard at entry. PG
117
132
  // emits a clear error at execution time if a UUID cast meets an
118
133
  // empty string.
119
- return await db.unsafe(statement.sql, params);
134
+ return await timedUnsafe<any[]>(db, statement.sql, params, signal, perRequest);
120
135
  }
121
136
 
122
137
  /**
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Wraps a Bun SQL Query so an AbortSignal can cancel the in-flight query
3
+ * via the underlying `query.cancel()` method. When the signal fires the
4
+ * server-side query receives a cancel request, the awaited promise rejects,
5
+ * any enclosing transaction triggers ROLLBACK, and the pooled backend
6
+ * connection is released. Without this, a wall-clock timeout leaks the
7
+ * backend into `idle in transaction` under pgbouncer transaction-mode.
8
+ *
9
+ * Rejection on abort is immediate (raced) rather than waiting for the
10
+ * driver to honor the cancel — some drivers (PGlite socket bridge) ignore
11
+ * `cancel()` entirely and would otherwise hang the caller until the query
12
+ * finishes on its own. The query's eventual settle is swallowed so it can't
13
+ * surface as an unhandled rejection after the race is lost.
14
+ */
15
+ export async function runWithSignal<T>(q: any, signal?: AbortSignal): Promise<T> {
16
+ if (!signal) return await q;
17
+ if (signal.aborted) {
18
+ try { q.cancel?.(); } catch { /* ignore */ }
19
+ throw signal.reason ?? new Error('Query aborted');
20
+ }
21
+ let onAbort: (() => void) | undefined;
22
+ const abortPromise = new Promise<never>((_, reject) => {
23
+ onAbort = () => {
24
+ try { q.cancel?.(); } catch { /* ignore */ }
25
+ Promise.resolve(q).catch(() => { /* swallow post-abort settle */ });
26
+ reject(signal.reason ?? new Error('Query aborted'));
27
+ };
28
+ signal.addEventListener('abort', onAbort, { once: true });
29
+ });
30
+ try {
31
+ return await Promise.race([q, abortPromise]);
32
+ } finally {
33
+ if (onAbort) signal.removeEventListener('abort', onAbort);
34
+ }
35
+ }
package/database/index.ts CHANGED
@@ -14,8 +14,14 @@ function createDatabase(): SQL {
14
14
  url = process.env.DB_CONNECTION_URL;
15
15
  }
16
16
 
17
- // Add statement_timeout only when explicitly configured (opt-in)
18
- // Note: PgBouncer rejects statement_timeout as a startup parameter
17
+ // DB_STATEMENT_TIMEOUT (opt-in, server-side query cancellation):
18
+ // - Skipped under PgBouncer because it rejects startup parameters.
19
+ // - DB_QUERY_TIMEOUT (default 30 s) is JS-side only: it raises a client error
20
+ // after the deadline but does NOT cancel the in-flight PostgreSQL query,
21
+ // which continues running and holds locks until the server decides to stop it.
22
+ // - Production deployments NOT behind PgBouncer should set DB_STATEMENT_TIMEOUT
23
+ // (e.g. DB_STATEMENT_TIMEOUT=30000) so the server itself kills runaway queries
24
+ // and releases locks promptly.
19
25
  if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
20
26
  try {
21
27
  const urlObj = new URL(url);
@@ -29,10 +35,16 @@ function createDatabase(): SQL {
29
35
  const redactedUrl = url.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
30
36
  logger.info(`Database connection URL: ${redactedUrl}`);
31
37
 
32
- const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
38
+ const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '20', 10);
33
39
  logger.info(`Connection pool size: ${max} connections`);
34
40
  logger.info(`Query timeout: ${QUERY_TIMEOUT_MS}ms`);
35
41
 
42
+ // DB_CONNECTION_TIMEOUT (default 30 s): the pool waits this long for a free
43
+ // slot before rejecting the caller. At 30 s, pool exhaustion queues HTTP
44
+ // requests for up to 30 s each, holding sockets and consuming memory.
45
+ // User-facing services should consider 5 s for fast-fail so clients get
46
+ // an error quickly rather than a slow timeout. Long-running background
47
+ // workers (schedulers, outbox, migrations) can keep higher values.
36
48
  const connTimeout = parseInt(process.env.DB_CONNECTION_TIMEOUT ?? '30', 10);
37
49
 
38
50
  return new SQL({
@@ -0,0 +1,141 @@
1
+ import type { SQL } from "bun";
2
+ import { logger as MainLogger } from "../core/Logger";
3
+ import { runWithSignal } from "./cancellable";
4
+
5
+ const logger = MainLogger.child({ scope: "db" });
6
+
7
+ const SLOW_MS = parseInt(process.env.BUNSANE_DB_SLOW_MS ?? '500', 10);
8
+
9
+ export type DataLoaderKind = 'entity' | 'component' | 'relation';
10
+
11
+ interface DbStatsInternal {
12
+ totalCount: number;
13
+ totalMs: number;
14
+ maxMs: number;
15
+ slowCount: number;
16
+ abortedCount: number;
17
+ inFlight: number;
18
+ inFlightMax: number;
19
+ dataLoaderCalls: { entity: number; component: number; relation: number };
20
+ }
21
+
22
+ const stats: DbStatsInternal = {
23
+ totalCount: 0,
24
+ totalMs: 0,
25
+ maxMs: 0,
26
+ slowCount: 0,
27
+ abortedCount: 0,
28
+ inFlight: 0,
29
+ inFlightMax: 0,
30
+ dataLoaderCalls: { entity: 0, component: 0, relation: 0 },
31
+ };
32
+
33
+ /**
34
+ * Per-request counter incremented when current request context is reachable
35
+ * via the (request as any).__bunsaneStats pointer. We accept that as a
36
+ * parameter from the call site so this module stays free of GraphQL imports.
37
+ */
38
+ export interface PerRequestCounters {
39
+ dbQueryCount: number;
40
+ }
41
+
42
+ /**
43
+ * Execute `db.unsafe(sql, params)` with optional AbortSignal cancellation
44
+ * and roundtrip telemetry. On abort the in-flight query is cancelled via
45
+ * `Query.cancel()`. Total ms is recorded into module-level stats; calls
46
+ * over `BUNSANE_DB_SLOW_MS` increment slowCount and emit a warn log.
47
+ */
48
+ export async function timedUnsafe<T = any>(
49
+ db: SQL,
50
+ sql: string,
51
+ params: any[],
52
+ signal?: AbortSignal,
53
+ perRequest?: PerRequestCounters,
54
+ ): Promise<T> {
55
+ const t0 = performance.now();
56
+ stats.inFlight++;
57
+ if (stats.inFlight > stats.inFlightMax) stats.inFlightMax = stats.inFlight;
58
+ if (perRequest) perRequest.dbQueryCount++;
59
+ let aborted = false;
60
+ try {
61
+ const q = (db as any).unsafe(sql, params);
62
+ return await runWithSignal<T>(q, signal);
63
+ } catch (err) {
64
+ if ((err as Error)?.name === 'AbortError' || signal?.aborted) {
65
+ aborted = true;
66
+ stats.abortedCount++;
67
+ }
68
+ throw err;
69
+ } finally {
70
+ const dt = performance.now() - t0;
71
+ stats.inFlight--;
72
+ stats.totalCount++;
73
+ stats.totalMs += dt;
74
+ if (dt > stats.maxMs) stats.maxMs = dt;
75
+ if (SLOW_MS > 0 && dt > SLOW_MS && !aborted) {
76
+ stats.slowCount++;
77
+ logger.warn(
78
+ {
79
+ durationMs: Math.round(dt),
80
+ thresholdMs: SLOW_MS,
81
+ sqlSnippet: sql.length > 200 ? sql.slice(0, 200) + '…' : sql,
82
+ msg: 'Slow DB call',
83
+ },
84
+ 'Slow DB call',
85
+ );
86
+ }
87
+ }
88
+ }
89
+
90
+ /**
91
+ * Increment the per-kind DataLoader counter. Called from inside DataLoader
92
+ * batch functions so /metrics + access log can attribute load patterns.
93
+ *
94
+ * `perRequest` is loosely typed because RequestContext's `RequestStats`
95
+ * (defined in core/RequestContext.ts) extends `PerRequestCounters` with
96
+ * extra fields like `dataLoaderCalls`. We accept either shape here without
97
+ * importing the higher-level type (which would create a cycle).
98
+ */
99
+ export function incrementDataLoaderCall(
100
+ kind: DataLoaderKind,
101
+ perRequest?: PerRequestCounters | { dataLoaderCalls?: { entity: number; component: number; relation: number } },
102
+ ): void {
103
+ stats.dataLoaderCalls[kind]++;
104
+ const dlc = (perRequest as any)?.dataLoaderCalls;
105
+ if (dlc) dlc[kind]++;
106
+ }
107
+
108
+ /**
109
+ * Snapshot of accumulated DB stats for the /metrics endpoint.
110
+ */
111
+ export function getDbStats() {
112
+ const avgMs = stats.totalCount > 0 ? stats.totalMs / stats.totalCount : 0;
113
+ return {
114
+ totalCount: stats.totalCount,
115
+ totalMs: Math.round(stats.totalMs),
116
+ maxMs: Math.round(stats.maxMs),
117
+ avgMs: Number(avgMs.toFixed(2)),
118
+ slowCount: stats.slowCount,
119
+ abortedCount: stats.abortedCount,
120
+ inFlight: stats.inFlight,
121
+ inFlightMax: stats.inFlightMax,
122
+ slowThresholdMs: SLOW_MS,
123
+ dataLoaderCalls: { ...stats.dataLoaderCalls },
124
+ };
125
+ }
126
+
127
+ /**
128
+ * Reset counters. Intended for tests only.
129
+ */
130
+ export function resetDbStats(): void {
131
+ stats.totalCount = 0;
132
+ stats.totalMs = 0;
133
+ stats.maxMs = 0;
134
+ stats.slowCount = 0;
135
+ stats.abortedCount = 0;
136
+ stats.inFlight = 0;
137
+ stats.inFlightMax = 0;
138
+ stats.dataLoaderCalls.entity = 0;
139
+ stats.dataLoaderCalls.component = 0;
140
+ stats.dataLoaderCalls.relation = 0;
141
+ }