bunsane 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/.claude/settings.local.json +47 -0
  2. package/.claude/skills/update-memory.md +74 -0
  3. package/.prettierrc +4 -0
  4. package/.serena/memories/architectural-decision-no-dependency-injection.md +76 -0
  5. package/.serena/memories/architecture.md +154 -0
  6. package/.serena/memories/cache-interface-refactoring-2026-01-24.md +165 -0
  7. package/.serena/memories/code_style_and_conventions.md +76 -0
  8. package/.serena/memories/project_overview.md +43 -0
  9. package/.serena/memories/schema-dsl-plan.md +107 -0
  10. package/.serena/memories/suggested_commands.md +80 -0
  11. package/.serena/memories/typescript-compilation-status.md +54 -0
  12. package/.serena/project.yml +114 -0
  13. package/TODO.md +1 -7
  14. package/bun.lock +150 -4
  15. package/bunfig.toml +10 -0
  16. package/config/cache.config.ts +77 -0
  17. package/config/upload.config.ts +4 -5
  18. package/core/App.ts +870 -123
  19. package/core/ArcheType.ts +2268 -377
  20. package/core/BatchLoader.ts +181 -71
  21. package/core/Config.ts +153 -0
  22. package/core/Decorators.ts +4 -1
  23. package/core/Entity.ts +621 -92
  24. package/core/EntityHookManager.ts +1 -1
  25. package/core/EntityInterface.ts +3 -1
  26. package/core/EntityManager.ts +1 -13
  27. package/core/ErrorHandler.ts +8 -2
  28. package/core/Logger.ts +9 -0
  29. package/core/Middleware.ts +34 -0
  30. package/core/RequestContext.ts +5 -1
  31. package/core/RequestLoaders.ts +227 -93
  32. package/core/SchedulerManager.ts +193 -52
  33. package/core/cache/CacheAnalytics.ts +399 -0
  34. package/core/cache/CacheFactory.ts +145 -0
  35. package/core/cache/CacheManager.ts +520 -0
  36. package/core/cache/CacheProvider.ts +34 -0
  37. package/core/cache/CacheWarmer.ts +157 -0
  38. package/core/cache/CompressionUtils.ts +110 -0
  39. package/core/cache/MemoryCache.ts +251 -0
  40. package/core/cache/MultiLevelCache.ts +180 -0
  41. package/core/cache/NoOpCache.ts +53 -0
  42. package/core/cache/RedisCache.ts +464 -0
  43. package/core/cache/TTLStrategy.ts +254 -0
  44. package/core/cache/index.ts +6 -0
  45. package/core/components/BaseComponent.ts +120 -0
  46. package/core/{ComponentRegistry.ts → components/ComponentRegistry.ts} +148 -54
  47. package/core/components/Decorators.ts +88 -0
  48. package/core/components/Interfaces.ts +7 -0
  49. package/core/components/index.ts +5 -0
  50. package/core/decorators/EntityHooks.ts +0 -3
  51. package/core/decorators/IndexedField.ts +26 -0
  52. package/core/decorators/ScheduledTask.ts +0 -47
  53. package/core/events/EntityLifecycleEvents.ts +1 -1
  54. package/core/health.ts +112 -0
  55. package/core/metadata/definitions/ArcheType.ts +14 -0
  56. package/core/metadata/definitions/Component.ts +9 -0
  57. package/core/metadata/definitions/gqlObject.ts +1 -1
  58. package/core/metadata/index.ts +42 -1
  59. package/core/metadata/metadata-storage.ts +28 -2
  60. package/core/middleware/AccessLog.ts +59 -0
  61. package/core/middleware/RequestId.ts +38 -0
  62. package/core/middleware/SecurityHeaders.ts +62 -0
  63. package/core/middleware/index.ts +3 -0
  64. package/core/scheduler/DistributedLock.ts +266 -0
  65. package/core/scheduler/index.ts +15 -0
  66. package/core/validateEnv.ts +92 -0
  67. package/database/DatabaseHelper.ts +416 -40
  68. package/database/IndexingStrategy.ts +342 -0
  69. package/database/PreparedStatementCache.ts +226 -0
  70. package/database/index.ts +32 -7
  71. package/database/sqlHelpers.ts +14 -2
  72. package/endpoints/archetypes.ts +362 -0
  73. package/endpoints/components.ts +58 -0
  74. package/endpoints/entity.ts +80 -0
  75. package/endpoints/index.ts +27 -0
  76. package/endpoints/query.ts +93 -0
  77. package/endpoints/stats.ts +76 -0
  78. package/endpoints/tables.ts +212 -0
  79. package/endpoints/types.ts +155 -0
  80. package/gql/ArchetypeOperations.ts +32 -86
  81. package/gql/Generator.ts +27 -315
  82. package/gql/GeneratorV2.ts +37 -0
  83. package/gql/builders/InputTypeBuilder.ts +99 -0
  84. package/gql/builders/ResolverBuilder.ts +234 -0
  85. package/gql/builders/TypeDefBuilder.ts +105 -0
  86. package/gql/builders/index.ts +3 -0
  87. package/gql/decorators/Upload.ts +1 -1
  88. package/gql/depthLimit.ts +85 -0
  89. package/gql/graph/GraphNode.ts +224 -0
  90. package/gql/graph/SchemaGraph.ts +278 -0
  91. package/gql/helpers.ts +8 -2
  92. package/gql/index.ts +56 -4
  93. package/gql/middleware.ts +79 -0
  94. package/gql/orchestration/GraphQLSchemaOrchestrator.ts +241 -0
  95. package/gql/orchestration/index.ts +1 -0
  96. package/gql/scanner/ServiceScanner.ts +347 -0
  97. package/gql/schema/index.ts +458 -0
  98. package/gql/strategies/TypeGenerationStrategy.ts +329 -0
  99. package/gql/types.ts +1 -0
  100. package/gql/utils/TypeSignature.ts +220 -0
  101. package/gql/utils/index.ts +1 -0
  102. package/gql/visitors/ArchetypePreprocessorVisitor.ts +80 -0
  103. package/gql/visitors/DeduplicationVisitor.ts +82 -0
  104. package/gql/visitors/GraphVisitor.ts +78 -0
  105. package/gql/visitors/ResolverGeneratorVisitor.ts +122 -0
  106. package/gql/visitors/SchemaGeneratorVisitor.ts +851 -0
  107. package/gql/visitors/TypeCollectorVisitor.ts +79 -0
  108. package/gql/visitors/VisitorComposer.ts +96 -0
  109. package/gql/visitors/index.ts +7 -0
  110. package/package.json +59 -37
  111. package/plugins/index.ts +2 -2
  112. package/query/CTENode.ts +97 -0
  113. package/query/ComponentInclusionNode.ts +689 -0
  114. package/query/FilterBuilder.ts +127 -0
  115. package/query/FilterBuilderRegistry.ts +202 -0
  116. package/query/OrNode.ts +517 -0
  117. package/query/OrQuery.ts +42 -0
  118. package/query/Query.ts +1022 -0
  119. package/query/QueryContext.ts +170 -0
  120. package/query/QueryDAG.ts +122 -0
  121. package/query/QueryNode.ts +65 -0
  122. package/query/SourceNode.ts +53 -0
  123. package/query/builders/FullTextSearchBuilder.ts +236 -0
  124. package/query/index.ts +21 -0
  125. package/scheduler/index.ts +40 -8
  126. package/service/Service.ts +2 -1
  127. package/service/ServiceRegistry.ts +6 -5
  128. package/{core/storage → storage}/LocalStorageProvider.ts +2 -2
  129. package/storage/S3StorageProvider.ts +316 -0
  130. package/{core/storage → storage}/StorageProvider.ts +7 -3
  131. package/studio/bun.lock +482 -0
  132. package/studio/index.html +13 -0
  133. package/studio/package.json +39 -0
  134. package/studio/postcss.config.js +6 -0
  135. package/studio/src/components/DataTable.tsx +211 -0
  136. package/studio/src/components/Layout.tsx +13 -0
  137. package/studio/src/components/PageContainer.tsx +9 -0
  138. package/studio/src/components/PageHeader.tsx +13 -0
  139. package/studio/src/components/SearchBar.tsx +57 -0
  140. package/studio/src/components/Sidebar.tsx +294 -0
  141. package/studio/src/components/ui/button.tsx +56 -0
  142. package/studio/src/components/ui/checkbox.tsx +26 -0
  143. package/studio/src/components/ui/input.tsx +25 -0
  144. package/studio/src/hooks/useDataTable.ts +131 -0
  145. package/studio/src/index.css +36 -0
  146. package/studio/src/lib/api.ts +186 -0
  147. package/studio/src/lib/utils.ts +13 -0
  148. package/studio/src/main.tsx +17 -0
  149. package/studio/src/pages/ArcheType.tsx +239 -0
  150. package/studio/src/pages/Components.tsx +124 -0
  151. package/studio/src/pages/EntityInspector.tsx +302 -0
  152. package/studio/src/pages/QueryRunner.tsx +246 -0
  153. package/studio/src/pages/Table.tsx +94 -0
  154. package/studio/src/pages/Welcome.tsx +241 -0
  155. package/studio/src/routes.tsx +45 -0
  156. package/studio/src/store/archeTypeSettings.ts +30 -0
  157. package/studio/src/store/studio.ts +65 -0
  158. package/studio/src/utils/columnHelpers.tsx +114 -0
  159. package/studio/studio-instructions.md +81 -0
  160. package/studio/tailwind.config.js +77 -0
  161. package/studio/tsconfig.json +24 -0
  162. package/studio/utils.ts +54 -0
  163. package/studio/vite.config.js +19 -0
  164. package/swagger/generator.ts +1 -1
  165. package/tests/e2e/http.test.ts +126 -0
  166. package/tests/fixtures/archetypes/TestUserArchetype.ts +21 -0
  167. package/tests/fixtures/components/TestOrder.ts +23 -0
  168. package/tests/fixtures/components/TestProduct.ts +23 -0
  169. package/tests/fixtures/components/TestUser.ts +20 -0
  170. package/tests/fixtures/components/index.ts +6 -0
  171. package/tests/graphql/SchemaGeneration.test.ts +90 -0
  172. package/tests/graphql/builders/ResolverBuilder.test.ts +223 -0
  173. package/tests/graphql/builders/TypeDefBuilder.test.ts +153 -0
  174. package/tests/integration/archetype/ArcheType.persistence.test.ts +241 -0
  175. package/tests/integration/cache/CacheInvalidation.test.ts +259 -0
  176. package/tests/integration/entity/Entity.persistence.test.ts +333 -0
  177. package/tests/integration/query/Query.exec.test.ts +523 -0
  178. package/tests/pglite-setup.ts +61 -0
  179. package/tests/setup.ts +164 -0
  180. package/tests/stress/BenchmarkRunner.ts +203 -0
  181. package/tests/stress/DataSeeder.ts +190 -0
  182. package/tests/stress/StressTestReporter.ts +229 -0
  183. package/tests/stress/cursor-perf-test.ts +171 -0
  184. package/tests/stress/fixtures/StressTestComponents.ts +58 -0
  185. package/tests/stress/index.ts +7 -0
  186. package/tests/stress/scenarios/query-benchmarks.test.ts +285 -0
  187. package/tests/unit/BatchLoader.test.ts +82 -0
  188. package/tests/unit/archetype/ArcheType.test.ts +107 -0
  189. package/tests/unit/cache/CacheManager.test.ts +347 -0
  190. package/tests/unit/cache/MemoryCache.test.ts +260 -0
  191. package/tests/unit/cache/RedisCache.test.ts +411 -0
  192. package/tests/unit/entity/Entity.components.test.ts +244 -0
  193. package/tests/unit/entity/Entity.test.ts +345 -0
  194. package/tests/unit/gql/depthLimit.test.ts +203 -0
  195. package/tests/unit/gql/operationMiddleware.test.ts +293 -0
  196. package/tests/unit/health/Health.test.ts +129 -0
  197. package/tests/unit/middleware/AccessLog.test.ts +37 -0
  198. package/tests/unit/middleware/Middleware.test.ts +98 -0
  199. package/tests/unit/middleware/RequestId.test.ts +54 -0
  200. package/tests/unit/middleware/SecurityHeaders.test.ts +66 -0
  201. package/tests/unit/query/FilterBuilder.test.ts +111 -0
  202. package/tests/unit/query/Query.test.ts +308 -0
  203. package/tests/unit/scheduler/DistributedLock.test.ts +274 -0
  204. package/tests/unit/schema/schema-integration.test.ts +426 -0
  205. package/tests/unit/schema/schema.test.ts +580 -0
  206. package/tests/unit/storage/S3StorageProvider.test.ts +571 -0
  207. package/tests/unit/upload/RestUpload.test.ts +267 -0
  208. package/tests/unit/validateEnv.test.ts +82 -0
  209. package/tests/utils/entity-tracker.ts +57 -0
  210. package/tests/utils/index.ts +13 -0
  211. package/tests/utils/test-context.ts +149 -0
  212. package/tsconfig.json +5 -1
  213. package/types/archetype.types.ts +6 -0
  214. package/types/hooks.types.ts +1 -1
  215. package/types/query.types.ts +110 -0
  216. package/types/scheduler.types.ts +68 -7
  217. package/types/upload.types.ts +1 -0
  218. package/{core → upload}/FileValidator.ts +10 -1
  219. package/upload/RestUpload.ts +130 -0
  220. package/{core/components → upload}/UploadComponent.ts +11 -11
  221. package/{core → upload}/UploadManager.ts +3 -3
  222. package/upload/index.ts +23 -7
  223. package/utils/UploadHelper.ts +27 -6
  224. package/utils/cronParser.ts +16 -6
  225. package/.github/workflows/deploy-docs.yml +0 -57
  226. package/core/Components.ts +0 -202
  227. package/core/EntityCache.ts +0 -15
  228. package/core/Query.ts +0 -880
  229. package/docs/README.md +0 -149
  230. package/docs/_coverpage.md +0 -36
  231. package/docs/_sidebar.md +0 -23
  232. package/docs/api/core.md +0 -568
  233. package/docs/api/hooks.md +0 -554
  234. package/docs/api/index.md +0 -222
  235. package/docs/api/query.md +0 -678
  236. package/docs/api/service.md +0 -744
  237. package/docs/core-concepts/archetypes.md +0 -512
  238. package/docs/core-concepts/components.md +0 -498
  239. package/docs/core-concepts/entity.md +0 -314
  240. package/docs/core-concepts/hooks.md +0 -683
  241. package/docs/core-concepts/query.md +0 -588
  242. package/docs/core-concepts/services.md +0 -647
  243. package/docs/examples/code-examples.md +0 -425
  244. package/docs/getting-started.md +0 -337
  245. package/docs/index.html +0 -97
  246. package/tests/bench/insert.bench.ts +0 -60
  247. package/tests/bench/relations.bench.ts +0 -270
  248. package/tests/bench/sorting.bench.ts +0 -416
  249. package/tests/component-hooks-simple.test.ts +0 -117
  250. package/tests/component-hooks.test.ts +0 -1461
  251. package/tests/component.test.ts +0 -339
  252. package/tests/errorHandling.test.ts +0 -155
  253. package/tests/hooks.test.ts +0 -667
  254. package/tests/query-sorting.test.ts +0 -101
  255. package/tests/query.test.ts +0 -81
  256. package/tests/relations.test.ts +0 -170
  257. package/tests/scheduler.test.ts +0 -724
@@ -1,5 +1,7 @@
1
- import db from "database";
2
- import { logger as MainLogger } from "core/Logger";
1
+ import db from "./index";
2
+ import { logger as MainLogger } from "../core/Logger";
3
+ import { getMetadataStorage } from "../core/metadata";
4
+ import { ensureMultipleJSONBPathIndexes } from "./IndexingStrategy";
3
5
  const logger = MainLogger.child({ scope: "DatabaseHelper" });
4
6
 
5
7
  const BUNSANE_RELATION_TYPED_COLUMN = process.env.BUNSANE_RELATION_TYPED_COLUMN === 'true' || false;
@@ -21,10 +23,15 @@ const retryWithBackoff = async (fn: () => Promise<void>, maxRetries: number = 3,
21
23
  try {
22
24
  await fn();
23
25
  return;
24
- } catch (error) {
26
+ } catch (error: any) {
27
+ const isDeadlock = error?.code === '40P01' || error?.message?.includes('deadlock');
25
28
  if (i === maxRetries - 1) throw error;
26
29
  const delay = baseDelay * Math.pow(2, i);
27
- logger.warn(`Operation failed, retrying in ${delay}ms: ${error}`);
30
+ if (isDeadlock) {
31
+ logger.warn(`Deadlock detected, retrying in ${delay}ms (attempt ${i + 1}/${maxRetries})`);
32
+ } else {
33
+ logger.warn(`Operation failed, retrying in ${delay}ms: ${error}`);
34
+ }
28
35
  await sleep(delay);
29
36
  }
30
37
  }
@@ -68,6 +75,7 @@ export const PrepareDatabase = async () => {
68
75
  }
69
76
  try {
70
77
  await CreateEntityComponentTable();
78
+ await PopulateComponentIds();
71
79
  } catch (error) {
72
80
  logger.error(`Failed to create entity component table: ${error}`);
73
81
  throw error;
@@ -103,32 +111,156 @@ export const CreateEntityTable = async () => {
103
111
  }
104
112
 
105
113
  export const CreateComponentTable = async () => {
114
+ const partitionStrategy = process.env.BUNSANE_PARTITION_STRATEGY === 'hash' ? 'hash' : 'list'; // Default to list (LIST+Direct is the recommended strategy)
115
+
116
+ // Check if the table already exists and what partitioning strategy it uses
117
+ const existingStrategy = await GetPartitionStrategy();
118
+ const tableExists = await db.unsafe(`
119
+ SELECT 1 FROM information_schema.tables
120
+ WHERE table_name = 'components'
121
+ AND table_schema = 'public'
122
+ `);
123
+
124
+ // If the table exists but has a different partitioning strategy, we need to recreate it
125
+ if (tableExists.length > 0 && existingStrategy !== partitionStrategy) {
126
+ logger.info(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
127
+
128
+ // Drop the existing table and all its partitions
129
+ await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
130
+
131
+ // Also clean up any orphaned partition tables
132
+ const orphanedPartitions = await db.unsafe(`
133
+ SELECT tablename
134
+ FROM pg_tables
135
+ WHERE tablename LIKE 'components_%'
136
+ AND schemaname = 'public'
137
+ `);
138
+
139
+ for (const partition of orphanedPartitions) {
140
+ await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
141
+ }
142
+ }
143
+
144
+ if (partitionStrategy === 'hash') {
145
+ // Clean up any existing LIST partition tables before creating HASH partitions
146
+ await cleanupOldListPartitions();
147
+ await CreateHashPartitionedComponentTable();
148
+ } else {
149
+ // Original LIST partitioning
150
+ await db`CREATE TABLE IF NOT EXISTS components (
151
+ id UUID,
152
+ entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
153
+ type_id varchar(64) NOT NULL,
154
+ name varchar(128),
155
+ data jsonb,
156
+ created_at TIMESTAMP DEFAULT NOW(),
157
+ updated_at TIMESTAMP DEFAULT NOW(),
158
+ deleted_at TIMESTAMP,
159
+ PRIMARY KEY (id, type_id),
160
+ UNIQUE(entity_id, type_id)
161
+ ) PARTITION BY LIST (type_id);`;
162
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_id ON components (entity_id)`;
163
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id)`;
164
+ await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data)`;
165
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at)`;
166
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL`;
167
+ await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL`;
168
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_created_desc ON components (entity_id, created_at DESC)`;
169
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_entity_created ON components (type_id, entity_id, created_at DESC)`;
170
+ }
171
+ }
172
+
173
+ const cleanupOldListPartitions = async () => {
174
+ try {
175
+ logger.info(`Cleaning up old LIST partition tables before creating HASH partitions`);
176
+
177
+ // Get all existing LIST partition tables
178
+ const existingPartitions = await db.unsafe(`
179
+ SELECT tablename
180
+ FROM pg_tables
181
+ WHERE tablename LIKE 'components_%'
182
+ AND schemaname = 'public'
183
+ AND tablename != 'components'
184
+ `);
185
+
186
+ for (const row of existingPartitions) {
187
+ const tableName = row.tablename;
188
+ logger.trace(`Dropping old LIST partition table: ${tableName}`);
189
+ await db.unsafe(`DROP TABLE IF EXISTS ${tableName} CASCADE`);
190
+ }
191
+
192
+ // Drop the main components table if it exists (to recreate with HASH partitioning)
193
+ const mainTableExists = await db.unsafe(`
194
+ SELECT 1 FROM information_schema.tables
195
+ WHERE table_name = 'components'
196
+ AND table_schema = 'public'
197
+ `);
198
+
199
+ if (mainTableExists.length > 0) {
200
+ logger.trace(`Dropping existing components table for HASH partitioning`);
201
+ await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
202
+ }
203
+
204
+ logger.info(`Cleaned up ${existingPartitions.length} old partition tables`);
205
+ } catch (error) {
206
+ logger.warn(`Could not clean up old LIST partitions: ${error}`);
207
+ // Continue anyway - the table creation might still work
208
+ }
209
+ }
210
+
211
+ export const CreateHashPartitionedComponentTable = async (partitionCount: number = 16) => {
106
212
  await db`CREATE TABLE IF NOT EXISTS components (
107
213
  id UUID,
108
- entity_id UUID REFERENCES entities(id),
214
+ entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
109
215
  type_id varchar(64) NOT NULL,
110
216
  name varchar(128),
111
217
  data jsonb,
112
218
  created_at TIMESTAMP DEFAULT NOW(),
113
219
  updated_at TIMESTAMP DEFAULT NOW(),
114
220
  deleted_at TIMESTAMP,
115
- PRIMARY KEY (id, type_id, entity_id)
116
- ) PARTITION BY LIST (type_id);`;
117
- await db`CREATE INDEX IF NOT EXISTS idx_components_entity_id ON components (entity_id);`
118
- await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id);`
119
- await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data);`
120
-
121
- // Phase 2A: Add composite indexes for sorting optimization
122
- await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at);`
123
- await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL;`
124
- await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL;`
125
- }
221
+ PRIMARY KEY (id, type_id),
222
+ UNIQUE(entity_id, type_id)
223
+ ) PARTITION BY HASH (type_id);`;
126
224
 
225
+ // Create hash partitions
226
+ for (let i = 0; i < partitionCount; i++) {
227
+ await db.unsafe(`CREATE TABLE IF NOT EXISTS components_p${i}
228
+ PARTITION OF components
229
+ FOR VALUES WITH (MODULUS ${partitionCount}, REMAINDER ${i});`);
230
+ }
231
+
232
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_id ON components (entity_id)`;
233
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id)`;
234
+ await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data)`;
235
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at)`;
236
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL`;
237
+ await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL`;
238
+ await db`CREATE INDEX IF NOT EXISTS idx_components_entity_created_desc ON components (entity_id, created_at DESC)`;
239
+ await db`CREATE INDEX IF NOT EXISTS idx_components_type_entity_created ON components (type_id, entity_id, created_at DESC)`;
240
+ }
127
241
  export const UpdateComponentIndexes = async (table_name: string, indexedProperties: string[]) => {
128
242
  try {
129
243
  table_name = validateIdentifier(table_name);
130
244
  indexedProperties = indexedProperties.map(prop => validateIdentifier(prop));
131
245
  logger.trace(`Updating indexes for component table: ${table_name}`);
246
+
247
+ // Check if this is a hash partitioned table
248
+ const partitionStrategy = await GetPartitionStrategy();
249
+ if (partitionStrategy === 'hash' && table_name !== 'components') {
250
+ // For hash partitioning, indexes should be on the parent table
251
+ logger.trace(`Redirecting index update to parent table 'components' for hash partitioning`);
252
+ table_name = 'components';
253
+ }
254
+
255
+ // Check if table is partitioned
256
+ const partitionCheck = await db.unsafe(`
257
+ SELECT relkind
258
+ FROM pg_class
259
+ WHERE relname = '${table_name}' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
260
+ `);
261
+ const isPartitioned = partitionCheck.length > 0 && partitionCheck[0].relkind === 'p';
262
+ const useConcurrently = !isPartitioned && !process.env.USE_PGLITE; // Cannot use CONCURRENTLY on partitioned tables or PGlite
263
+
132
264
  const indexes_list = await db.unsafe(`
133
265
  SELECT indexname
134
266
  FROM pg_indexes
@@ -144,7 +276,16 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
144
276
  if (!existingIndexes.includes(indexName)) {
145
277
  logger.trace(`Creating missing index ${indexName} for property ${prop}`);
146
278
  await retryWithBackoff(async () => {
147
- await db.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${indexName} ON ${table_name} USING GIN ((data->'${prop}'))`);
279
+ try {
280
+ await db.unsafe(`CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName} ON ${table_name} USING GIN ((data->'${prop}'))`);
281
+ } catch (error: any) {
282
+ // Check if the error is about duplicate key (index already exists)
283
+ if (error.message && error.message.includes('duplicate key value violates unique constraint "pg_class_relname_nsp_index"')) {
284
+ logger.trace(`Index ${indexName} already exists (confirmed by error), skipping creation`);
285
+ return;
286
+ }
287
+ throw error;
288
+ }
148
289
  });
149
290
  addedIndexes.add(indexName);
150
291
  } else {
@@ -160,7 +301,16 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
160
301
  const prop = match[1];
161
302
  if (!indexedProperties.includes(prop) && !addedIndexes.has(index)) {
162
303
  await retryWithBackoff(async () => {
163
- await db.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`);
304
+ try {
305
+ await db.unsafe(`DROP INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF EXISTS ${index}`);
306
+ } catch (error: any) {
307
+ // Check if the error is about relation does not exist
308
+ if (error.message && (error.message.includes('does not exist') || error.message.includes('not found'))) {
309
+ logger.trace(`Index ${index} does not exist, skipping drop`);
310
+ return;
311
+ }
312
+ throw error;
313
+ }
164
314
  });
165
315
  logger.info(`Dropped obsolete index ${index} for property ${prop}`);
166
316
  }
@@ -176,12 +326,34 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
176
326
  export const CreateComponentPartitionTable = async (comp_name: string, type_id: string) => {
177
327
  try {
178
328
  comp_name = validateIdentifier(comp_name);
179
- // // type_id is a value, not identifier, so no validation
180
- // if (indexedProperties) {
181
- // indexedProperties = indexedProperties.map(prop => validateIdentifier(prop));
182
- // }
183
329
  logger.trace(`Attempt adding partition table for component: ${comp_name}`);
184
- // const table_name = `components_${comp_name.toLowerCase().replace(/\s+/g, '_')}`;
330
+
331
+ // Check partitioning strategy
332
+ const partitionStrategy = await GetPartitionStrategy();
333
+ logger.trace(`Current partition strategy: ${partitionStrategy}`);
334
+
335
+ if (partitionStrategy === 'hash') {
336
+ // For HASH partitioning, partitions are pre-created and data is automatically distributed
337
+ // We just need to ensure indexes are created for this component type
338
+ logger.info(`Component ${comp_name} will use existing hash partitions`);
339
+
340
+ // For hash partitioning, indexes are created at the parent table level
341
+ // But we can still create component-specific indexes if needed
342
+ const storage = getMetadataStorage();
343
+ const componentId = storage.getComponentId(comp_name);
344
+ const indexedFields = storage.getIndexedFields(componentId);
345
+
346
+ if (indexedFields.length > 0) {
347
+ logger.trace(`Ensuring specialized indexes for ${comp_name} on hash partitions`);
348
+ // For hash partitioning, indexes on parent table should suffice
349
+ // But we can add component-specific logic here if needed
350
+ logger.trace(`Hash partitioning handles indexes at parent table level`);
351
+ }
352
+
353
+ return;
354
+ }
355
+
356
+ // Original LIST partitioning logic
185
357
  const table_name = GenerateTableName(comp_name);
186
358
  logger.trace(`Checking for existing partition table: ${table_name}`);
187
359
  const existingPartition = await db.unsafe(`SELECT 1 FROM information_schema.tables
@@ -202,17 +374,22 @@ export const CreateComponentPartitionTable = async (comp_name: string, type_id:
202
374
  });
203
375
  logger.trace(`Successfully created partition table: ${table_name}`);
204
376
 
205
- // TODO: Not sure if this is needed here or should be handled separately
206
- // if (BUNSANE_RELATION_TYPED_COLUMN && indexedProperties?.includes('value')) {
207
- // logger.trace(`Adding typed FK column for ${table_name}`);
208
- // await retryWithBackoff(async () => {
209
- // await db.unsafe(`ALTER TABLE ${table_name} ADD COLUMN IF NOT EXISTS fk_id UUID GENERATED ALWAYS AS ((data->>'value')::UUID) STORED`);
210
- // });
211
- // await retryWithBackoff(async () => {
212
- // await db.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${table_name}_fk_id ON ${table_name} (fk_id)`);
213
- // });
214
- // logger.trace(`Added fk_id column and index for ${table_name}`);
215
- // }
377
+ // Automatically create indexes based on component metadata
378
+ const storage = getMetadataStorage();
379
+ const componentId = storage.getComponentId(comp_name);
380
+ const indexedFields = storage.getIndexedFields(componentId);
381
+
382
+ if (indexedFields.length > 0) {
383
+ logger.trace(`Creating ${indexedFields.length} specialized indexes for ${comp_name}`);
384
+ const indexDefinitions = indexedFields.map(field => ({
385
+ tableName: table_name,
386
+ field: field.propertyKey,
387
+ indexType: field.indexType,
388
+ isDateField: field.isDateField
389
+ }));
390
+ await ensureMultipleJSONBPathIndexes(table_name, indexDefinitions);
391
+ logger.trace(`Created specialized indexes for ${comp_name}`);
392
+ }
216
393
 
217
394
  } catch (error) {
218
395
  logger.error(`Failed to create component partition table for ${comp_name}: ${error}`);
@@ -223,6 +400,18 @@ export const CreateComponentPartitionTable = async (comp_name: string, type_id:
223
400
  export const DeleteComponentPartitionTable = async (comp_name: string) => {
224
401
  try {
225
402
  comp_name = validateIdentifier(comp_name);
403
+
404
+ // Check partitioning strategy
405
+ const partitionStrategy = await GetPartitionStrategy();
406
+
407
+ if (partitionStrategy === 'hash') {
408
+ // For HASH partitioning, partitions are managed automatically
409
+ // No individual partition tables to delete
410
+ logger.info(`Component ${comp_name} uses hash partitions - no individual table to delete`);
411
+ return;
412
+ }
413
+
414
+ // Original LIST partitioning logic
226
415
  const table_name = `components_${comp_name.toLowerCase().replace(/\s+/g, '_')}`;
227
416
 
228
417
  const existingPartition = await db.unsafe(`
@@ -249,17 +438,204 @@ export const DeleteComponentPartitionTable = async (comp_name: string) => {
249
438
 
250
439
  export const CreateEntityComponentTable = async () => {
251
440
  await db`CREATE TABLE IF NOT EXISTS entity_components (
252
- entity_id UUID REFERENCES entities(id),
441
+ entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
253
442
  type_id VARCHAR(64) NOT NULL,
443
+ component_id UUID,
444
+ created_at TIMESTAMP DEFAULT NOW(),
445
+ updated_at TIMESTAMP DEFAULT NOW(),
254
446
  deleted_at TIMESTAMP,
255
447
  UNIQUE(entity_id, type_id)
256
448
  );`;
257
- await db`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_components_entity_id ON entity_components (entity_id);`
258
- await db`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_components_type_id ON entity_components (type_id);`
259
- await db`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_components_type_entity ON entity_components (type_id, entity_id);`
449
+ const concurrently = process.env.USE_PGLITE ? '' : ' CONCURRENTLY';
450
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_entity_id ON entity_components (entity_id)`);
451
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_id ON entity_components (type_id)`);
452
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_entity ON entity_components (type_id, entity_id)`);
453
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_entity_deleted ON entity_components (type_id, entity_id, deleted_at)`);
454
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_deleted_type ON entity_components (deleted_at, type_id) WHERE deleted_at IS NULL`);
455
+ await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_component_id ON entity_components (component_id)`);
456
+
457
+ // Add component_id column if it doesn't exist (for backward compatibility)
458
+ try {
459
+ await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
460
+ logger.info(`Added component_id column to entity_components table`);
461
+ } catch (error) {
462
+ logger.warn(`Could not add component_id column to entity_components table: ${error}`);
463
+ }
464
+ }
465
+
466
+ export const PopulateComponentIds = async () => {
467
+ try {
468
+ // Populate component_id for existing rows that don't have it set
469
+ await db`UPDATE entity_components
470
+ SET component_id = c.id
471
+ FROM components c
472
+ WHERE entity_components.entity_id = c.entity_id
473
+ AND entity_components.type_id = c.type_id
474
+ AND entity_components.component_id IS NULL`;
475
+
476
+ logger.info(`Populated component_id for existing entity_components rows`);
477
+ } catch (error) {
478
+ logger.warn(`Could not populate component_id for existing rows: ${error}`);
479
+ }
480
+ }
260
481
 
261
- await db`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_components_type_entity_deleted ON entity_components (type_id, entity_id, deleted_at);`
262
- await db`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_components_deleted_type ON entity_components (deleted_at, type_id) WHERE deleted_at IS NULL;`
482
+ export const EnsureDatabaseMigrations = async () => {
483
+ logger.trace(`Checking for database migrations...`);
484
+
485
+ try {
486
+ // First, ensure the table exists and has the basic structure
487
+ await CreateEntityComponentTable();
488
+
489
+ // Check if entity_components table has component_id column
490
+ const columnCheck = await db`SELECT column_name FROM information_schema.columns
491
+ WHERE table_name = 'entity_components'
492
+ AND column_name = 'component_id'
493
+ AND table_schema = 'public'`;
494
+
495
+ if (columnCheck.length === 0) {
496
+ logger.info(`entity_components table missing component_id column, adding it...`);
497
+ // Add the column
498
+ await db`ALTER TABLE entity_components ADD COLUMN component_id UUID`;
499
+ logger.info(`Added component_id column to entity_components table`);
500
+
501
+ // Wait a bit for the column to be available
502
+ await new Promise(resolve => setTimeout(resolve, 500));
503
+
504
+ // Populate existing data
505
+ await PopulateComponentIds();
506
+ } else {
507
+ logger.trace(`entity_components table already has component_id column`);
508
+ }
509
+ } catch (error) {
510
+ logger.error(`Failed during database migration: ${error}`);
511
+ // Try to add the column anyway in case the check failed
512
+ try {
513
+ await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
514
+ logger.info(`Attempted to add component_id column as fallback`);
515
+ } catch (fallbackError) {
516
+ logger.error(`Fallback column addition also failed: ${fallbackError}`);
517
+ }
518
+ }
519
+ }
520
+
521
+ export const AnalyzeAllComponentTables = async (): Promise<void> => {
522
+ try {
523
+ logger.trace(`Analyzing all component tables`);
524
+
525
+ // Check partitioning strategy
526
+ const partitionStrategy = await GetPartitionStrategy();
527
+
528
+ let tablePattern: string;
529
+ if (partitionStrategy === 'hash') {
530
+ // For hash partitioning, analyze the hash partition tables
531
+ tablePattern = 'components_p%';
532
+ } else {
533
+ // For list partitioning, analyze the component-specific partition tables
534
+ tablePattern = 'components_%';
535
+ }
536
+
537
+ // Get all component partition tables
538
+ const tables = await db.unsafe(`
539
+ SELECT tablename
540
+ FROM pg_tables
541
+ WHERE tablename LIKE '${tablePattern}' AND schemaname = 'public'
542
+ `);
543
+
544
+ for (const row of tables) {
545
+ logger.trace(`Running ANALYZE on table ${row.tablename}`);
546
+ await db.unsafe(`ANALYZE ${row.tablename}`);
547
+ logger.trace(`Completed ANALYZE on table ${row.tablename}`);
548
+ }
549
+
550
+ logger.info(`Completed ANALYZE on ${tables.length} component tables`);
551
+ } catch (error) {
552
+ logger.error(`Failed to analyze component tables: ${error}`);
553
+ throw error;
554
+ }
555
+ }
556
+
557
+ export const GetPartitionStrategy = async (): Promise<'list' | 'hash' | null> => {
558
+ try {
559
+ const result = await db.unsafe(`
560
+ SELECT
561
+ CASE
562
+ WHEN partstrat = 'l' THEN 'list'
563
+ WHEN partstrat = 'h' THEN 'hash'
564
+ ELSE NULL
565
+ END as strategy
566
+ FROM pg_partitioned_table
567
+ WHERE partrelid = (SELECT oid FROM pg_class WHERE relname = 'components' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public'))
568
+ `);
569
+ return result.length > 0 ? result[0].strategy : null;
570
+ } catch (error) {
571
+ logger.warn(`Could not determine partition strategy: ${error}`);
572
+ return null;
573
+ }
574
+ }
575
+
576
+ export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 16, 32]) => {
577
+ const results: Array<{partitionCount: number, planningTime: number, executionTime: number}> = [];
578
+
579
+ for (const count of partitionCounts) {
580
+ logger.info(`Benchmarking with ${count} partitions`);
581
+
582
+ // Create temporary hash partitioned table
583
+ const tempTableName = `components_benchmark_${count}`;
584
+ await db.unsafe(`CREATE TABLE ${tempTableName} (
585
+ id UUID,
586
+ entity_id UUID,
587
+ type_id varchar(64) NOT NULL,
588
+ name varchar(128),
589
+ data jsonb,
590
+ created_at TIMESTAMP DEFAULT NOW(),
591
+ updated_at TIMESTAMP DEFAULT NOW(),
592
+ deleted_at TIMESTAMP,
593
+ PRIMARY KEY (id, type_id),
594
+ UNIQUE(entity_id, type_id)
595
+ ) PARTITION BY HASH (type_id);`);
596
+
597
+ // Create partitions
598
+ for (let i = 0; i < count; i++) {
599
+ await db.unsafe(`CREATE TABLE ${tempTableName}_p${i}
600
+ PARTITION OF ${tempTableName}
601
+ FOR VALUES WITH (MODULUS ${count}, REMAINDER ${i});`);
602
+ }
603
+
604
+ // Copy sample data (limit to avoid long benchmark)
605
+ await db.unsafe(`INSERT INTO ${tempTableName} (id, entity_id, type_id, name, data, created_at, updated_at, deleted_at)
606
+ SELECT id, entity_id, type_id, name, data, created_at, updated_at, deleted_at
607
+ FROM components
608
+ TABLESAMPLE BERNOULLI(10) -- Sample 10% of data
609
+ LIMIT 10000;`);
610
+
611
+ // Create indexes
612
+ await db.unsafe(`CREATE INDEX idx_${tempTableName}_type_id ON ${tempTableName} (type_id)`);
613
+ await db.unsafe(`ANALYZE ${tempTableName}`);
614
+
615
+ // Run benchmark query
616
+ const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
617
+ SELECT DISTINCT ec.entity_id as id
618
+ FROM entity_components ec
619
+ WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1)
620
+ AND ec.deleted_at IS NULL`);
621
+
622
+ const plan = explainResult[0]['QUERY PLAN'] ? JSON.parse(explainResult[0]['QUERY PLAN']) : explainResult[0];
623
+ const planningTime = plan.Planning ? plan.Planning.Time : 0;
624
+ const executionTime = plan.Execution ? plan.Execution.Time : 0;
625
+
626
+ results.push({
627
+ partitionCount: count,
628
+ planningTime,
629
+ executionTime
630
+ });
631
+
632
+ // Clean up
633
+ await db.unsafe(`DROP TABLE ${tempTableName} CASCADE;`);
634
+
635
+ logger.info(`Partition count ${count}: planning=${planningTime}ms, execution=${executionTime}ms`);
636
+ }
637
+
638
+ return results;
263
639
  }
264
640
 
265
641
  export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;