bunsane 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +47 -0
- package/.claude/skills/update-memory.md +74 -0
- package/.prettierrc +4 -0
- package/.serena/memories/architectural-decision-no-dependency-injection.md +76 -0
- package/.serena/memories/architecture.md +154 -0
- package/.serena/memories/cache-interface-refactoring-2026-01-24.md +165 -0
- package/.serena/memories/code_style_and_conventions.md +76 -0
- package/.serena/memories/project_overview.md +43 -0
- package/.serena/memories/schema-dsl-plan.md +107 -0
- package/.serena/memories/suggested_commands.md +80 -0
- package/.serena/memories/typescript-compilation-status.md +54 -0
- package/.serena/project.yml +114 -0
- package/TODO.md +1 -7
- package/bun.lock +150 -4
- package/bunfig.toml +10 -0
- package/config/cache.config.ts +77 -0
- package/config/upload.config.ts +4 -5
- package/core/App.ts +870 -123
- package/core/ArcheType.ts +2268 -377
- package/core/BatchLoader.ts +181 -71
- package/core/Config.ts +153 -0
- package/core/Decorators.ts +4 -1
- package/core/Entity.ts +621 -92
- package/core/EntityHookManager.ts +1 -1
- package/core/EntityInterface.ts +3 -1
- package/core/EntityManager.ts +1 -13
- package/core/ErrorHandler.ts +8 -2
- package/core/Logger.ts +9 -0
- package/core/Middleware.ts +34 -0
- package/core/RequestContext.ts +5 -1
- package/core/RequestLoaders.ts +227 -93
- package/core/SchedulerManager.ts +193 -52
- package/core/cache/CacheAnalytics.ts +399 -0
- package/core/cache/CacheFactory.ts +145 -0
- package/core/cache/CacheManager.ts +520 -0
- package/core/cache/CacheProvider.ts +34 -0
- package/core/cache/CacheWarmer.ts +157 -0
- package/core/cache/CompressionUtils.ts +110 -0
- package/core/cache/MemoryCache.ts +251 -0
- package/core/cache/MultiLevelCache.ts +180 -0
- package/core/cache/NoOpCache.ts +53 -0
- package/core/cache/RedisCache.ts +464 -0
- package/core/cache/TTLStrategy.ts +254 -0
- package/core/cache/index.ts +6 -0
- package/core/components/BaseComponent.ts +120 -0
- package/core/{ComponentRegistry.ts → components/ComponentRegistry.ts} +148 -54
- package/core/components/Decorators.ts +88 -0
- package/core/components/Interfaces.ts +7 -0
- package/core/components/index.ts +5 -0
- package/core/decorators/EntityHooks.ts +0 -3
- package/core/decorators/IndexedField.ts +26 -0
- package/core/decorators/ScheduledTask.ts +0 -47
- package/core/events/EntityLifecycleEvents.ts +1 -1
- package/core/health.ts +112 -0
- package/core/metadata/definitions/ArcheType.ts +14 -0
- package/core/metadata/definitions/Component.ts +9 -0
- package/core/metadata/definitions/gqlObject.ts +1 -1
- package/core/metadata/index.ts +42 -1
- package/core/metadata/metadata-storage.ts +28 -2
- package/core/middleware/AccessLog.ts +59 -0
- package/core/middleware/RequestId.ts +38 -0
- package/core/middleware/SecurityHeaders.ts +62 -0
- package/core/middleware/index.ts +3 -0
- package/core/scheduler/DistributedLock.ts +266 -0
- package/core/scheduler/index.ts +15 -0
- package/core/validateEnv.ts +92 -0
- package/database/DatabaseHelper.ts +416 -40
- package/database/IndexingStrategy.ts +342 -0
- package/database/PreparedStatementCache.ts +226 -0
- package/database/index.ts +32 -7
- package/database/sqlHelpers.ts +14 -2
- package/endpoints/archetypes.ts +362 -0
- package/endpoints/components.ts +58 -0
- package/endpoints/entity.ts +80 -0
- package/endpoints/index.ts +27 -0
- package/endpoints/query.ts +93 -0
- package/endpoints/stats.ts +76 -0
- package/endpoints/tables.ts +212 -0
- package/endpoints/types.ts +155 -0
- package/gql/ArchetypeOperations.ts +32 -86
- package/gql/Generator.ts +27 -315
- package/gql/GeneratorV2.ts +37 -0
- package/gql/builders/InputTypeBuilder.ts +99 -0
- package/gql/builders/ResolverBuilder.ts +234 -0
- package/gql/builders/TypeDefBuilder.ts +105 -0
- package/gql/builders/index.ts +3 -0
- package/gql/decorators/Upload.ts +1 -1
- package/gql/depthLimit.ts +85 -0
- package/gql/graph/GraphNode.ts +224 -0
- package/gql/graph/SchemaGraph.ts +278 -0
- package/gql/helpers.ts +8 -2
- package/gql/index.ts +56 -4
- package/gql/middleware.ts +79 -0
- package/gql/orchestration/GraphQLSchemaOrchestrator.ts +241 -0
- package/gql/orchestration/index.ts +1 -0
- package/gql/scanner/ServiceScanner.ts +347 -0
- package/gql/schema/index.ts +458 -0
- package/gql/strategies/TypeGenerationStrategy.ts +329 -0
- package/gql/types.ts +1 -0
- package/gql/utils/TypeSignature.ts +220 -0
- package/gql/utils/index.ts +1 -0
- package/gql/visitors/ArchetypePreprocessorVisitor.ts +80 -0
- package/gql/visitors/DeduplicationVisitor.ts +82 -0
- package/gql/visitors/GraphVisitor.ts +78 -0
- package/gql/visitors/ResolverGeneratorVisitor.ts +122 -0
- package/gql/visitors/SchemaGeneratorVisitor.ts +851 -0
- package/gql/visitors/TypeCollectorVisitor.ts +79 -0
- package/gql/visitors/VisitorComposer.ts +96 -0
- package/gql/visitors/index.ts +7 -0
- package/package.json +59 -37
- package/plugins/index.ts +2 -2
- package/query/CTENode.ts +97 -0
- package/query/ComponentInclusionNode.ts +689 -0
- package/query/FilterBuilder.ts +127 -0
- package/query/FilterBuilderRegistry.ts +202 -0
- package/query/OrNode.ts +517 -0
- package/query/OrQuery.ts +42 -0
- package/query/Query.ts +1022 -0
- package/query/QueryContext.ts +170 -0
- package/query/QueryDAG.ts +122 -0
- package/query/QueryNode.ts +65 -0
- package/query/SourceNode.ts +53 -0
- package/query/builders/FullTextSearchBuilder.ts +236 -0
- package/query/index.ts +21 -0
- package/scheduler/index.ts +40 -8
- package/service/Service.ts +2 -1
- package/service/ServiceRegistry.ts +6 -5
- package/{core/storage → storage}/LocalStorageProvider.ts +2 -2
- package/storage/S3StorageProvider.ts +316 -0
- package/{core/storage → storage}/StorageProvider.ts +7 -3
- package/studio/bun.lock +482 -0
- package/studio/index.html +13 -0
- package/studio/package.json +39 -0
- package/studio/postcss.config.js +6 -0
- package/studio/src/components/DataTable.tsx +211 -0
- package/studio/src/components/Layout.tsx +13 -0
- package/studio/src/components/PageContainer.tsx +9 -0
- package/studio/src/components/PageHeader.tsx +13 -0
- package/studio/src/components/SearchBar.tsx +57 -0
- package/studio/src/components/Sidebar.tsx +294 -0
- package/studio/src/components/ui/button.tsx +56 -0
- package/studio/src/components/ui/checkbox.tsx +26 -0
- package/studio/src/components/ui/input.tsx +25 -0
- package/studio/src/hooks/useDataTable.ts +131 -0
- package/studio/src/index.css +36 -0
- package/studio/src/lib/api.ts +186 -0
- package/studio/src/lib/utils.ts +13 -0
- package/studio/src/main.tsx +17 -0
- package/studio/src/pages/ArcheType.tsx +239 -0
- package/studio/src/pages/Components.tsx +124 -0
- package/studio/src/pages/EntityInspector.tsx +302 -0
- package/studio/src/pages/QueryRunner.tsx +246 -0
- package/studio/src/pages/Table.tsx +94 -0
- package/studio/src/pages/Welcome.tsx +241 -0
- package/studio/src/routes.tsx +45 -0
- package/studio/src/store/archeTypeSettings.ts +30 -0
- package/studio/src/store/studio.ts +65 -0
- package/studio/src/utils/columnHelpers.tsx +114 -0
- package/studio/studio-instructions.md +81 -0
- package/studio/tailwind.config.js +77 -0
- package/studio/tsconfig.json +24 -0
- package/studio/utils.ts +54 -0
- package/studio/vite.config.js +19 -0
- package/swagger/generator.ts +1 -1
- package/tests/e2e/http.test.ts +126 -0
- package/tests/fixtures/archetypes/TestUserArchetype.ts +21 -0
- package/tests/fixtures/components/TestOrder.ts +23 -0
- package/tests/fixtures/components/TestProduct.ts +23 -0
- package/tests/fixtures/components/TestUser.ts +20 -0
- package/tests/fixtures/components/index.ts +6 -0
- package/tests/graphql/SchemaGeneration.test.ts +90 -0
- package/tests/graphql/builders/ResolverBuilder.test.ts +223 -0
- package/tests/graphql/builders/TypeDefBuilder.test.ts +153 -0
- package/tests/integration/archetype/ArcheType.persistence.test.ts +241 -0
- package/tests/integration/cache/CacheInvalidation.test.ts +259 -0
- package/tests/integration/entity/Entity.persistence.test.ts +333 -0
- package/tests/integration/query/Query.exec.test.ts +523 -0
- package/tests/pglite-setup.ts +61 -0
- package/tests/setup.ts +164 -0
- package/tests/stress/BenchmarkRunner.ts +203 -0
- package/tests/stress/DataSeeder.ts +190 -0
- package/tests/stress/StressTestReporter.ts +229 -0
- package/tests/stress/cursor-perf-test.ts +171 -0
- package/tests/stress/fixtures/StressTestComponents.ts +58 -0
- package/tests/stress/index.ts +7 -0
- package/tests/stress/scenarios/query-benchmarks.test.ts +285 -0
- package/tests/unit/BatchLoader.test.ts +82 -0
- package/tests/unit/archetype/ArcheType.test.ts +107 -0
- package/tests/unit/cache/CacheManager.test.ts +347 -0
- package/tests/unit/cache/MemoryCache.test.ts +260 -0
- package/tests/unit/cache/RedisCache.test.ts +411 -0
- package/tests/unit/entity/Entity.components.test.ts +244 -0
- package/tests/unit/entity/Entity.test.ts +345 -0
- package/tests/unit/gql/depthLimit.test.ts +203 -0
- package/tests/unit/gql/operationMiddleware.test.ts +293 -0
- package/tests/unit/health/Health.test.ts +129 -0
- package/tests/unit/middleware/AccessLog.test.ts +37 -0
- package/tests/unit/middleware/Middleware.test.ts +98 -0
- package/tests/unit/middleware/RequestId.test.ts +54 -0
- package/tests/unit/middleware/SecurityHeaders.test.ts +66 -0
- package/tests/unit/query/FilterBuilder.test.ts +111 -0
- package/tests/unit/query/Query.test.ts +308 -0
- package/tests/unit/scheduler/DistributedLock.test.ts +274 -0
- package/tests/unit/schema/schema-integration.test.ts +426 -0
- package/tests/unit/schema/schema.test.ts +580 -0
- package/tests/unit/storage/S3StorageProvider.test.ts +571 -0
- package/tests/unit/upload/RestUpload.test.ts +267 -0
- package/tests/unit/validateEnv.test.ts +82 -0
- package/tests/utils/entity-tracker.ts +57 -0
- package/tests/utils/index.ts +13 -0
- package/tests/utils/test-context.ts +149 -0
- package/tsconfig.json +5 -1
- package/types/archetype.types.ts +6 -0
- package/types/hooks.types.ts +1 -1
- package/types/query.types.ts +110 -0
- package/types/scheduler.types.ts +68 -7
- package/types/upload.types.ts +1 -0
- package/{core → upload}/FileValidator.ts +10 -1
- package/upload/RestUpload.ts +130 -0
- package/{core/components → upload}/UploadComponent.ts +11 -11
- package/{core → upload}/UploadManager.ts +3 -3
- package/upload/index.ts +23 -7
- package/utils/UploadHelper.ts +27 -6
- package/utils/cronParser.ts +16 -6
- package/.github/workflows/deploy-docs.yml +0 -57
- package/core/Components.ts +0 -202
- package/core/EntityCache.ts +0 -15
- package/core/Query.ts +0 -880
- package/docs/README.md +0 -149
- package/docs/_coverpage.md +0 -36
- package/docs/_sidebar.md +0 -23
- package/docs/api/core.md +0 -568
- package/docs/api/hooks.md +0 -554
- package/docs/api/index.md +0 -222
- package/docs/api/query.md +0 -678
- package/docs/api/service.md +0 -744
- package/docs/core-concepts/archetypes.md +0 -512
- package/docs/core-concepts/components.md +0 -498
- package/docs/core-concepts/entity.md +0 -314
- package/docs/core-concepts/hooks.md +0 -683
- package/docs/core-concepts/query.md +0 -588
- package/docs/core-concepts/services.md +0 -647
- package/docs/examples/code-examples.md +0 -425
- package/docs/getting-started.md +0 -337
- package/docs/index.html +0 -97
- package/tests/bench/insert.bench.ts +0 -60
- package/tests/bench/relations.bench.ts +0 -270
- package/tests/bench/sorting.bench.ts +0 -416
- package/tests/component-hooks-simple.test.ts +0 -117
- package/tests/component-hooks.test.ts +0 -1461
- package/tests/component.test.ts +0 -339
- package/tests/errorHandling.test.ts +0 -155
- package/tests/hooks.test.ts +0 -667
- package/tests/query-sorting.test.ts +0 -101
- package/tests/query.test.ts +0 -81
- package/tests/relations.test.ts +0 -170
- package/tests/scheduler.test.ts +0 -724
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import db from "
|
|
2
|
-
import { logger as MainLogger } from "core/Logger";
|
|
1
|
+
import db from "./index";
|
|
2
|
+
import { logger as MainLogger } from "../core/Logger";
|
|
3
|
+
import { getMetadataStorage } from "../core/metadata";
|
|
4
|
+
import { ensureMultipleJSONBPathIndexes } from "./IndexingStrategy";
|
|
3
5
|
const logger = MainLogger.child({ scope: "DatabaseHelper" });
|
|
4
6
|
|
|
5
7
|
const BUNSANE_RELATION_TYPED_COLUMN = process.env.BUNSANE_RELATION_TYPED_COLUMN === 'true' || false;
|
|
@@ -21,10 +23,15 @@ const retryWithBackoff = async (fn: () => Promise<void>, maxRetries: number = 3,
|
|
|
21
23
|
try {
|
|
22
24
|
await fn();
|
|
23
25
|
return;
|
|
24
|
-
} catch (error) {
|
|
26
|
+
} catch (error: any) {
|
|
27
|
+
const isDeadlock = error?.code === '40P01' || error?.message?.includes('deadlock');
|
|
25
28
|
if (i === maxRetries - 1) throw error;
|
|
26
29
|
const delay = baseDelay * Math.pow(2, i);
|
|
27
|
-
|
|
30
|
+
if (isDeadlock) {
|
|
31
|
+
logger.warn(`Deadlock detected, retrying in ${delay}ms (attempt ${i + 1}/${maxRetries})`);
|
|
32
|
+
} else {
|
|
33
|
+
logger.warn(`Operation failed, retrying in ${delay}ms: ${error}`);
|
|
34
|
+
}
|
|
28
35
|
await sleep(delay);
|
|
29
36
|
}
|
|
30
37
|
}
|
|
@@ -68,6 +75,7 @@ export const PrepareDatabase = async () => {
|
|
|
68
75
|
}
|
|
69
76
|
try {
|
|
70
77
|
await CreateEntityComponentTable();
|
|
78
|
+
await PopulateComponentIds();
|
|
71
79
|
} catch (error) {
|
|
72
80
|
logger.error(`Failed to create entity component table: ${error}`);
|
|
73
81
|
throw error;
|
|
@@ -103,32 +111,156 @@ export const CreateEntityTable = async () => {
|
|
|
103
111
|
}
|
|
104
112
|
|
|
105
113
|
export const CreateComponentTable = async () => {
|
|
114
|
+
const partitionStrategy = process.env.BUNSANE_PARTITION_STRATEGY === 'hash' ? 'hash' : 'list'; // Default to list (LIST+Direct is the recommended strategy)
|
|
115
|
+
|
|
116
|
+
// Check if the table already exists and what partitioning strategy it uses
|
|
117
|
+
const existingStrategy = await GetPartitionStrategy();
|
|
118
|
+
const tableExists = await db.unsafe(`
|
|
119
|
+
SELECT 1 FROM information_schema.tables
|
|
120
|
+
WHERE table_name = 'components'
|
|
121
|
+
AND table_schema = 'public'
|
|
122
|
+
`);
|
|
123
|
+
|
|
124
|
+
// If the table exists but has a different partitioning strategy, we need to recreate it
|
|
125
|
+
if (tableExists.length > 0 && existingStrategy !== partitionStrategy) {
|
|
126
|
+
logger.info(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
|
|
127
|
+
|
|
128
|
+
// Drop the existing table and all its partitions
|
|
129
|
+
await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
|
|
130
|
+
|
|
131
|
+
// Also clean up any orphaned partition tables
|
|
132
|
+
const orphanedPartitions = await db.unsafe(`
|
|
133
|
+
SELECT tablename
|
|
134
|
+
FROM pg_tables
|
|
135
|
+
WHERE tablename LIKE 'components_%'
|
|
136
|
+
AND schemaname = 'public'
|
|
137
|
+
`);
|
|
138
|
+
|
|
139
|
+
for (const partition of orphanedPartitions) {
|
|
140
|
+
await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (partitionStrategy === 'hash') {
|
|
145
|
+
// Clean up any existing LIST partition tables before creating HASH partitions
|
|
146
|
+
await cleanupOldListPartitions();
|
|
147
|
+
await CreateHashPartitionedComponentTable();
|
|
148
|
+
} else {
|
|
149
|
+
// Original LIST partitioning
|
|
150
|
+
await db`CREATE TABLE IF NOT EXISTS components (
|
|
151
|
+
id UUID,
|
|
152
|
+
entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
|
|
153
|
+
type_id varchar(64) NOT NULL,
|
|
154
|
+
name varchar(128),
|
|
155
|
+
data jsonb,
|
|
156
|
+
created_at TIMESTAMP DEFAULT NOW(),
|
|
157
|
+
updated_at TIMESTAMP DEFAULT NOW(),
|
|
158
|
+
deleted_at TIMESTAMP,
|
|
159
|
+
PRIMARY KEY (id, type_id),
|
|
160
|
+
UNIQUE(entity_id, type_id)
|
|
161
|
+
) PARTITION BY LIST (type_id);`;
|
|
162
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_id ON components (entity_id)`;
|
|
163
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id)`;
|
|
164
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data)`;
|
|
165
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at)`;
|
|
166
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL`;
|
|
167
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL`;
|
|
168
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_created_desc ON components (entity_id, created_at DESC)`;
|
|
169
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_entity_created ON components (type_id, entity_id, created_at DESC)`;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const cleanupOldListPartitions = async () => {
|
|
174
|
+
try {
|
|
175
|
+
logger.info(`Cleaning up old LIST partition tables before creating HASH partitions`);
|
|
176
|
+
|
|
177
|
+
// Get all existing LIST partition tables
|
|
178
|
+
const existingPartitions = await db.unsafe(`
|
|
179
|
+
SELECT tablename
|
|
180
|
+
FROM pg_tables
|
|
181
|
+
WHERE tablename LIKE 'components_%'
|
|
182
|
+
AND schemaname = 'public'
|
|
183
|
+
AND tablename != 'components'
|
|
184
|
+
`);
|
|
185
|
+
|
|
186
|
+
for (const row of existingPartitions) {
|
|
187
|
+
const tableName = row.tablename;
|
|
188
|
+
logger.trace(`Dropping old LIST partition table: ${tableName}`);
|
|
189
|
+
await db.unsafe(`DROP TABLE IF EXISTS ${tableName} CASCADE`);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Drop the main components table if it exists (to recreate with HASH partitioning)
|
|
193
|
+
const mainTableExists = await db.unsafe(`
|
|
194
|
+
SELECT 1 FROM information_schema.tables
|
|
195
|
+
WHERE table_name = 'components'
|
|
196
|
+
AND table_schema = 'public'
|
|
197
|
+
`);
|
|
198
|
+
|
|
199
|
+
if (mainTableExists.length > 0) {
|
|
200
|
+
logger.trace(`Dropping existing components table for HASH partitioning`);
|
|
201
|
+
await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
logger.info(`Cleaned up ${existingPartitions.length} old partition tables`);
|
|
205
|
+
} catch (error) {
|
|
206
|
+
logger.warn(`Could not clean up old LIST partitions: ${error}`);
|
|
207
|
+
// Continue anyway - the table creation might still work
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export const CreateHashPartitionedComponentTable = async (partitionCount: number = 16) => {
|
|
106
212
|
await db`CREATE TABLE IF NOT EXISTS components (
|
|
107
213
|
id UUID,
|
|
108
|
-
entity_id UUID REFERENCES entities(id),
|
|
214
|
+
entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
|
|
109
215
|
type_id varchar(64) NOT NULL,
|
|
110
216
|
name varchar(128),
|
|
111
217
|
data jsonb,
|
|
112
218
|
created_at TIMESTAMP DEFAULT NOW(),
|
|
113
219
|
updated_at TIMESTAMP DEFAULT NOW(),
|
|
114
220
|
deleted_at TIMESTAMP,
|
|
115
|
-
PRIMARY KEY (id, type_id,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id);`
|
|
119
|
-
await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data);`
|
|
120
|
-
|
|
121
|
-
// Phase 2A: Add composite indexes for sorting optimization
|
|
122
|
-
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at);`
|
|
123
|
-
await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL;`
|
|
124
|
-
await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL;`
|
|
125
|
-
}
|
|
221
|
+
PRIMARY KEY (id, type_id),
|
|
222
|
+
UNIQUE(entity_id, type_id)
|
|
223
|
+
) PARTITION BY HASH (type_id);`;
|
|
126
224
|
|
|
225
|
+
// Create hash partitions
|
|
226
|
+
for (let i = 0; i < partitionCount; i++) {
|
|
227
|
+
await db.unsafe(`CREATE TABLE IF NOT EXISTS components_p${i}
|
|
228
|
+
PARTITION OF components
|
|
229
|
+
FOR VALUES WITH (MODULUS ${partitionCount}, REMAINDER ${i});`);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_id ON components (entity_id)`;
|
|
233
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_id ON components (type_id)`;
|
|
234
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_data_gin ON components USING GIN (data)`;
|
|
235
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_type_deleted ON components (entity_id, type_id, deleted_at)`;
|
|
236
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_deleted ON components (type_id, deleted_at) WHERE deleted_at IS NULL`;
|
|
237
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_deleted_entity ON components (deleted_at, entity_id) WHERE deleted_at IS NULL`;
|
|
238
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_entity_created_desc ON components (entity_id, created_at DESC)`;
|
|
239
|
+
await db`CREATE INDEX IF NOT EXISTS idx_components_type_entity_created ON components (type_id, entity_id, created_at DESC)`;
|
|
240
|
+
}
|
|
127
241
|
export const UpdateComponentIndexes = async (table_name: string, indexedProperties: string[]) => {
|
|
128
242
|
try {
|
|
129
243
|
table_name = validateIdentifier(table_name);
|
|
130
244
|
indexedProperties = indexedProperties.map(prop => validateIdentifier(prop));
|
|
131
245
|
logger.trace(`Updating indexes for component table: ${table_name}`);
|
|
246
|
+
|
|
247
|
+
// Check if this is a hash partitioned table
|
|
248
|
+
const partitionStrategy = await GetPartitionStrategy();
|
|
249
|
+
if (partitionStrategy === 'hash' && table_name !== 'components') {
|
|
250
|
+
// For hash partitioning, indexes should be on the parent table
|
|
251
|
+
logger.trace(`Redirecting index update to parent table 'components' for hash partitioning`);
|
|
252
|
+
table_name = 'components';
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Check if table is partitioned
|
|
256
|
+
const partitionCheck = await db.unsafe(`
|
|
257
|
+
SELECT relkind
|
|
258
|
+
FROM pg_class
|
|
259
|
+
WHERE relname = '${table_name}' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
|
|
260
|
+
`);
|
|
261
|
+
const isPartitioned = partitionCheck.length > 0 && partitionCheck[0].relkind === 'p';
|
|
262
|
+
const useConcurrently = !isPartitioned && !process.env.USE_PGLITE; // Cannot use CONCURRENTLY on partitioned tables or PGlite
|
|
263
|
+
|
|
132
264
|
const indexes_list = await db.unsafe(`
|
|
133
265
|
SELECT indexname
|
|
134
266
|
FROM pg_indexes
|
|
@@ -144,7 +276,16 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
|
|
|
144
276
|
if (!existingIndexes.includes(indexName)) {
|
|
145
277
|
logger.trace(`Creating missing index ${indexName} for property ${prop}`);
|
|
146
278
|
await retryWithBackoff(async () => {
|
|
147
|
-
|
|
279
|
+
try {
|
|
280
|
+
await db.unsafe(`CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName} ON ${table_name} USING GIN ((data->'${prop}'))`);
|
|
281
|
+
} catch (error: any) {
|
|
282
|
+
// Check if the error is about duplicate key (index already exists)
|
|
283
|
+
if (error.message && error.message.includes('duplicate key value violates unique constraint "pg_class_relname_nsp_index"')) {
|
|
284
|
+
logger.trace(`Index ${indexName} already exists (confirmed by error), skipping creation`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
148
289
|
});
|
|
149
290
|
addedIndexes.add(indexName);
|
|
150
291
|
} else {
|
|
@@ -160,7 +301,16 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
|
|
|
160
301
|
const prop = match[1];
|
|
161
302
|
if (!indexedProperties.includes(prop) && !addedIndexes.has(index)) {
|
|
162
303
|
await retryWithBackoff(async () => {
|
|
163
|
-
|
|
304
|
+
try {
|
|
305
|
+
await db.unsafe(`DROP INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF EXISTS ${index}`);
|
|
306
|
+
} catch (error: any) {
|
|
307
|
+
// Check if the error is about relation does not exist
|
|
308
|
+
if (error.message && (error.message.includes('does not exist') || error.message.includes('not found'))) {
|
|
309
|
+
logger.trace(`Index ${index} does not exist, skipping drop`);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
throw error;
|
|
313
|
+
}
|
|
164
314
|
});
|
|
165
315
|
logger.info(`Dropped obsolete index ${index} for property ${prop}`);
|
|
166
316
|
}
|
|
@@ -176,12 +326,34 @@ export const UpdateComponentIndexes = async (table_name: string, indexedProperti
|
|
|
176
326
|
export const CreateComponentPartitionTable = async (comp_name: string, type_id: string) => {
|
|
177
327
|
try {
|
|
178
328
|
comp_name = validateIdentifier(comp_name);
|
|
179
|
-
// // type_id is a value, not identifier, so no validation
|
|
180
|
-
// if (indexedProperties) {
|
|
181
|
-
// indexedProperties = indexedProperties.map(prop => validateIdentifier(prop));
|
|
182
|
-
// }
|
|
183
329
|
logger.trace(`Attempt adding partition table for component: ${comp_name}`);
|
|
184
|
-
|
|
330
|
+
|
|
331
|
+
// Check partitioning strategy
|
|
332
|
+
const partitionStrategy = await GetPartitionStrategy();
|
|
333
|
+
logger.trace(`Current partition strategy: ${partitionStrategy}`);
|
|
334
|
+
|
|
335
|
+
if (partitionStrategy === 'hash') {
|
|
336
|
+
// For HASH partitioning, partitions are pre-created and data is automatically distributed
|
|
337
|
+
// We just need to ensure indexes are created for this component type
|
|
338
|
+
logger.info(`Component ${comp_name} will use existing hash partitions`);
|
|
339
|
+
|
|
340
|
+
// For hash partitioning, indexes are created at the parent table level
|
|
341
|
+
// But we can still create component-specific indexes if needed
|
|
342
|
+
const storage = getMetadataStorage();
|
|
343
|
+
const componentId = storage.getComponentId(comp_name);
|
|
344
|
+
const indexedFields = storage.getIndexedFields(componentId);
|
|
345
|
+
|
|
346
|
+
if (indexedFields.length > 0) {
|
|
347
|
+
logger.trace(`Ensuring specialized indexes for ${comp_name} on hash partitions`);
|
|
348
|
+
// For hash partitioning, indexes on parent table should suffice
|
|
349
|
+
// But we can add component-specific logic here if needed
|
|
350
|
+
logger.trace(`Hash partitioning handles indexes at parent table level`);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Original LIST partitioning logic
|
|
185
357
|
const table_name = GenerateTableName(comp_name);
|
|
186
358
|
logger.trace(`Checking for existing partition table: ${table_name}`);
|
|
187
359
|
const existingPartition = await db.unsafe(`SELECT 1 FROM information_schema.tables
|
|
@@ -202,17 +374,22 @@ export const CreateComponentPartitionTable = async (comp_name: string, type_id:
|
|
|
202
374
|
});
|
|
203
375
|
logger.trace(`Successfully created partition table: ${table_name}`);
|
|
204
376
|
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
377
|
+
// Automatically create indexes based on component metadata
|
|
378
|
+
const storage = getMetadataStorage();
|
|
379
|
+
const componentId = storage.getComponentId(comp_name);
|
|
380
|
+
const indexedFields = storage.getIndexedFields(componentId);
|
|
381
|
+
|
|
382
|
+
if (indexedFields.length > 0) {
|
|
383
|
+
logger.trace(`Creating ${indexedFields.length} specialized indexes for ${comp_name}`);
|
|
384
|
+
const indexDefinitions = indexedFields.map(field => ({
|
|
385
|
+
tableName: table_name,
|
|
386
|
+
field: field.propertyKey,
|
|
387
|
+
indexType: field.indexType,
|
|
388
|
+
isDateField: field.isDateField
|
|
389
|
+
}));
|
|
390
|
+
await ensureMultipleJSONBPathIndexes(table_name, indexDefinitions);
|
|
391
|
+
logger.trace(`Created specialized indexes for ${comp_name}`);
|
|
392
|
+
}
|
|
216
393
|
|
|
217
394
|
} catch (error) {
|
|
218
395
|
logger.error(`Failed to create component partition table for ${comp_name}: ${error}`);
|
|
@@ -223,6 +400,18 @@ export const CreateComponentPartitionTable = async (comp_name: string, type_id:
|
|
|
223
400
|
export const DeleteComponentPartitionTable = async (comp_name: string) => {
|
|
224
401
|
try {
|
|
225
402
|
comp_name = validateIdentifier(comp_name);
|
|
403
|
+
|
|
404
|
+
// Check partitioning strategy
|
|
405
|
+
const partitionStrategy = await GetPartitionStrategy();
|
|
406
|
+
|
|
407
|
+
if (partitionStrategy === 'hash') {
|
|
408
|
+
// For HASH partitioning, partitions are managed automatically
|
|
409
|
+
// No individual partition tables to delete
|
|
410
|
+
logger.info(`Component ${comp_name} uses hash partitions - no individual table to delete`);
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Original LIST partitioning logic
|
|
226
415
|
const table_name = `components_${comp_name.toLowerCase().replace(/\s+/g, '_')}`;
|
|
227
416
|
|
|
228
417
|
const existingPartition = await db.unsafe(`
|
|
@@ -249,17 +438,204 @@ export const DeleteComponentPartitionTable = async (comp_name: string) => {
|
|
|
249
438
|
|
|
250
439
|
export const CreateEntityComponentTable = async () => {
|
|
251
440
|
await db`CREATE TABLE IF NOT EXISTS entity_components (
|
|
252
|
-
entity_id UUID REFERENCES entities(id),
|
|
441
|
+
entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,
|
|
253
442
|
type_id VARCHAR(64) NOT NULL,
|
|
443
|
+
component_id UUID,
|
|
444
|
+
created_at TIMESTAMP DEFAULT NOW(),
|
|
445
|
+
updated_at TIMESTAMP DEFAULT NOW(),
|
|
254
446
|
deleted_at TIMESTAMP,
|
|
255
447
|
UNIQUE(entity_id, type_id)
|
|
256
448
|
);`;
|
|
257
|
-
|
|
258
|
-
await db`CREATE INDEX
|
|
259
|
-
await db`CREATE INDEX
|
|
449
|
+
const concurrently = process.env.USE_PGLITE ? '' : ' CONCURRENTLY';
|
|
450
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_entity_id ON entity_components (entity_id)`);
|
|
451
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_id ON entity_components (type_id)`);
|
|
452
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_entity ON entity_components (type_id, entity_id)`);
|
|
453
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_type_entity_deleted ON entity_components (type_id, entity_id, deleted_at)`);
|
|
454
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_deleted_type ON entity_components (deleted_at, type_id) WHERE deleted_at IS NULL`);
|
|
455
|
+
await db.unsafe(`CREATE INDEX${concurrently} IF NOT EXISTS idx_entity_components_component_id ON entity_components (component_id)`);
|
|
456
|
+
|
|
457
|
+
// Add component_id column if it doesn't exist (for backward compatibility)
|
|
458
|
+
try {
|
|
459
|
+
await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
|
|
460
|
+
logger.info(`Added component_id column to entity_components table`);
|
|
461
|
+
} catch (error) {
|
|
462
|
+
logger.warn(`Could not add component_id column to entity_components table: ${error}`);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
export const PopulateComponentIds = async () => {
|
|
467
|
+
try {
|
|
468
|
+
// Populate component_id for existing rows that don't have it set
|
|
469
|
+
await db`UPDATE entity_components
|
|
470
|
+
SET component_id = c.id
|
|
471
|
+
FROM components c
|
|
472
|
+
WHERE entity_components.entity_id = c.entity_id
|
|
473
|
+
AND entity_components.type_id = c.type_id
|
|
474
|
+
AND entity_components.component_id IS NULL`;
|
|
475
|
+
|
|
476
|
+
logger.info(`Populated component_id for existing entity_components rows`);
|
|
477
|
+
} catch (error) {
|
|
478
|
+
logger.warn(`Could not populate component_id for existing rows: ${error}`);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
260
481
|
|
|
261
|
-
|
|
262
|
-
|
|
482
|
+
export const EnsureDatabaseMigrations = async () => {
|
|
483
|
+
logger.trace(`Checking for database migrations...`);
|
|
484
|
+
|
|
485
|
+
try {
|
|
486
|
+
// First, ensure the table exists and has the basic structure
|
|
487
|
+
await CreateEntityComponentTable();
|
|
488
|
+
|
|
489
|
+
// Check if entity_components table has component_id column
|
|
490
|
+
const columnCheck = await db`SELECT column_name FROM information_schema.columns
|
|
491
|
+
WHERE table_name = 'entity_components'
|
|
492
|
+
AND column_name = 'component_id'
|
|
493
|
+
AND table_schema = 'public'`;
|
|
494
|
+
|
|
495
|
+
if (columnCheck.length === 0) {
|
|
496
|
+
logger.info(`entity_components table missing component_id column, adding it...`);
|
|
497
|
+
// Add the column
|
|
498
|
+
await db`ALTER TABLE entity_components ADD COLUMN component_id UUID`;
|
|
499
|
+
logger.info(`Added component_id column to entity_components table`);
|
|
500
|
+
|
|
501
|
+
// Wait a bit for the column to be available
|
|
502
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
503
|
+
|
|
504
|
+
// Populate existing data
|
|
505
|
+
await PopulateComponentIds();
|
|
506
|
+
} else {
|
|
507
|
+
logger.trace(`entity_components table already has component_id column`);
|
|
508
|
+
}
|
|
509
|
+
} catch (error) {
|
|
510
|
+
logger.error(`Failed during database migration: ${error}`);
|
|
511
|
+
// Try to add the column anyway in case the check failed
|
|
512
|
+
try {
|
|
513
|
+
await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
|
|
514
|
+
logger.info(`Attempted to add component_id column as fallback`);
|
|
515
|
+
} catch (fallbackError) {
|
|
516
|
+
logger.error(`Fallback column addition also failed: ${fallbackError}`);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
export const AnalyzeAllComponentTables = async (): Promise<void> => {
|
|
522
|
+
try {
|
|
523
|
+
logger.trace(`Analyzing all component tables`);
|
|
524
|
+
|
|
525
|
+
// Check partitioning strategy
|
|
526
|
+
const partitionStrategy = await GetPartitionStrategy();
|
|
527
|
+
|
|
528
|
+
let tablePattern: string;
|
|
529
|
+
if (partitionStrategy === 'hash') {
|
|
530
|
+
// For hash partitioning, analyze the hash partition tables
|
|
531
|
+
tablePattern = 'components_p%';
|
|
532
|
+
} else {
|
|
533
|
+
// For list partitioning, analyze the component-specific partition tables
|
|
534
|
+
tablePattern = 'components_%';
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// Get all component partition tables
|
|
538
|
+
const tables = await db.unsafe(`
|
|
539
|
+
SELECT tablename
|
|
540
|
+
FROM pg_tables
|
|
541
|
+
WHERE tablename LIKE '${tablePattern}' AND schemaname = 'public'
|
|
542
|
+
`);
|
|
543
|
+
|
|
544
|
+
for (const row of tables) {
|
|
545
|
+
logger.trace(`Running ANALYZE on table ${row.tablename}`);
|
|
546
|
+
await db.unsafe(`ANALYZE ${row.tablename}`);
|
|
547
|
+
logger.trace(`Completed ANALYZE on table ${row.tablename}`);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
logger.info(`Completed ANALYZE on ${tables.length} component tables`);
|
|
551
|
+
} catch (error) {
|
|
552
|
+
logger.error(`Failed to analyze component tables: ${error}`);
|
|
553
|
+
throw error;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
export const GetPartitionStrategy = async (): Promise<'list' | 'hash' | null> => {
|
|
558
|
+
try {
|
|
559
|
+
const result = await db.unsafe(`
|
|
560
|
+
SELECT
|
|
561
|
+
CASE
|
|
562
|
+
WHEN partstrat = 'l' THEN 'list'
|
|
563
|
+
WHEN partstrat = 'h' THEN 'hash'
|
|
564
|
+
ELSE NULL
|
|
565
|
+
END as strategy
|
|
566
|
+
FROM pg_partitioned_table
|
|
567
|
+
WHERE partrelid = (SELECT oid FROM pg_class WHERE relname = 'components' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public'))
|
|
568
|
+
`);
|
|
569
|
+
return result.length > 0 ? result[0].strategy : null;
|
|
570
|
+
} catch (error) {
|
|
571
|
+
logger.warn(`Could not determine partition strategy: ${error}`);
|
|
572
|
+
return null;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 16, 32]) => {
|
|
577
|
+
const results: Array<{partitionCount: number, planningTime: number, executionTime: number}> = [];
|
|
578
|
+
|
|
579
|
+
for (const count of partitionCounts) {
|
|
580
|
+
logger.info(`Benchmarking with ${count} partitions`);
|
|
581
|
+
|
|
582
|
+
// Create temporary hash partitioned table
|
|
583
|
+
const tempTableName = `components_benchmark_${count}`;
|
|
584
|
+
await db.unsafe(`CREATE TABLE ${tempTableName} (
|
|
585
|
+
id UUID,
|
|
586
|
+
entity_id UUID,
|
|
587
|
+
type_id varchar(64) NOT NULL,
|
|
588
|
+
name varchar(128),
|
|
589
|
+
data jsonb,
|
|
590
|
+
created_at TIMESTAMP DEFAULT NOW(),
|
|
591
|
+
updated_at TIMESTAMP DEFAULT NOW(),
|
|
592
|
+
deleted_at TIMESTAMP,
|
|
593
|
+
PRIMARY KEY (id, type_id),
|
|
594
|
+
UNIQUE(entity_id, type_id)
|
|
595
|
+
) PARTITION BY HASH (type_id);`);
|
|
596
|
+
|
|
597
|
+
// Create partitions
|
|
598
|
+
for (let i = 0; i < count; i++) {
|
|
599
|
+
await db.unsafe(`CREATE TABLE ${tempTableName}_p${i}
|
|
600
|
+
PARTITION OF ${tempTableName}
|
|
601
|
+
FOR VALUES WITH (MODULUS ${count}, REMAINDER ${i});`);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// Copy sample data (limit to avoid long benchmark)
|
|
605
|
+
await db.unsafe(`INSERT INTO ${tempTableName} (id, entity_id, type_id, name, data, created_at, updated_at, deleted_at)
|
|
606
|
+
SELECT id, entity_id, type_id, name, data, created_at, updated_at, deleted_at
|
|
607
|
+
FROM components
|
|
608
|
+
TABLESAMPLE BERNOULLI(10) -- Sample 10% of data
|
|
609
|
+
LIMIT 10000;`);
|
|
610
|
+
|
|
611
|
+
// Create indexes
|
|
612
|
+
await db.unsafe(`CREATE INDEX idx_${tempTableName}_type_id ON ${tempTableName} (type_id)`);
|
|
613
|
+
await db.unsafe(`ANALYZE ${tempTableName}`);
|
|
614
|
+
|
|
615
|
+
// Run benchmark query
|
|
616
|
+
const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
|
|
617
|
+
SELECT DISTINCT ec.entity_id as id
|
|
618
|
+
FROM entity_components ec
|
|
619
|
+
WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1)
|
|
620
|
+
AND ec.deleted_at IS NULL`);
|
|
621
|
+
|
|
622
|
+
const plan = explainResult[0]['QUERY PLAN'] ? JSON.parse(explainResult[0]['QUERY PLAN']) : explainResult[0];
|
|
623
|
+
const planningTime = plan.Planning ? plan.Planning.Time : 0;
|
|
624
|
+
const executionTime = plan.Execution ? plan.Execution.Time : 0;
|
|
625
|
+
|
|
626
|
+
results.push({
|
|
627
|
+
partitionCount: count,
|
|
628
|
+
planningTime,
|
|
629
|
+
executionTime
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
// Clean up
|
|
633
|
+
await db.unsafe(`DROP TABLE ${tempTableName} CASCADE;`);
|
|
634
|
+
|
|
635
|
+
logger.info(`Partition count ${count}: planning=${planningTime}ms, execution=${executionTime}ms`);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
return results;
|
|
263
639
|
}
|
|
264
640
|
|
|
265
641
|
export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;
|