@njdamstra/appwrite-utils-cli 1.8.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +1133 -0
- package/dist/adapters/AdapterFactory.d.ts +94 -0
- package/dist/adapters/AdapterFactory.js +405 -0
- package/dist/adapters/DatabaseAdapter.d.ts +233 -0
- package/dist/adapters/DatabaseAdapter.js +50 -0
- package/dist/adapters/LegacyAdapter.d.ts +50 -0
- package/dist/adapters/LegacyAdapter.js +612 -0
- package/dist/adapters/TablesDBAdapter.d.ts +45 -0
- package/dist/adapters/TablesDBAdapter.js +571 -0
- package/dist/adapters/index.d.ts +11 -0
- package/dist/adapters/index.js +12 -0
- package/dist/backups/operations/bucketBackup.d.ts +19 -0
- package/dist/backups/operations/bucketBackup.js +197 -0
- package/dist/backups/operations/collectionBackup.d.ts +30 -0
- package/dist/backups/operations/collectionBackup.js +201 -0
- package/dist/backups/operations/comprehensiveBackup.d.ts +25 -0
- package/dist/backups/operations/comprehensiveBackup.js +238 -0
- package/dist/backups/schemas/bucketManifest.d.ts +93 -0
- package/dist/backups/schemas/bucketManifest.js +33 -0
- package/dist/backups/schemas/comprehensiveManifest.d.ts +108 -0
- package/dist/backups/schemas/comprehensiveManifest.js +32 -0
- package/dist/backups/tracking/centralizedTracking.d.ts +34 -0
- package/dist/backups/tracking/centralizedTracking.js +274 -0
- package/dist/cli/commands/configCommands.d.ts +8 -0
- package/dist/cli/commands/configCommands.js +166 -0
- package/dist/cli/commands/databaseCommands.d.ts +13 -0
- package/dist/cli/commands/databaseCommands.js +554 -0
- package/dist/cli/commands/functionCommands.d.ts +7 -0
- package/dist/cli/commands/functionCommands.js +330 -0
- package/dist/cli/commands/schemaCommands.d.ts +7 -0
- package/dist/cli/commands/schemaCommands.js +169 -0
- package/dist/cli/commands/storageCommands.d.ts +5 -0
- package/dist/cli/commands/storageCommands.js +143 -0
- package/dist/cli/commands/transferCommands.d.ts +5 -0
- package/dist/cli/commands/transferCommands.js +384 -0
- package/dist/collections/attributes.d.ts +13 -0
- package/dist/collections/attributes.js +1364 -0
- package/dist/collections/indexes.d.ts +12 -0
- package/dist/collections/indexes.js +217 -0
- package/dist/collections/methods.d.ts +19 -0
- package/dist/collections/methods.js +682 -0
- package/dist/collections/tableOperations.d.ts +86 -0
- package/dist/collections/tableOperations.js +434 -0
- package/dist/collections/transferOperations.d.ts +8 -0
- package/dist/collections/transferOperations.js +412 -0
- package/dist/collections/wipeOperations.d.ts +16 -0
- package/dist/collections/wipeOperations.js +233 -0
- package/dist/config/ConfigManager.d.ts +445 -0
- package/dist/config/ConfigManager.js +625 -0
- package/dist/config/configMigration.d.ts +87 -0
- package/dist/config/configMigration.js +390 -0
- package/dist/config/configValidation.d.ts +66 -0
- package/dist/config/configValidation.js +358 -0
- package/dist/config/index.d.ts +8 -0
- package/dist/config/index.js +7 -0
- package/dist/config/services/ConfigDiscoveryService.d.ts +126 -0
- package/dist/config/services/ConfigDiscoveryService.js +374 -0
- package/dist/config/services/ConfigLoaderService.d.ts +129 -0
- package/dist/config/services/ConfigLoaderService.js +540 -0
- package/dist/config/services/ConfigMergeService.d.ts +208 -0
- package/dist/config/services/ConfigMergeService.js +308 -0
- package/dist/config/services/ConfigValidationService.d.ts +214 -0
- package/dist/config/services/ConfigValidationService.js +310 -0
- package/dist/config/services/SessionAuthService.d.ts +225 -0
- package/dist/config/services/SessionAuthService.js +456 -0
- package/dist/config/services/__tests__/ConfigMergeService.test.d.ts +1 -0
- package/dist/config/services/__tests__/ConfigMergeService.test.js +271 -0
- package/dist/config/services/index.d.ts +13 -0
- package/dist/config/services/index.js +10 -0
- package/dist/config/yamlConfig.d.ts +722 -0
- package/dist/config/yamlConfig.js +702 -0
- package/dist/databases/methods.d.ts +6 -0
- package/dist/databases/methods.js +35 -0
- package/dist/databases/setup.d.ts +5 -0
- package/dist/databases/setup.js +45 -0
- package/dist/examples/yamlTerminologyExample.d.ts +42 -0
- package/dist/examples/yamlTerminologyExample.js +272 -0
- package/dist/functions/deployments.d.ts +4 -0
- package/dist/functions/deployments.js +146 -0
- package/dist/functions/fnConfigDiscovery.d.ts +3 -0
- package/dist/functions/fnConfigDiscovery.js +108 -0
- package/dist/functions/methods.d.ts +16 -0
- package/dist/functions/methods.js +162 -0
- package/dist/functions/pathResolution.d.ts +37 -0
- package/dist/functions/pathResolution.js +185 -0
- package/dist/functions/templates/count-docs-in-collection/README.md +54 -0
- package/dist/functions/templates/count-docs-in-collection/src/main.ts +159 -0
- package/dist/functions/templates/count-docs-in-collection/src/request.ts +9 -0
- package/dist/functions/templates/hono-typescript/README.md +286 -0
- package/dist/functions/templates/hono-typescript/src/adapters/request.ts +74 -0
- package/dist/functions/templates/hono-typescript/src/adapters/response.ts +106 -0
- package/dist/functions/templates/hono-typescript/src/app.ts +180 -0
- package/dist/functions/templates/hono-typescript/src/context.ts +103 -0
- package/dist/functions/templates/hono-typescript/src/index.ts +54 -0
- package/dist/functions/templates/hono-typescript/src/middleware/appwrite.ts +119 -0
- package/dist/functions/templates/typescript-node/README.md +32 -0
- package/dist/functions/templates/typescript-node/src/context.ts +103 -0
- package/dist/functions/templates/typescript-node/src/index.ts +29 -0
- package/dist/functions/templates/uv/README.md +31 -0
- package/dist/functions/templates/uv/pyproject.toml +30 -0
- package/dist/functions/templates/uv/src/__init__.py +0 -0
- package/dist/functions/templates/uv/src/context.py +125 -0
- package/dist/functions/templates/uv/src/index.py +46 -0
- package/dist/init.d.ts +2 -0
- package/dist/init.js +57 -0
- package/dist/interactiveCLI.d.ts +31 -0
- package/dist/interactiveCLI.js +898 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.js +1172 -0
- package/dist/migrations/afterImportActions.d.ts +17 -0
- package/dist/migrations/afterImportActions.js +306 -0
- package/dist/migrations/appwriteToX.d.ts +211 -0
- package/dist/migrations/appwriteToX.js +491 -0
- package/dist/migrations/comprehensiveTransfer.d.ts +147 -0
- package/dist/migrations/comprehensiveTransfer.js +1317 -0
- package/dist/migrations/dataLoader.d.ts +753 -0
- package/dist/migrations/dataLoader.js +1250 -0
- package/dist/migrations/importController.d.ts +23 -0
- package/dist/migrations/importController.js +268 -0
- package/dist/migrations/importDataActions.d.ts +50 -0
- package/dist/migrations/importDataActions.js +230 -0
- package/dist/migrations/relationships.d.ts +29 -0
- package/dist/migrations/relationships.js +204 -0
- package/dist/migrations/services/DataTransformationService.d.ts +55 -0
- package/dist/migrations/services/DataTransformationService.js +158 -0
- package/dist/migrations/services/FileHandlerService.d.ts +75 -0
- package/dist/migrations/services/FileHandlerService.js +236 -0
- package/dist/migrations/services/ImportOrchestrator.d.ts +97 -0
- package/dist/migrations/services/ImportOrchestrator.js +485 -0
- package/dist/migrations/services/RateLimitManager.d.ts +138 -0
- package/dist/migrations/services/RateLimitManager.js +279 -0
- package/dist/migrations/services/RelationshipResolver.d.ts +120 -0
- package/dist/migrations/services/RelationshipResolver.js +332 -0
- package/dist/migrations/services/UserMappingService.d.ts +109 -0
- package/dist/migrations/services/UserMappingService.js +277 -0
- package/dist/migrations/services/ValidationService.d.ts +74 -0
- package/dist/migrations/services/ValidationService.js +260 -0
- package/dist/migrations/transfer.d.ts +26 -0
- package/dist/migrations/transfer.js +608 -0
- package/dist/migrations/yaml/YamlImportConfigLoader.d.ts +131 -0
- package/dist/migrations/yaml/YamlImportConfigLoader.js +383 -0
- package/dist/migrations/yaml/YamlImportIntegration.d.ts +93 -0
- package/dist/migrations/yaml/YamlImportIntegration.js +341 -0
- package/dist/migrations/yaml/generateImportSchemas.d.ts +30 -0
- package/dist/migrations/yaml/generateImportSchemas.js +1327 -0
- package/dist/schemas/authUser.d.ts +24 -0
- package/dist/schemas/authUser.js +17 -0
- package/dist/setup.d.ts +2 -0
- package/dist/setup.js +5 -0
- package/dist/setupCommands.d.ts +58 -0
- package/dist/setupCommands.js +490 -0
- package/dist/setupController.d.ts +9 -0
- package/dist/setupController.js +34 -0
- package/dist/shared/attributeMapper.d.ts +20 -0
- package/dist/shared/attributeMapper.js +203 -0
- package/dist/shared/backupMetadataSchema.d.ts +94 -0
- package/dist/shared/backupMetadataSchema.js +38 -0
- package/dist/shared/backupTracking.d.ts +18 -0
- package/dist/shared/backupTracking.js +176 -0
- package/dist/shared/confirmationDialogs.d.ts +75 -0
- package/dist/shared/confirmationDialogs.js +236 -0
- package/dist/shared/errorUtils.d.ts +54 -0
- package/dist/shared/errorUtils.js +95 -0
- package/dist/shared/functionManager.d.ts +48 -0
- package/dist/shared/functionManager.js +336 -0
- package/dist/shared/indexManager.d.ts +24 -0
- package/dist/shared/indexManager.js +151 -0
- package/dist/shared/jsonSchemaGenerator.d.ts +50 -0
- package/dist/shared/jsonSchemaGenerator.js +290 -0
- package/dist/shared/logging.d.ts +61 -0
- package/dist/shared/logging.js +116 -0
- package/dist/shared/messageFormatter.d.ts +39 -0
- package/dist/shared/messageFormatter.js +162 -0
- package/dist/shared/migrationHelpers.d.ts +61 -0
- package/dist/shared/migrationHelpers.js +145 -0
- package/dist/shared/operationLogger.d.ts +10 -0
- package/dist/shared/operationLogger.js +12 -0
- package/dist/shared/operationQueue.d.ts +40 -0
- package/dist/shared/operationQueue.js +311 -0
- package/dist/shared/operationsTable.d.ts +26 -0
- package/dist/shared/operationsTable.js +286 -0
- package/dist/shared/operationsTableSchema.d.ts +48 -0
- package/dist/shared/operationsTableSchema.js +35 -0
- package/dist/shared/progressManager.d.ts +62 -0
- package/dist/shared/progressManager.js +215 -0
- package/dist/shared/pydanticModelGenerator.d.ts +17 -0
- package/dist/shared/pydanticModelGenerator.js +615 -0
- package/dist/shared/relationshipExtractor.d.ts +56 -0
- package/dist/shared/relationshipExtractor.js +138 -0
- package/dist/shared/schemaGenerator.d.ts +40 -0
- package/dist/shared/schemaGenerator.js +556 -0
- package/dist/shared/selectionDialogs.d.ts +214 -0
- package/dist/shared/selectionDialogs.js +544 -0
- package/dist/storage/backupCompression.d.ts +20 -0
- package/dist/storage/backupCompression.js +67 -0
- package/dist/storage/methods.d.ts +32 -0
- package/dist/storage/methods.js +472 -0
- package/dist/storage/schemas.d.ts +842 -0
- package/dist/storage/schemas.js +175 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.js +3 -0
- package/dist/users/methods.d.ts +16 -0
- package/dist/users/methods.js +277 -0
- package/dist/utils/ClientFactory.d.ts +87 -0
- package/dist/utils/ClientFactory.js +212 -0
- package/dist/utils/configDiscovery.d.ts +78 -0
- package/dist/utils/configDiscovery.js +472 -0
- package/dist/utils/configMigration.d.ts +1 -0
- package/dist/utils/configMigration.js +261 -0
- package/dist/utils/constantsGenerator.d.ts +31 -0
- package/dist/utils/constantsGenerator.js +321 -0
- package/dist/utils/dataConverters.d.ts +46 -0
- package/dist/utils/dataConverters.js +139 -0
- package/dist/utils/directoryUtils.d.ts +22 -0
- package/dist/utils/directoryUtils.js +59 -0
- package/dist/utils/getClientFromConfig.d.ts +39 -0
- package/dist/utils/getClientFromConfig.js +199 -0
- package/dist/utils/helperFunctions.d.ts +63 -0
- package/dist/utils/helperFunctions.js +156 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/loadConfigs.d.ts +50 -0
- package/dist/utils/loadConfigs.js +358 -0
- package/dist/utils/pathResolvers.d.ts +53 -0
- package/dist/utils/pathResolvers.js +72 -0
- package/dist/utils/projectConfig.d.ts +119 -0
- package/dist/utils/projectConfig.js +171 -0
- package/dist/utils/retryFailedPromises.d.ts +2 -0
- package/dist/utils/retryFailedPromises.js +23 -0
- package/dist/utils/sessionAuth.d.ts +48 -0
- package/dist/utils/sessionAuth.js +164 -0
- package/dist/utils/setupFiles.d.ts +4 -0
- package/dist/utils/setupFiles.js +1192 -0
- package/dist/utils/typeGuards.d.ts +35 -0
- package/dist/utils/typeGuards.js +57 -0
- package/dist/utils/validationRules.d.ts +43 -0
- package/dist/utils/validationRules.js +42 -0
- package/dist/utils/versionDetection.d.ts +58 -0
- package/dist/utils/versionDetection.js +251 -0
- package/dist/utils/yamlConverter.d.ts +100 -0
- package/dist/utils/yamlConverter.js +428 -0
- package/dist/utils/yamlLoader.d.ts +70 -0
- package/dist/utils/yamlLoader.js +267 -0
- package/dist/utilsController.d.ts +106 -0
- package/dist/utilsController.js +863 -0
- package/package.json +75 -0
- package/scripts/copy-templates.ts +23 -0
- package/src/adapters/AdapterFactory.ts +510 -0
- package/src/adapters/DatabaseAdapter.ts +306 -0
- package/src/adapters/LegacyAdapter.ts +841 -0
- package/src/adapters/TablesDBAdapter.ts +773 -0
- package/src/adapters/index.ts +37 -0
- package/src/backups/operations/bucketBackup.ts +277 -0
- package/src/backups/operations/collectionBackup.ts +310 -0
- package/src/backups/operations/comprehensiveBackup.ts +342 -0
- package/src/backups/schemas/bucketManifest.ts +78 -0
- package/src/backups/schemas/comprehensiveManifest.ts +76 -0
- package/src/backups/tracking/centralizedTracking.ts +352 -0
- package/src/cli/commands/configCommands.ts +201 -0
- package/src/cli/commands/databaseCommands.ts +749 -0
- package/src/cli/commands/functionCommands.ts +418 -0
- package/src/cli/commands/schemaCommands.ts +200 -0
- package/src/cli/commands/storageCommands.ts +152 -0
- package/src/cli/commands/transferCommands.ts +457 -0
- package/src/collections/attributes.ts +2054 -0
- package/src/collections/attributes.ts.backup +1555 -0
- package/src/collections/indexes.ts +352 -0
- package/src/collections/methods.ts +745 -0
- package/src/collections/tableOperations.ts +506 -0
- package/src/collections/transferOperations.ts +590 -0
- package/src/collections/wipeOperations.ts +346 -0
- package/src/config/ConfigManager.ts +808 -0
- package/src/config/README.md +274 -0
- package/src/config/configMigration.ts +575 -0
- package/src/config/configValidation.ts +445 -0
- package/src/config/index.ts +10 -0
- package/src/config/services/ConfigDiscoveryService.ts +463 -0
- package/src/config/services/ConfigLoaderService.ts +740 -0
- package/src/config/services/ConfigMergeService.ts +388 -0
- package/src/config/services/ConfigValidationService.ts +394 -0
- package/src/config/services/SessionAuthService.ts +565 -0
- package/src/config/services/__tests__/ConfigMergeService.test.ts +351 -0
- package/src/config/services/index.ts +29 -0
- package/src/config/yamlConfig.ts +761 -0
- package/src/databases/methods.ts +49 -0
- package/src/databases/setup.ts +77 -0
- package/src/examples/yamlTerminologyExample.ts +346 -0
- package/src/functions/deployments.ts +220 -0
- package/src/functions/fnConfigDiscovery.ts +103 -0
- package/src/functions/methods.ts +271 -0
- package/src/functions/pathResolution.ts +227 -0
- package/src/functions/templates/count-docs-in-collection/README.md +54 -0
- package/src/functions/templates/count-docs-in-collection/src/main.ts +159 -0
- package/src/functions/templates/count-docs-in-collection/src/request.ts +9 -0
- package/src/functions/templates/hono-typescript/README.md +286 -0
- package/src/functions/templates/hono-typescript/src/adapters/request.ts +74 -0
- package/src/functions/templates/hono-typescript/src/adapters/response.ts +106 -0
- package/src/functions/templates/hono-typescript/src/app.ts +180 -0
- package/src/functions/templates/hono-typescript/src/context.ts +103 -0
- package/src/functions/templates/hono-typescript/src/index.ts +54 -0
- package/src/functions/templates/hono-typescript/src/middleware/appwrite.ts +119 -0
- package/src/functions/templates/typescript-node/README.md +32 -0
- package/src/functions/templates/typescript-node/src/context.ts +103 -0
- package/src/functions/templates/typescript-node/src/index.ts +29 -0
- package/src/functions/templates/uv/README.md +31 -0
- package/src/functions/templates/uv/pyproject.toml +30 -0
- package/src/functions/templates/uv/src/__init__.py +0 -0
- package/src/functions/templates/uv/src/context.py +125 -0
- package/src/functions/templates/uv/src/index.py +46 -0
- package/src/init.ts +62 -0
- package/src/interactiveCLI.ts +1136 -0
- package/src/main.ts +1661 -0
- package/src/migrations/afterImportActions.ts +580 -0
- package/src/migrations/appwriteToX.ts +664 -0
- package/src/migrations/comprehensiveTransfer.ts +2285 -0
- package/src/migrations/dataLoader.ts +1702 -0
- package/src/migrations/importController.ts +428 -0
- package/src/migrations/importDataActions.ts +315 -0
- package/src/migrations/relationships.ts +334 -0
- package/src/migrations/services/DataTransformationService.ts +196 -0
- package/src/migrations/services/FileHandlerService.ts +311 -0
- package/src/migrations/services/ImportOrchestrator.ts +666 -0
- package/src/migrations/services/RateLimitManager.ts +363 -0
- package/src/migrations/services/RelationshipResolver.ts +461 -0
- package/src/migrations/services/UserMappingService.ts +345 -0
- package/src/migrations/services/ValidationService.ts +349 -0
- package/src/migrations/transfer.ts +1068 -0
- package/src/migrations/yaml/YamlImportConfigLoader.ts +439 -0
- package/src/migrations/yaml/YamlImportIntegration.ts +446 -0
- package/src/migrations/yaml/generateImportSchemas.ts +1354 -0
- package/src/schemas/authUser.ts +23 -0
- package/src/setup.ts +8 -0
- package/src/setupCommands.ts +603 -0
- package/src/setupController.ts +43 -0
- package/src/shared/attributeMapper.ts +229 -0
- package/src/shared/backupMetadataSchema.ts +93 -0
- package/src/shared/backupTracking.ts +211 -0
- package/src/shared/confirmationDialogs.ts +327 -0
- package/src/shared/errorUtils.ts +110 -0
- package/src/shared/functionManager.ts +525 -0
- package/src/shared/indexManager.ts +254 -0
- package/src/shared/jsonSchemaGenerator.ts +383 -0
- package/src/shared/logging.ts +149 -0
- package/src/shared/messageFormatter.ts +208 -0
- package/src/shared/migrationHelpers.ts +232 -0
- package/src/shared/operationLogger.ts +20 -0
- package/src/shared/operationQueue.ts +377 -0
- package/src/shared/operationsTable.ts +338 -0
- package/src/shared/operationsTableSchema.ts +60 -0
- package/src/shared/progressManager.ts +278 -0
- package/src/shared/pydanticModelGenerator.ts +618 -0
- package/src/shared/relationshipExtractor.ts +214 -0
- package/src/shared/schemaGenerator.ts +644 -0
- package/src/shared/selectionDialogs.ts +749 -0
- package/src/storage/backupCompression.ts +88 -0
- package/src/storage/methods.ts +698 -0
- package/src/storage/schemas.ts +205 -0
- package/src/types/node-appwrite-tablesdb.d.ts +44 -0
- package/src/types.ts +9 -0
- package/src/users/methods.ts +359 -0
- package/src/utils/ClientFactory.ts +240 -0
- package/src/utils/configDiscovery.ts +557 -0
- package/src/utils/configMigration.ts +348 -0
- package/src/utils/constantsGenerator.ts +369 -0
- package/src/utils/dataConverters.ts +159 -0
- package/src/utils/directoryUtils.ts +61 -0
- package/src/utils/getClientFromConfig.ts +257 -0
- package/src/utils/helperFunctions.ts +228 -0
- package/src/utils/index.ts +2 -0
- package/src/utils/loadConfigs.ts +449 -0
- package/src/utils/pathResolvers.ts +81 -0
- package/src/utils/projectConfig.ts +299 -0
- package/src/utils/retryFailedPromises.ts +29 -0
- package/src/utils/sessionAuth.ts +230 -0
- package/src/utils/setupFiles.ts +1238 -0
- package/src/utils/typeGuards.ts +65 -0
- package/src/utils/validationRules.ts +88 -0
- package/src/utils/versionDetection.ts +292 -0
- package/src/utils/yamlConverter.ts +542 -0
- package/src/utils/yamlLoader.ts +371 -0
- package/src/utilsController.ts +1203 -0
- package/tests/README.md +497 -0
- package/tests/adapters/AdapterFactory.test.ts +277 -0
- package/tests/integration/syncOperations.test.ts +463 -0
- package/tests/jest.config.js +25 -0
- package/tests/migration/configMigration.test.ts +546 -0
- package/tests/setup.ts +62 -0
- package/tests/testUtils.ts +340 -0
- package/tests/utils/loadConfigs.test.ts +350 -0
- package/tests/validation/configValidation.test.ts +412 -0
- package/tsconfig.json +44 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { type Databases } from "node-appwrite";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
|
|
4
|
+
/**
|
|
5
|
+
* Object that contains the context for an action that needs to be executed after import
|
|
6
|
+
* Used in the afterImportActionsDefinitions
|
|
7
|
+
* @type {ContextObject}
|
|
8
|
+
* @typedef {Object} ContextObject
|
|
9
|
+
* @property {string} collectionId - The ID of the collection
|
|
10
|
+
* @property {any} finalItem - The final item that was imported
|
|
11
|
+
* @property {string} action - The name of the action
|
|
12
|
+
* @property {string[]} params - The parameters for the action
|
|
13
|
+
* @property {Object} context - The context object for the action (all the data of this specific item)
|
|
14
|
+
*/
|
|
15
|
+
export declare const ContextObject: z.ZodObject<{
|
|
16
|
+
dbId: z.ZodString;
|
|
17
|
+
collectionId: z.ZodString;
|
|
18
|
+
finalItem: z.ZodAny;
|
|
19
|
+
attributeMappings: z.ZodArray<z.ZodObject<{
|
|
20
|
+
oldKey: z.ZodOptional<z.ZodString>;
|
|
21
|
+
oldKeys: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
22
|
+
targetKey: z.ZodString;
|
|
23
|
+
valueToSet: z.ZodOptional<z.ZodAny>;
|
|
24
|
+
fileData: z.ZodOptional<z.ZodObject<{
|
|
25
|
+
name: z.ZodString;
|
|
26
|
+
path: z.ZodString;
|
|
27
|
+
}, z.core.$strip>>;
|
|
28
|
+
converters: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodString>>>;
|
|
29
|
+
validationActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
30
|
+
action: z.ZodString;
|
|
31
|
+
params: z.ZodArray<z.ZodString>;
|
|
32
|
+
}, z.core.$strip>>>>;
|
|
33
|
+
postImportActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
34
|
+
action: z.ZodString;
|
|
35
|
+
params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>>;
|
|
36
|
+
}, z.core.$strip>>>>;
|
|
37
|
+
}, z.core.$strip>>;
|
|
38
|
+
context: z.ZodAny;
|
|
39
|
+
}, z.core.$strip>;
|
|
40
|
+
export type ContextObject = z.infer<typeof ContextObject>;
|
|
41
|
+
export declare const createOrFindAfterImportOperation: (database: Databases, collectionId: string, context: ContextObject) => Promise<void>;
|
|
42
|
+
export declare const addBatch: (database: Databases, data: string) => Promise<string>;
|
|
43
|
+
export declare const getAfterImportOperations: (database: Databases, collectionId: string) => Promise<{
|
|
44
|
+
$id: string;
|
|
45
|
+
$createdAt: string;
|
|
46
|
+
$updatedAt: string;
|
|
47
|
+
operationType: string;
|
|
48
|
+
collectionId: string;
|
|
49
|
+
data: any;
|
|
50
|
+
progress: number;
|
|
51
|
+
total: number;
|
|
52
|
+
error: string;
|
|
53
|
+
status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
|
|
54
|
+
batches?: string[] | undefined;
|
|
55
|
+
}[]>;
|
|
56
|
+
export declare const findOrCreateOperation: (db: DatabaseAdapter, databaseId: string, operationType: string, collectionId?: string, data?: any) => Promise<any>;
|
|
57
|
+
export declare const updateOperation: (db: DatabaseAdapter, databaseId: string, operationId: string, updates: any) => Promise<any>;
|
|
58
|
+
export declare const getOperation: (db: DatabaseAdapter, databaseId: string, operationId: string) => Promise<any>;
|
|
59
|
+
export declare const maxDataLength = 1073741820;
|
|
60
|
+
export declare const maxBatchItems = 25;
|
|
61
|
+
export declare const splitIntoBatches: (data: any[]) => any[][];
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { ID, Query } from "node-appwrite";
|
|
2
|
+
import { BatchSchema, OperationSchema } from "../storage/schemas.js";
|
|
3
|
+
import { AttributeMappingsSchema } from "@njdamstra/appwrite-utils";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { logger } from "./logging.js";
|
|
6
|
+
import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
|
|
7
|
+
import { findOrCreateOperation as findOrCreateOp, updateOperation as updateOp, getOperation as getOp } from "./operationsTable.js";
|
|
8
|
+
import { MessageFormatter } from "./messageFormatter.js";
|
|
9
|
+
/**
|
|
10
|
+
* Object that contains the context for an action that needs to be executed after import
|
|
11
|
+
* Used in the afterImportActionsDefinitions
|
|
12
|
+
* @type {ContextObject}
|
|
13
|
+
* @typedef {Object} ContextObject
|
|
14
|
+
* @property {string} collectionId - The ID of the collection
|
|
15
|
+
* @property {any} finalItem - The final item that was imported
|
|
16
|
+
* @property {string} action - The name of the action
|
|
17
|
+
* @property {string[]} params - The parameters for the action
|
|
18
|
+
* @property {Object} context - The context object for the action (all the data of this specific item)
|
|
19
|
+
*/
|
|
20
|
+
export const ContextObject = z.object({
|
|
21
|
+
dbId: z.string(),
|
|
22
|
+
collectionId: z.string(),
|
|
23
|
+
finalItem: z.any(),
|
|
24
|
+
attributeMappings: AttributeMappingsSchema,
|
|
25
|
+
context: z.any(),
|
|
26
|
+
});
|
|
27
|
+
export const createOrFindAfterImportOperation = async (database, collectionId, context) => {
|
|
28
|
+
let operation = await findOrCreateOperationLegacy(database, collectionId, "afterImportAction");
|
|
29
|
+
if (!operation.batches) {
|
|
30
|
+
operation.batches = [];
|
|
31
|
+
}
|
|
32
|
+
// Directly create a new batch for the context without checking for an existing batch
|
|
33
|
+
const contextData = JSON.stringify(context);
|
|
34
|
+
// Create a new batch with the contextData
|
|
35
|
+
const newBatchId = await addBatch(database, contextData);
|
|
36
|
+
// Update the operation with the new batch's $id
|
|
37
|
+
operation.batches = [...operation.batches, newBatchId];
|
|
38
|
+
await database.updateDocument("migrations", "currentOperations", operation.$id, { batches: operation.batches });
|
|
39
|
+
};
|
|
40
|
+
export const addBatch = async (database, data) => {
|
|
41
|
+
const batch = await database.createDocument("migrations", "batches", ID.unique(), {
|
|
42
|
+
data,
|
|
43
|
+
processed: false,
|
|
44
|
+
});
|
|
45
|
+
return batch.$id;
|
|
46
|
+
};
|
|
47
|
+
export const getAfterImportOperations = async (database, collectionId) => {
|
|
48
|
+
let lastDocumentId;
|
|
49
|
+
const allOperations = [];
|
|
50
|
+
let total = 0;
|
|
51
|
+
do {
|
|
52
|
+
const query = [
|
|
53
|
+
Query.equal("collectionId", collectionId),
|
|
54
|
+
Query.equal("operationType", "afterImportAction"),
|
|
55
|
+
Query.limit(100),
|
|
56
|
+
];
|
|
57
|
+
if (lastDocumentId) {
|
|
58
|
+
query.push(Query.cursorAfter(lastDocumentId));
|
|
59
|
+
}
|
|
60
|
+
const operations = await database.listDocuments("migrations", "currentOperations", query);
|
|
61
|
+
total = operations.total; // Update total with the latest fetch
|
|
62
|
+
allOperations.push(...operations.documents);
|
|
63
|
+
if (operations.documents.length > 0 && operations.documents.length >= 100) {
|
|
64
|
+
lastDocumentId =
|
|
65
|
+
operations.documents[operations.documents.length - 1].$id;
|
|
66
|
+
}
|
|
67
|
+
} while (allOperations.length < total);
|
|
68
|
+
const allOps = allOperations.map((op) => OperationSchema.parse(op));
|
|
69
|
+
return allOps;
|
|
70
|
+
};
|
|
71
|
+
// Legacy function for backward compatibility with old migrations database
|
|
72
|
+
const findOrCreateOperationLegacy = async (database, collectionId, operationType, additionalQueries) => {
|
|
73
|
+
const operations = await tryAwaitWithRetry(async () => await database.listDocuments("migrations", "currentOperations", [
|
|
74
|
+
Query.equal("collectionId", collectionId),
|
|
75
|
+
Query.equal("operationType", operationType),
|
|
76
|
+
Query.equal("status", "pending"),
|
|
77
|
+
...(additionalQueries || []),
|
|
78
|
+
]));
|
|
79
|
+
if (operations.documents.length > 0) {
|
|
80
|
+
return OperationSchema.parse(operations.documents[0]);
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
const op = await tryAwaitWithRetry(async () => await database.createDocument("migrations", "currentOperations", ID.unique(), {
|
|
84
|
+
operationType,
|
|
85
|
+
collectionId,
|
|
86
|
+
status: "pending",
|
|
87
|
+
batches: [],
|
|
88
|
+
progress: 0,
|
|
89
|
+
total: 0,
|
|
90
|
+
error: "",
|
|
91
|
+
}));
|
|
92
|
+
return OperationSchema.parse(op);
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
export const findOrCreateOperation = async (db, databaseId, operationType, collectionId, data) => {
|
|
96
|
+
// Use new operations table system
|
|
97
|
+
return await findOrCreateOp(db, databaseId, operationType, {
|
|
98
|
+
targetCollection: collectionId,
|
|
99
|
+
data: data
|
|
100
|
+
});
|
|
101
|
+
};
|
|
102
|
+
export const updateOperation = async (db, databaseId, operationId, updates) => {
|
|
103
|
+
// Use new operations table system
|
|
104
|
+
return await updateOp(db, databaseId, operationId, updates);
|
|
105
|
+
};
|
|
106
|
+
export const getOperation = async (db, databaseId, operationId) => {
|
|
107
|
+
// Use new operations table system
|
|
108
|
+
return await getOp(db, databaseId, operationId);
|
|
109
|
+
};
|
|
110
|
+
// Actual max 1073741824
|
|
111
|
+
export const maxDataLength = 1073741820;
|
|
112
|
+
export const maxBatchItems = 25;
|
|
113
|
+
export const splitIntoBatches = (data) => {
|
|
114
|
+
let batches = [];
|
|
115
|
+
let currentBatch = [];
|
|
116
|
+
let currentBatchLength = 0;
|
|
117
|
+
let currentBatchItemCount = 0;
|
|
118
|
+
data.forEach((item, index) => {
|
|
119
|
+
const itemLength = JSON.stringify(item).length;
|
|
120
|
+
if (itemLength > maxDataLength) {
|
|
121
|
+
MessageFormatter.warning(`Large item found at index ${index} with length ${itemLength}`, { prefix: "Batch Splitter" });
|
|
122
|
+
logger.debug("Large item data:", item);
|
|
123
|
+
}
|
|
124
|
+
// Check if adding the current item would exceed the max length or max items per batch
|
|
125
|
+
if (currentBatchLength + itemLength >= maxDataLength ||
|
|
126
|
+
currentBatchItemCount >= maxBatchItems) {
|
|
127
|
+
// If so, start a new batch
|
|
128
|
+
batches.push(currentBatch);
|
|
129
|
+
currentBatch = [item];
|
|
130
|
+
currentBatchLength = itemLength;
|
|
131
|
+
currentBatchItemCount = 1; // Reset item count for the new batch
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
// Otherwise, add the item to the current batch
|
|
135
|
+
currentBatch.push(item);
|
|
136
|
+
currentBatchLength += itemLength;
|
|
137
|
+
currentBatchItemCount++;
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
// Don't forget to add the last batch if it's not empty
|
|
141
|
+
if (currentBatch.length > 0) {
|
|
142
|
+
batches.push(currentBatch);
|
|
143
|
+
}
|
|
144
|
+
return batches;
|
|
145
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Databases, Models } from "node-appwrite";
|
|
2
|
+
import type { OperationCreate } from "../storage/schemas.js";
|
|
3
|
+
/**
|
|
4
|
+
* Legacy operation logger - deprecated
|
|
5
|
+
* This function is maintained for backward compatibility but no longer performs any logging.
|
|
6
|
+
* The operations table system has been refactored to use the dynamic adapter pattern.
|
|
7
|
+
*
|
|
8
|
+
* @deprecated This function will be removed in a future version
|
|
9
|
+
*/
|
|
10
|
+
export declare const logOperation: (db: Databases, dbId: string, operationDetails: OperationCreate, operationId?: string) => Promise<Models.Document | null>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Legacy operation logger - deprecated
|
|
3
|
+
* This function is maintained for backward compatibility but no longer performs any logging.
|
|
4
|
+
* The operations table system has been refactored to use the dynamic adapter pattern.
|
|
5
|
+
*
|
|
6
|
+
* @deprecated This function will be removed in a future version
|
|
7
|
+
*/
|
|
8
|
+
export const logOperation = async (db, dbId, operationDetails, operationId) => {
|
|
9
|
+
// No-op: Operations logging has been moved to the new operations table system
|
|
10
|
+
// Callers should migrate to using operationsTable.ts functions directly
|
|
11
|
+
return null;
|
|
12
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { type Databases, type Models } from "node-appwrite";
|
|
2
|
+
import type { Attribute } from "@njdamstra/appwrite-utils";
|
|
3
|
+
import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
|
|
4
|
+
export interface QueuedOperation {
|
|
5
|
+
type: "attribute";
|
|
6
|
+
collectionId?: string;
|
|
7
|
+
attribute?: Attribute;
|
|
8
|
+
collection?: Models.Collection;
|
|
9
|
+
dependencies?: string[];
|
|
10
|
+
}
|
|
11
|
+
export declare const queuedOperations: QueuedOperation[];
|
|
12
|
+
export declare const nameToIdMapping: Map<string, string>;
|
|
13
|
+
export declare const processedCollections: Set<string>;
|
|
14
|
+
export declare const processedAttributes: Set<string>;
|
|
15
|
+
export declare const enqueueOperation: (operation: QueuedOperation) => void;
|
|
16
|
+
/**
|
|
17
|
+
* Clear all caches and processing state - use between operations
|
|
18
|
+
*/
|
|
19
|
+
export declare const clearProcessingState: () => void;
|
|
20
|
+
/**
|
|
21
|
+
* Check if a collection has already been fully processed
|
|
22
|
+
*/
|
|
23
|
+
export declare const isCollectionProcessed: (collectionId: string, databaseId: string) => boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Mark a collection as fully processed
|
|
26
|
+
*/
|
|
27
|
+
export declare const markCollectionProcessed: (collectionId: string, collectionName: string | undefined, databaseId: string) => void;
|
|
28
|
+
/**
|
|
29
|
+
* Check if a specific attribute has been processed
|
|
30
|
+
*/
|
|
31
|
+
export declare const isAttributeProcessed: (databaseId: string, collectionId: string, attributeKey: string) => boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Mark a specific attribute as processed
|
|
34
|
+
*/
|
|
35
|
+
export declare const markAttributeProcessed: (databaseId: string, collectionId: string, attributeKey: string) => void;
|
|
36
|
+
/**
|
|
37
|
+
* Process only specific attributes in the queue, not entire collections
|
|
38
|
+
* This prevents triggering full collection re-processing cycles
|
|
39
|
+
*/
|
|
40
|
+
export declare const processQueue: (db: Databases | DatabaseAdapter, dbId: string) => Promise<void>;
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { Query } from "node-appwrite";
|
|
2
|
+
import { createOrUpdateAttributeWithStatusCheck } from "../collections/attributes.js";
|
|
3
|
+
import { fetchAndCacheCollectionByName } from "../collections/methods.js";
|
|
4
|
+
import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
|
|
5
|
+
import { logger } from "../shared/logging.js";
|
|
6
|
+
import { MessageFormatter } from "../shared/messageFormatter.js";
|
|
7
|
+
// Global state management
|
|
8
|
+
export const queuedOperations = [];
|
|
9
|
+
export const nameToIdMapping = new Map();
|
|
10
|
+
// Keys are scoped per database to avoid cross-database collisions
|
|
11
|
+
// Collections key format: `${databaseId}::${collectionId}`
|
|
12
|
+
// Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
|
|
13
|
+
export const processedCollections = new Set();
|
|
14
|
+
export const processedAttributes = new Set();
|
|
15
|
+
// Helpers to build scoped keys
|
|
16
|
+
const collectionKey = (databaseId, collectionId) => `${databaseId}::${collectionId}`;
|
|
17
|
+
const attributeKeyScoped = (databaseId, collectionId, key) => `${databaseId}::${collectionId}::${key}`;
|
|
18
|
+
export const enqueueOperation = (operation) => {
|
|
19
|
+
// Avoid duplicate queue entries for same attribute
|
|
20
|
+
const attributeKey = operation.attribute?.key;
|
|
21
|
+
const collectionId = operation.collectionId;
|
|
22
|
+
logger.info('Enqueueing operation', {
|
|
23
|
+
type: operation.type,
|
|
24
|
+
attributeKey,
|
|
25
|
+
collectionId,
|
|
26
|
+
dependencies: operation.dependencies,
|
|
27
|
+
queueSizeBefore: queuedOperations.length,
|
|
28
|
+
operation: 'enqueueOperation'
|
|
29
|
+
});
|
|
30
|
+
if (attributeKey && collectionId) {
|
|
31
|
+
const duplicateIndex = queuedOperations.findIndex((op) => op.collectionId === collectionId && op.attribute?.key === attributeKey);
|
|
32
|
+
if (duplicateIndex !== -1) {
|
|
33
|
+
MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
|
|
34
|
+
logger.info('Replacing duplicate queue entry', {
|
|
35
|
+
attributeKey,
|
|
36
|
+
collectionId,
|
|
37
|
+
duplicateIndex,
|
|
38
|
+
operation: 'enqueueOperation'
|
|
39
|
+
});
|
|
40
|
+
queuedOperations[duplicateIndex] = operation;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
queuedOperations.push(operation);
|
|
45
|
+
logger.debug('Operation enqueued successfully', {
|
|
46
|
+
attributeKey,
|
|
47
|
+
collectionId,
|
|
48
|
+
queueSizeAfter: queuedOperations.length,
|
|
49
|
+
operation: 'enqueueOperation'
|
|
50
|
+
});
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Clear all caches and processing state - use between operations
|
|
54
|
+
*/
|
|
55
|
+
export const clearProcessingState = () => {
|
|
56
|
+
const sizeBefore = {
|
|
57
|
+
collections: processedCollections.size,
|
|
58
|
+
attributes: processedAttributes.size,
|
|
59
|
+
nameMapping: nameToIdMapping.size
|
|
60
|
+
};
|
|
61
|
+
processedCollections.clear();
|
|
62
|
+
processedAttributes.clear();
|
|
63
|
+
nameToIdMapping.clear();
|
|
64
|
+
logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
|
|
65
|
+
logger.info('Processing state cleared', {
|
|
66
|
+
sizeBefore,
|
|
67
|
+
operation: 'clearProcessingState'
|
|
68
|
+
});
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Check if a collection has already been fully processed
|
|
72
|
+
*/
|
|
73
|
+
export const isCollectionProcessed = (collectionId, databaseId) => {
|
|
74
|
+
return processedCollections.has(collectionKey(databaseId, collectionId));
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Mark a collection as fully processed
|
|
78
|
+
*/
|
|
79
|
+
export const markCollectionProcessed = (collectionId, collectionName, databaseId) => {
|
|
80
|
+
processedCollections.add(collectionKey(databaseId, collectionId));
|
|
81
|
+
const logData = {
|
|
82
|
+
databaseId,
|
|
83
|
+
collectionId,
|
|
84
|
+
collectionName,
|
|
85
|
+
totalProcessedCollections: processedCollections.size,
|
|
86
|
+
operation: 'markCollectionProcessed'
|
|
87
|
+
};
|
|
88
|
+
if (collectionName) {
|
|
89
|
+
MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
|
|
90
|
+
}
|
|
91
|
+
logger.info('Collection marked as processed', logData);
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* Check if a specific attribute has been processed
|
|
95
|
+
*/
|
|
96
|
+
export const isAttributeProcessed = (databaseId, collectionId, attributeKey) => {
|
|
97
|
+
return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Mark a specific attribute as processed
|
|
101
|
+
*/
|
|
102
|
+
export const markAttributeProcessed = (databaseId, collectionId, attributeKey) => {
|
|
103
|
+
const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
|
|
104
|
+
processedAttributes.add(identifier);
|
|
105
|
+
logger.debug('Attribute marked as processed', {
|
|
106
|
+
databaseId,
|
|
107
|
+
collectionId,
|
|
108
|
+
attributeKey,
|
|
109
|
+
identifier,
|
|
110
|
+
totalProcessedAttributes: processedAttributes.size,
|
|
111
|
+
operation: 'markAttributeProcessed'
|
|
112
|
+
});
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* Process only specific attributes in the queue, not entire collections
|
|
116
|
+
* This prevents triggering full collection re-processing cycles
|
|
117
|
+
*/
|
|
118
|
+
export const processQueue = async (db, dbId) => {
|
|
119
|
+
const startTime = Date.now();
|
|
120
|
+
if (queuedOperations.length === 0) {
|
|
121
|
+
MessageFormatter.info("No queued operations to process");
|
|
122
|
+
logger.info('Queue processing skipped - no operations', {
|
|
123
|
+
dbId,
|
|
124
|
+
operation: 'processQueue'
|
|
125
|
+
});
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
|
|
129
|
+
logger.info('Starting queue processing', {
|
|
130
|
+
dbId,
|
|
131
|
+
queueSize: queuedOperations.length,
|
|
132
|
+
operations: queuedOperations.map(op => ({
|
|
133
|
+
type: op.type,
|
|
134
|
+
attributeKey: op.attribute?.key,
|
|
135
|
+
collectionId: op.collectionId,
|
|
136
|
+
dependencies: op.dependencies
|
|
137
|
+
})),
|
|
138
|
+
operation: 'processQueue'
|
|
139
|
+
});
|
|
140
|
+
let progress = true;
|
|
141
|
+
let attempts = 0;
|
|
142
|
+
const maxAttempts = 3; // Prevent infinite loops
|
|
143
|
+
while (progress && attempts < maxAttempts) {
|
|
144
|
+
progress = false;
|
|
145
|
+
attempts++;
|
|
146
|
+
MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
|
|
147
|
+
logger.info('Queue processing attempt started', {
|
|
148
|
+
attempt: attempts,
|
|
149
|
+
maxAttempts,
|
|
150
|
+
remainingOperations: queuedOperations.length,
|
|
151
|
+
dbId,
|
|
152
|
+
operation: 'processQueue'
|
|
153
|
+
});
|
|
154
|
+
for (let i = queuedOperations.length - 1; i >= 0; i--) {
|
|
155
|
+
const operation = queuedOperations[i];
|
|
156
|
+
if (!operation.attribute || !operation.collectionId) {
|
|
157
|
+
MessageFormatter.warning("Invalid operation, removing from queue");
|
|
158
|
+
queuedOperations.splice(i, 1);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const attributeKey = operation.attribute.key;
|
|
162
|
+
const collectionId = operation.collectionId;
|
|
163
|
+
// Skip if this specific attribute was already processed (per database)
|
|
164
|
+
if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
|
|
165
|
+
MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
|
|
166
|
+
logger.debug('Removing already processed attribute from queue', {
|
|
167
|
+
attributeKey,
|
|
168
|
+
collectionId,
|
|
169
|
+
queueIndex: i,
|
|
170
|
+
operation: 'processQueue'
|
|
171
|
+
});
|
|
172
|
+
queuedOperations.splice(i, 1);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
let targetCollection;
|
|
176
|
+
// Resolve the target collection (where the attribute will be created)
|
|
177
|
+
try {
|
|
178
|
+
targetCollection = await tryAwaitWithRetry(async () => {
|
|
179
|
+
if ('getMetadata' in db && typeof db.getMetadata === 'function') {
|
|
180
|
+
// DatabaseAdapter
|
|
181
|
+
return (await db.getTable({ databaseId: dbId, tableId: collectionId })).data;
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
// Legacy Databases
|
|
185
|
+
return await db.getCollection(dbId, collectionId);
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
catch (e) {
|
|
190
|
+
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
191
|
+
MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
|
|
192
|
+
logger.error('Target collection not found during queue processing', {
|
|
193
|
+
collectionId,
|
|
194
|
+
attributeKey,
|
|
195
|
+
error: errorMessage,
|
|
196
|
+
operation: 'processQueue'
|
|
197
|
+
});
|
|
198
|
+
queuedOperations.splice(i, 1);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
// For relationship attributes, ensure the related collection exists
|
|
202
|
+
let canProcess = true;
|
|
203
|
+
if (operation.attribute.type === "relationship") {
|
|
204
|
+
const relatedCollection = operation.attribute.relatedCollection;
|
|
205
|
+
if (relatedCollection) {
|
|
206
|
+
// Try to resolve related collection by ID first, then by name
|
|
207
|
+
let relatedFound = false;
|
|
208
|
+
try {
|
|
209
|
+
await tryAwaitWithRetry(async () => {
|
|
210
|
+
if ('getMetadata' in db && typeof db.getMetadata === 'function') {
|
|
211
|
+
// DatabaseAdapter
|
|
212
|
+
return (await db.getTable({ databaseId: dbId, tableId: relatedCollection })).data;
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
// Legacy Databases
|
|
216
|
+
return await db.getCollection(dbId, relatedCollection);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
relatedFound = true;
|
|
220
|
+
nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
|
|
221
|
+
}
|
|
222
|
+
catch (_) {
|
|
223
|
+
// Try by name lookup
|
|
224
|
+
const cachedId = nameToIdMapping.get(relatedCollection);
|
|
225
|
+
if (cachedId) {
|
|
226
|
+
try {
|
|
227
|
+
await tryAwaitWithRetry(async () => {
|
|
228
|
+
if ('getMetadata' in db && typeof db.getMetadata === 'function') {
|
|
229
|
+
// DatabaseAdapter
|
|
230
|
+
return (await db.getTable({ databaseId: dbId, tableId: cachedId })).data;
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
// Legacy Databases
|
|
234
|
+
return await db.getCollection(dbId, cachedId);
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
relatedFound = true;
|
|
238
|
+
}
|
|
239
|
+
catch (_) {
|
|
240
|
+
nameToIdMapping.delete(relatedCollection); // Remove stale cache
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (!relatedFound) {
|
|
244
|
+
// Final attempt: search by name
|
|
245
|
+
try {
|
|
246
|
+
const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
|
|
247
|
+
? await db.listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
|
|
248
|
+
: await db.listCollections(dbId, [Query.equal("name", relatedCollection)]);
|
|
249
|
+
if (collections.total && collections.total > 0) {
|
|
250
|
+
const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
|
|
251
|
+
? collections.tables?.[0]
|
|
252
|
+
: collections.collections?.[0];
|
|
253
|
+
nameToIdMapping.set(relatedCollection, firstCollection.$id);
|
|
254
|
+
relatedFound = true;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
catch (_) {
|
|
258
|
+
// Related collection truly doesn't exist yet
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
if (!relatedFound) {
|
|
263
|
+
MessageFormatter.warning(`Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`);
|
|
264
|
+
canProcess = false;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
if (canProcess && targetCollection) {
|
|
269
|
+
MessageFormatter.progress(`Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`);
|
|
270
|
+
const success = await createOrUpdateAttributeWithStatusCheck(db, dbId, targetCollection, operation.attribute);
|
|
271
|
+
if (success) {
|
|
272
|
+
MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
|
|
273
|
+
logger.info('Queued attribute processed successfully', {
|
|
274
|
+
attributeKey,
|
|
275
|
+
collectionId,
|
|
276
|
+
targetCollectionName: targetCollection.name,
|
|
277
|
+
operation: 'processQueue'
|
|
278
|
+
});
|
|
279
|
+
markAttributeProcessed(dbId, collectionId, attributeKey);
|
|
280
|
+
queuedOperations.splice(i, 1);
|
|
281
|
+
progress = true;
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
|
|
285
|
+
logger.error('Failed to process queued attribute', {
|
|
286
|
+
attributeKey,
|
|
287
|
+
collectionId,
|
|
288
|
+
targetCollectionName: targetCollection.name,
|
|
289
|
+
operation: 'processQueue'
|
|
290
|
+
});
|
|
291
|
+
queuedOperations.splice(i, 1);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (queuedOperations.length === 0) {
|
|
296
|
+
break;
|
|
297
|
+
}
|
|
298
|
+
MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
|
|
299
|
+
}
|
|
300
|
+
if (queuedOperations.length > 0) {
|
|
301
|
+
MessageFormatter.warning(`${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`);
|
|
302
|
+
queuedOperations.forEach((op, index) => {
|
|
303
|
+
MessageFormatter.warning(` ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`);
|
|
304
|
+
});
|
|
305
|
+
MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
MessageFormatter.success("All queued operations processed successfully");
|
|
309
|
+
}
|
|
310
|
+
MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
|
|
311
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
|
|
2
|
+
import { type OperationRecord } from "./operationsTableSchema.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates the operations tracking table in the specified database
|
|
5
|
+
* Table is created with underscore prefix to indicate it's a system table
|
|
6
|
+
*/
|
|
7
|
+
export declare function createOperationsTable(db: DatabaseAdapter, databaseId: string): Promise<void>;
|
|
8
|
+
/**
|
|
9
|
+
* Finds an existing operation or creates a new one
|
|
10
|
+
* Useful for resuming interrupted operations
|
|
11
|
+
*/
|
|
12
|
+
export declare function findOrCreateOperation(db: DatabaseAdapter, databaseId: string, operationType: string, params?: Partial<OperationRecord>): Promise<OperationRecord>;
|
|
13
|
+
/**
|
|
14
|
+
* Updates an existing operation record
|
|
15
|
+
*/
|
|
16
|
+
export declare function updateOperation(db: DatabaseAdapter, databaseId: string, operationId: string, updates: Partial<OperationRecord>): Promise<OperationRecord>;
|
|
17
|
+
/**
|
|
18
|
+
* Gets a single operation by ID
|
|
19
|
+
*/
|
|
20
|
+
export declare function getOperation(db: DatabaseAdapter, databaseId: string, operationId: string): Promise<OperationRecord | null>;
|
|
21
|
+
/**
|
|
22
|
+
* Cleans up old completed operations
|
|
23
|
+
* @param olderThan - Optional date, defaults to operations older than 7 days
|
|
24
|
+
* @returns Number of operations deleted
|
|
25
|
+
*/
|
|
26
|
+
export declare function cleanupOperations(db: DatabaseAdapter, databaseId: string, olderThan?: Date): Promise<number>;
|