appwrite-utils-cli 1.11.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/{src/adapters/index.ts → dist/adapters/index.d.ts} +0 -1
  2. package/dist/adapters/index.js +10 -0
  3. package/dist/backups/operations/bucketBackup.d.ts +19 -0
  4. package/dist/backups/operations/bucketBackup.js +197 -0
  5. package/dist/backups/operations/collectionBackup.d.ts +30 -0
  6. package/dist/backups/operations/collectionBackup.js +201 -0
  7. package/dist/backups/operations/comprehensiveBackup.d.ts +25 -0
  8. package/dist/backups/operations/comprehensiveBackup.js +238 -0
  9. package/dist/backups/schemas/bucketManifest.d.ts +93 -0
  10. package/dist/backups/schemas/bucketManifest.js +33 -0
  11. package/dist/backups/schemas/comprehensiveManifest.d.ts +108 -0
  12. package/dist/backups/schemas/comprehensiveManifest.js +32 -0
  13. package/dist/backups/tracking/centralizedTracking.d.ts +34 -0
  14. package/dist/backups/tracking/centralizedTracking.js +274 -0
  15. package/dist/cli/commands/configCommands.d.ts +8 -0
  16. package/dist/cli/commands/configCommands.js +210 -0
  17. package/dist/cli/commands/databaseCommands.d.ts +14 -0
  18. package/dist/cli/commands/databaseCommands.js +696 -0
  19. package/dist/cli/commands/functionCommands.d.ts +7 -0
  20. package/dist/cli/commands/functionCommands.js +330 -0
  21. package/dist/cli/commands/importFileCommands.d.ts +7 -0
  22. package/dist/cli/commands/importFileCommands.js +674 -0
  23. package/dist/cli/commands/schemaCommands.d.ts +7 -0
  24. package/dist/cli/commands/schemaCommands.js +169 -0
  25. package/dist/cli/commands/storageCommands.d.ts +5 -0
  26. package/dist/cli/commands/storageCommands.js +142 -0
  27. package/dist/cli/commands/transferCommands.d.ts +5 -0
  28. package/dist/cli/commands/transferCommands.js +382 -0
  29. package/dist/collections/columns.d.ts +13 -0
  30. package/dist/collections/columns.js +1339 -0
  31. package/dist/collections/indexes.d.ts +12 -0
  32. package/dist/collections/indexes.js +215 -0
  33. package/dist/collections/methods.d.ts +19 -0
  34. package/dist/collections/methods.js +605 -0
  35. package/dist/collections/tableOperations.d.ts +87 -0
  36. package/dist/collections/tableOperations.js +466 -0
  37. package/dist/collections/transferOperations.d.ts +8 -0
  38. package/dist/collections/transferOperations.js +411 -0
  39. package/dist/collections/wipeOperations.d.ts +17 -0
  40. package/dist/collections/wipeOperations.js +306 -0
  41. package/dist/databases/methods.d.ts +6 -0
  42. package/dist/databases/methods.js +35 -0
  43. package/dist/databases/setup.d.ts +5 -0
  44. package/dist/databases/setup.js +45 -0
  45. package/dist/examples/yamlTerminologyExample.d.ts +42 -0
  46. package/dist/examples/yamlTerminologyExample.js +272 -0
  47. package/dist/functions/deployments.d.ts +4 -0
  48. package/dist/functions/deployments.js +146 -0
  49. package/dist/functions/fnConfigDiscovery.d.ts +3 -0
  50. package/dist/functions/fnConfigDiscovery.js +108 -0
  51. package/dist/functions/methods.d.ts +16 -0
  52. package/dist/functions/methods.js +174 -0
  53. package/dist/init.d.ts +2 -0
  54. package/dist/init.js +57 -0
  55. package/dist/interactiveCLI.d.ts +36 -0
  56. package/dist/interactiveCLI.js +952 -0
  57. package/dist/main.d.ts +2 -0
  58. package/dist/main.js +1125 -0
  59. package/dist/migrations/afterImportActions.d.ts +17 -0
  60. package/dist/migrations/afterImportActions.js +305 -0
  61. package/dist/migrations/appwriteToX.d.ts +211 -0
  62. package/dist/migrations/appwriteToX.js +493 -0
  63. package/dist/migrations/comprehensiveTransfer.d.ts +147 -0
  64. package/dist/migrations/comprehensiveTransfer.js +1315 -0
  65. package/dist/migrations/dataLoader.d.ts +755 -0
  66. package/dist/migrations/dataLoader.js +1272 -0
  67. package/dist/migrations/importController.d.ts +25 -0
  68. package/dist/migrations/importController.js +283 -0
  69. package/dist/migrations/importDataActions.d.ts +50 -0
  70. package/dist/migrations/importDataActions.js +230 -0
  71. package/dist/migrations/relationships.d.ts +29 -0
  72. package/dist/migrations/relationships.js +203 -0
  73. package/dist/migrations/services/DataTransformationService.d.ts +55 -0
  74. package/dist/migrations/services/DataTransformationService.js +158 -0
  75. package/dist/migrations/services/FileHandlerService.d.ts +75 -0
  76. package/dist/migrations/services/FileHandlerService.js +236 -0
  77. package/dist/migrations/services/ImportOrchestrator.d.ts +99 -0
  78. package/dist/migrations/services/ImportOrchestrator.js +493 -0
  79. package/dist/migrations/services/RateLimitManager.d.ts +138 -0
  80. package/dist/migrations/services/RateLimitManager.js +279 -0
  81. package/dist/migrations/services/RelationshipResolver.d.ts +120 -0
  82. package/dist/migrations/services/RelationshipResolver.js +332 -0
  83. package/dist/migrations/services/UserMappingService.d.ts +109 -0
  84. package/dist/migrations/services/UserMappingService.js +277 -0
  85. package/dist/migrations/services/ValidationService.d.ts +74 -0
  86. package/dist/migrations/services/ValidationService.js +260 -0
  87. package/dist/migrations/transfer.d.ts +30 -0
  88. package/dist/migrations/transfer.js +661 -0
  89. package/dist/migrations/yaml/YamlImportConfigLoader.d.ts +131 -0
  90. package/dist/migrations/yaml/YamlImportConfigLoader.js +383 -0
  91. package/dist/migrations/yaml/YamlImportIntegration.d.ts +93 -0
  92. package/dist/migrations/yaml/YamlImportIntegration.js +341 -0
  93. package/dist/migrations/yaml/generateImportSchemas.d.ts +30 -0
  94. package/dist/migrations/yaml/generateImportSchemas.js +1327 -0
  95. package/dist/schemas/authUser.d.ts +24 -0
  96. package/dist/schemas/authUser.js +17 -0
  97. package/dist/setup.d.ts +2 -0
  98. package/{src/setup.ts → dist/setup.js} +0 -3
  99. package/dist/setupCommands.d.ts +58 -0
  100. package/dist/setupCommands.js +489 -0
  101. package/dist/setupController.d.ts +9 -0
  102. package/dist/setupController.js +34 -0
  103. package/dist/shared/backupMetadataSchema.d.ts +94 -0
  104. package/dist/shared/backupMetadataSchema.js +38 -0
  105. package/dist/shared/backupTracking.d.ts +18 -0
  106. package/dist/shared/backupTracking.js +176 -0
  107. package/dist/shared/confirmationDialogs.d.ts +75 -0
  108. package/dist/shared/confirmationDialogs.js +236 -0
  109. package/dist/shared/migrationHelpers.d.ts +61 -0
  110. package/dist/shared/migrationHelpers.js +145 -0
  111. package/{src/shared/operationLogger.ts → dist/shared/operationLogger.d.ts} +1 -11
  112. package/dist/shared/operationLogger.js +12 -0
  113. package/dist/shared/operationQueue.d.ts +40 -0
  114. package/dist/shared/operationQueue.js +310 -0
  115. package/dist/shared/operationsTable.d.ts +26 -0
  116. package/dist/shared/operationsTable.js +287 -0
  117. package/dist/shared/operationsTableSchema.d.ts +48 -0
  118. package/dist/shared/operationsTableSchema.js +35 -0
  119. package/dist/shared/progressManager.d.ts +62 -0
  120. package/dist/shared/progressManager.js +215 -0
  121. package/dist/shared/relationshipExtractor.d.ts +56 -0
  122. package/dist/shared/relationshipExtractor.js +138 -0
  123. package/dist/shared/selectionDialogs.d.ts +220 -0
  124. package/dist/shared/selectionDialogs.js +588 -0
  125. package/dist/storage/backupCompression.d.ts +20 -0
  126. package/dist/storage/backupCompression.js +67 -0
  127. package/dist/storage/methods.d.ts +44 -0
  128. package/dist/storage/methods.js +475 -0
  129. package/dist/storage/schemas.d.ts +842 -0
  130. package/dist/storage/schemas.js +175 -0
  131. package/dist/tables/indexManager.d.ts +65 -0
  132. package/dist/tables/indexManager.js +294 -0
  133. package/{src/types.ts → dist/types.d.ts} +1 -6
  134. package/dist/types.js +3 -0
  135. package/dist/users/methods.d.ts +16 -0
  136. package/dist/users/methods.js +276 -0
  137. package/dist/utils/configMigration.d.ts +1 -0
  138. package/dist/utils/configMigration.js +261 -0
  139. package/dist/utils/index.js +2 -0
  140. package/dist/utils/loadConfigs.d.ts +50 -0
  141. package/dist/utils/loadConfigs.js +357 -0
  142. package/dist/utils/setupFiles.d.ts +4 -0
  143. package/dist/utils/setupFiles.js +1190 -0
  144. package/dist/utilsController.d.ts +114 -0
  145. package/dist/utilsController.js +898 -0
  146. package/package.json +6 -3
  147. package/CHANGELOG.md +0 -35
  148. package/CONFIG_TODO.md +0 -1189
  149. package/SELECTION_DIALOGS.md +0 -146
  150. package/SERVICE_IMPLEMENTATION_REPORT.md +0 -462
  151. package/scripts/copy-templates.ts +0 -23
  152. package/src/backups/operations/bucketBackup.ts +0 -277
  153. package/src/backups/operations/collectionBackup.ts +0 -310
  154. package/src/backups/operations/comprehensiveBackup.ts +0 -342
  155. package/src/backups/schemas/bucketManifest.ts +0 -78
  156. package/src/backups/schemas/comprehensiveManifest.ts +0 -76
  157. package/src/backups/tracking/centralizedTracking.ts +0 -352
  158. package/src/cli/commands/configCommands.ts +0 -265
  159. package/src/cli/commands/databaseCommands.ts +0 -931
  160. package/src/cli/commands/functionCommands.ts +0 -419
  161. package/src/cli/commands/importFileCommands.ts +0 -815
  162. package/src/cli/commands/schemaCommands.ts +0 -200
  163. package/src/cli/commands/storageCommands.ts +0 -151
  164. package/src/cli/commands/transferCommands.ts +0 -454
  165. package/src/collections/attributes.ts.backup +0 -1555
  166. package/src/collections/columns.ts +0 -2025
  167. package/src/collections/indexes.ts +0 -350
  168. package/src/collections/methods.ts +0 -714
  169. package/src/collections/tableOperations.ts +0 -542
  170. package/src/collections/transferOperations.ts +0 -589
  171. package/src/collections/wipeOperations.ts +0 -449
  172. package/src/databases/methods.ts +0 -49
  173. package/src/databases/setup.ts +0 -77
  174. package/src/examples/yamlTerminologyExample.ts +0 -346
  175. package/src/functions/deployments.ts +0 -221
  176. package/src/functions/fnConfigDiscovery.ts +0 -103
  177. package/src/functions/methods.ts +0 -284
  178. package/src/init.ts +0 -62
  179. package/src/interactiveCLI.ts +0 -1201
  180. package/src/main.ts +0 -1517
  181. package/src/migrations/afterImportActions.ts +0 -579
  182. package/src/migrations/appwriteToX.ts +0 -668
  183. package/src/migrations/comprehensiveTransfer.ts +0 -2285
  184. package/src/migrations/dataLoader.ts +0 -1729
  185. package/src/migrations/importController.ts +0 -440
  186. package/src/migrations/importDataActions.ts +0 -315
  187. package/src/migrations/relationships.ts +0 -333
  188. package/src/migrations/services/DataTransformationService.ts +0 -196
  189. package/src/migrations/services/FileHandlerService.ts +0 -311
  190. package/src/migrations/services/ImportOrchestrator.ts +0 -675
  191. package/src/migrations/services/RateLimitManager.ts +0 -363
  192. package/src/migrations/services/RelationshipResolver.ts +0 -461
  193. package/src/migrations/services/UserMappingService.ts +0 -345
  194. package/src/migrations/services/ValidationService.ts +0 -349
  195. package/src/migrations/transfer.ts +0 -1113
  196. package/src/migrations/yaml/YamlImportConfigLoader.ts +0 -439
  197. package/src/migrations/yaml/YamlImportIntegration.ts +0 -446
  198. package/src/migrations/yaml/generateImportSchemas.ts +0 -1354
  199. package/src/schemas/authUser.ts +0 -23
  200. package/src/setupCommands.ts +0 -602
  201. package/src/setupController.ts +0 -43
  202. package/src/shared/backupMetadataSchema.ts +0 -93
  203. package/src/shared/backupTracking.ts +0 -211
  204. package/src/shared/confirmationDialogs.ts +0 -327
  205. package/src/shared/migrationHelpers.ts +0 -232
  206. package/src/shared/operationQueue.ts +0 -376
  207. package/src/shared/operationsTable.ts +0 -338
  208. package/src/shared/operationsTableSchema.ts +0 -60
  209. package/src/shared/progressManager.ts +0 -278
  210. package/src/shared/relationshipExtractor.ts +0 -214
  211. package/src/shared/selectionDialogs.ts +0 -802
  212. package/src/storage/backupCompression.ts +0 -88
  213. package/src/storage/methods.ts +0 -711
  214. package/src/storage/schemas.ts +0 -205
  215. package/src/tables/indexManager.ts +0 -409
  216. package/src/types/node-appwrite-tablesdb.d.ts +0 -44
  217. package/src/users/methods.ts +0 -358
  218. package/src/utils/configMigration.ts +0 -348
  219. package/src/utils/loadConfigs.ts +0 -457
  220. package/src/utils/setupFiles.ts +0 -1236
  221. package/src/utilsController.ts +0 -1263
  222. package/tests/README.md +0 -497
  223. package/tests/adapters/AdapterFactory.test.ts +0 -277
  224. package/tests/integration/syncOperations.test.ts +0 -463
  225. package/tests/jest.config.js +0 -25
  226. package/tests/migration/configMigration.test.ts +0 -546
  227. package/tests/setup.ts +0 -62
  228. package/tests/testUtils.ts +0 -340
  229. package/tests/utils/loadConfigs.test.ts +0 -350
  230. package/tests/validation/configValidation.test.ts +0 -412
  231. package/tsconfig.json +0 -44
  232. /package/{src → dist}/functions/templates/count-docs-in-collection/README.md +0 -0
  233. /package/{src → dist}/functions/templates/count-docs-in-collection/src/main.ts +0 -0
  234. /package/{src → dist}/functions/templates/count-docs-in-collection/src/request.ts +0 -0
  235. /package/{src → dist}/functions/templates/hono-typescript/README.md +0 -0
  236. /package/{src → dist}/functions/templates/hono-typescript/src/adapters/request.ts +0 -0
  237. /package/{src → dist}/functions/templates/hono-typescript/src/adapters/response.ts +0 -0
  238. /package/{src → dist}/functions/templates/hono-typescript/src/app.ts +0 -0
  239. /package/{src → dist}/functions/templates/hono-typescript/src/context.ts +0 -0
  240. /package/{src → dist}/functions/templates/hono-typescript/src/main.ts +0 -0
  241. /package/{src → dist}/functions/templates/hono-typescript/src/middleware/appwrite.ts +0 -0
  242. /package/{src → dist}/functions/templates/typescript-node/README.md +0 -0
  243. /package/{src → dist}/functions/templates/typescript-node/src/context.ts +0 -0
  244. /package/{src → dist}/functions/templates/typescript-node/src/main.ts +0 -0
  245. /package/{src → dist}/functions/templates/uv/README.md +0 -0
  246. /package/{src → dist}/functions/templates/uv/pyproject.toml +0 -0
  247. /package/{src → dist}/functions/templates/uv/src/__init__.py +0 -0
  248. /package/{src → dist}/functions/templates/uv/src/context.py +0 -0
  249. /package/{src → dist}/functions/templates/uv/src/main.py +0 -0
  250. /package/{src/utils/index.ts → dist/utils/index.d.ts} +0 -0
@@ -0,0 +1,25 @@
1
+ import { type Databases, type Models, type Storage } from "node-appwrite";
2
+ import type { AppwriteConfig, ConfigDatabase } from "appwrite-utils";
3
+ import type { ImportDataActions } from "./importDataActions.js";
4
+ import type { SetupOptions } from "../utilsController.js";
5
+ import { DataLoader } from "./dataLoader.js";
6
+ export declare class ImportController {
7
+ private config;
8
+ private database;
9
+ private storage;
10
+ private appwriteFolderPath;
11
+ private importDataActions;
12
+ private setupOptions;
13
+ private documentCache;
14
+ private batchLimit;
15
+ private hasImportedUsers;
16
+ private postImportActionsQueue;
17
+ private databasesToRun;
18
+ private _adapter;
19
+ private getAdapter;
20
+ constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions, databasesToRun?: Models.Database[]);
21
+ run(specificCollections?: string[]): Promise<void>;
22
+ updateOthersToFinalData(updatedDb: Models.Database, targetDb: Models.Database): Promise<void>;
23
+ importCollections(db: ConfigDatabase, dataLoader: DataLoader, specificCollections?: string[]): Promise<void>;
24
+ executePostImportActions(dbId: string, dataLoader: DataLoader, specificCollections?: string[]): Promise<void>;
25
+ }
@@ -0,0 +1,283 @@
1
+ import { AppwriteException, ID, Query, } from "node-appwrite";
2
+ import { areCollectionNamesSame, tryAwaitWithRetry } from "appwrite-utils-helpers";
3
+ import { resolveAndUpdateRelationships } from "./relationships.js";
4
+ import { UsersController } from "../users/methods.js";
5
+ import { logger, AdapterFactory } from "appwrite-utils-helpers";
6
+ import { updateOperation } from "../shared/migrationHelpers.js";
7
+ import { BatchSchema, OperationCreateSchema, OperationSchema, } from "../storage/schemas.js";
8
+ import { DataLoader } from "./dataLoader.js";
9
+ import { transferDatabaseLocalToLocal, transferStorageLocalToLocal, } from "./transfer.js";
10
+ import { MessageFormatter } from "appwrite-utils-helpers";
11
+ import { ProgressManager } from "../shared/progressManager.js";
12
+ export class ImportController {
13
+ config;
14
+ database;
15
+ storage;
16
+ appwriteFolderPath;
17
+ importDataActions;
18
+ setupOptions;
19
+ documentCache;
20
+ batchLimit = 25; // Define batch size limit
21
+ hasImportedUsers = false;
22
+ postImportActionsQueue = [];
23
+ databasesToRun;
24
+ _adapter = null;
25
+ async getAdapter() {
26
+ if (!this._adapter) {
27
+ const { adapter } = await AdapterFactory.createFromConfig(this.config);
28
+ this._adapter = adapter;
29
+ }
30
+ return this._adapter;
31
+ }
32
+ constructor(config, database, storage, appwriteFolderPath, importDataActions, setupOptions, databasesToRun) {
33
+ this.config = config;
34
+ this.database = database;
35
+ this.storage = storage;
36
+ this.appwriteFolderPath = appwriteFolderPath;
37
+ this.importDataActions = importDataActions;
38
+ this.setupOptions = setupOptions;
39
+ this.documentCache = new Map();
40
+ this.databasesToRun = databasesToRun || [];
41
+ }
42
+ async run(specificCollections) {
43
+ let databasesToProcess;
44
+ if (this.databasesToRun.length > 0) {
45
+ // Use the provided databases
46
+ databasesToProcess = this.databasesToRun;
47
+ }
48
+ else {
49
+ // If no databases are specified, fetch all databases
50
+ const allDatabases = await this.database.list();
51
+ databasesToProcess = allDatabases.databases;
52
+ }
53
+ let dataLoader;
54
+ let databaseRan;
55
+ for (let db of databasesToProcess) {
56
+ MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
57
+ if (!databaseRan) {
58
+ databaseRan = db;
59
+ dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
60
+ await dataLoader.setupMaps(db.$id, specificCollections);
61
+ await dataLoader.start(db.$id, specificCollections);
62
+ await this.importCollections(db, dataLoader, specificCollections);
63
+ await resolveAndUpdateRelationships(db.$id, this.database, this.config);
64
+ await this.executePostImportActions(db.$id, dataLoader, specificCollections);
65
+ }
66
+ else if (databaseRan.$id !== db.$id) {
67
+ await this.updateOthersToFinalData(databaseRan, db);
68
+ }
69
+ MessageFormatter.divider();
70
+ MessageFormatter.success(`Finished import data for database: ${db.name}`, { prefix: "Import" });
71
+ MessageFormatter.divider();
72
+ }
73
+ }
74
+ async updateOthersToFinalData(updatedDb, targetDb) {
75
+ if (this.database) {
76
+ await transferDatabaseLocalToLocal(this.database, updatedDb.$id, targetDb.$id);
77
+ }
78
+ if (this.storage) {
79
+ // Find the corresponding database configs
80
+ const updatedDbConfig = this.config.databases.find((db) => db.$id === updatedDb.$id);
81
+ const targetDbConfig = this.config.databases.find((db) => db.$id === targetDb.$id);
82
+ const allBuckets = await this.storage.listBuckets([Query.limit(1000)]);
83
+ const bucketsWithDbIdInThem = allBuckets.buckets.filter((bucket) => bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase()));
84
+ const configuredUpdatedBucketId = `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`;
85
+ const configuredTargetBucketId = `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`;
86
+ let sourceBucketId;
87
+ let targetBucketId;
88
+ if (bucketsWithDbIdInThem.find((bucket) => bucket.$id === configuredUpdatedBucketId)) {
89
+ sourceBucketId = configuredUpdatedBucketId;
90
+ }
91
+ else if (bucketsWithDbIdInThem.find((bucket) => bucket.$id === configuredTargetBucketId)) {
92
+ targetBucketId = configuredTargetBucketId;
93
+ }
94
+ if (!sourceBucketId) {
95
+ sourceBucketId =
96
+ updatedDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
97
+ }
98
+ if (!targetBucketId) {
99
+ targetBucketId =
100
+ targetDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
101
+ }
102
+ if (sourceBucketId && targetBucketId) {
103
+ await transferStorageLocalToLocal(this.storage, sourceBucketId, targetBucketId);
104
+ }
105
+ }
106
+ }
107
+ async importCollections(db, dataLoader, specificCollections) {
108
+ const collectionsToImport = specificCollections ||
109
+ (this.config.collections
110
+ ? this.config.collections.map((c) => c.name)
111
+ : []);
112
+ for (const collection of this.config.collections || []) {
113
+ if (collectionsToImport.includes(collection.name)) {
114
+ let isUsersCollection = this.config.usersCollectionName &&
115
+ dataLoader.getCollectionKey(this.config.usersCollectionName) ===
116
+ dataLoader.getCollectionKey(collection.name);
117
+ const importOperationId = dataLoader.collectionImportOperations.get(dataLoader.getCollectionKey(collection.name));
118
+ const createBatches = (finalData) => {
119
+ let maxBatchLength = 50;
120
+ const finalBatches = [];
121
+ for (let i = 0; i < finalData.length; i++) {
122
+ if (i % maxBatchLength === 0) {
123
+ finalBatches.push([]);
124
+ }
125
+ finalBatches[finalBatches.length - 1].push(finalData[i]);
126
+ }
127
+ return finalBatches;
128
+ };
129
+ if (isUsersCollection && !this.hasImportedUsers) {
130
+ const usersDataMap = dataLoader.importMap.get(dataLoader.getCollectionKey("users"));
131
+ const usersData = usersDataMap?.data;
132
+ const usersController = new UsersController(this.config, this.database);
133
+ if (usersData) {
134
+ console.log("Found users data", usersData.length);
135
+ const userDataBatches = createBatches(usersData);
136
+ for (const batch of userDataBatches) {
137
+ console.log("Importing users batch", batch.length);
138
+ const userBatchPromises = batch
139
+ .filter((item) => {
140
+ let itemId;
141
+ if (item.finalData.userId) {
142
+ itemId = item.finalData.userId;
143
+ }
144
+ else if (item.finalData.docId) {
145
+ itemId = item.finalData.docId;
146
+ }
147
+ if (!itemId) {
148
+ return false;
149
+ }
150
+ return (item &&
151
+ item.finalData &&
152
+ !dataLoader.userExistsMap.has(itemId));
153
+ })
154
+ .map((item) => {
155
+ dataLoader.userExistsMap.set(item.finalData.userId ||
156
+ item.finalData.docId ||
157
+ item.context.userId ||
158
+ item.context.docId, true);
159
+ return usersController.createUserAndReturn(item.finalData);
160
+ });
161
+ const promiseResults = await Promise.allSettled(userBatchPromises);
162
+ for (const item of batch) {
163
+ if (item && item.finalData) {
164
+ dataLoader.userExistsMap.set(item.finalData.userId ||
165
+ item.finalData.docId ||
166
+ item.context.userId ||
167
+ item.context.docId, true);
168
+ }
169
+ }
170
+ MessageFormatter.success("Finished importing users batch", { prefix: "Import" });
171
+ }
172
+ this.hasImportedUsers = true;
173
+ MessageFormatter.success("Finished importing users", { prefix: "Import" });
174
+ }
175
+ }
176
+ let importOperation = null;
177
+ if (importOperationId) {
178
+ try {
179
+ importOperation = await this.database.getDocument("migrations", "currentOperations", importOperationId);
180
+ const adapter = await this.getAdapter();
181
+ await updateOperation(adapter, db.$id, importOperation.$id, {
182
+ status: "in_progress",
183
+ });
184
+ }
185
+ catch {
186
+ importOperation = null;
187
+ }
188
+ }
189
+ const collectionData = dataLoader.importMap.get(dataLoader.getCollectionKey(collection.name));
190
+ MessageFormatter.processing(`Processing collection: ${collection.name}...`, { prefix: "Import" });
191
+ if (!collectionData) {
192
+ MessageFormatter.warning(`No collection data for ${collection.name}`, { prefix: "Import" });
193
+ continue;
194
+ }
195
+ const dataSplit = createBatches(collectionData.data);
196
+ let processedItems = 0;
197
+ for (let i = 0; i < dataSplit.length; i++) {
198
+ const batches = dataSplit[i];
199
+ MessageFormatter.progress(`Processing batch ${i + 1} of ${dataSplit.length}`, { prefix: "Import" });
200
+ const batchPromises = batches.map((item, index) => {
201
+ try {
202
+ const id = item.finalData.docId ||
203
+ item.finalData.userId ||
204
+ item.context.docId ||
205
+ item.context.userId;
206
+ if (item.finalData.hasOwnProperty("userId")) {
207
+ delete item.finalData.userId;
208
+ }
209
+ if (item.finalData.hasOwnProperty("docId")) {
210
+ delete item.finalData.docId;
211
+ }
212
+ if (!item.finalData) {
213
+ return Promise.resolve();
214
+ }
215
+ return tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, item.finalData));
216
+ }
217
+ catch (error) {
218
+ MessageFormatter.error("Error creating document", error instanceof Error ? error : new Error(String(error)), { prefix: "Import" });
219
+ return Promise.resolve();
220
+ }
221
+ });
222
+ // Wait for all promises in the current batch to resolve
223
+ await Promise.all(batchPromises);
224
+ MessageFormatter.success(`Completed batch ${i + 1} of ${dataSplit.length}`, { prefix: "Import" });
225
+ if (importOperation) {
226
+ try {
227
+ const opAdapter = await this.getAdapter();
228
+ await updateOperation(opAdapter, db.$id, importOperation.$id, {
229
+ progress: processedItems,
230
+ });
231
+ }
232
+ catch { /* operations tracking is non-fatal */ }
233
+ }
234
+ }
235
+ if (importOperation) {
236
+ try {
237
+ const opAdapter = await this.getAdapter();
238
+ await updateOperation(opAdapter, db.$id, importOperation.$id, {
239
+ status: "completed",
240
+ });
241
+ }
242
+ catch { /* operations tracking is non-fatal */ }
243
+ }
244
+ }
245
+ }
246
+ }
247
+ async executePostImportActions(dbId, dataLoader, specificCollections) {
248
+ MessageFormatter.info("Executing post-import actions...", { prefix: "Import" });
249
+ const collectionsToProcess = specificCollections && specificCollections.length > 0
250
+ ? specificCollections
251
+ : this.config.collections
252
+ ? this.config.collections.map((c) => c.name)
253
+ : Array.from(dataLoader.importMap.keys());
254
+ MessageFormatter.info(`Collections to process: ${collectionsToProcess.join(", ")}`, { prefix: "Import" });
255
+ // Iterate over each collection in the importMap
256
+ for (const [collectionKey, collectionData,] of dataLoader.importMap.entries()) {
257
+ const allCollectionKeys = collectionsToProcess.map((c) => dataLoader.getCollectionKey(c));
258
+ if (allCollectionKeys.includes(collectionKey)) {
259
+ MessageFormatter.processing(`Processing post-import actions for collection: ${collectionKey}`, { prefix: "Import" });
260
+ // Iterate over each item in the collectionData.data
261
+ for (const item of collectionData.data) {
262
+ // Assuming each item has attributeMappings that contain actions to be executed
263
+ if (item.importDef && item.importDef.attributeMappings) {
264
+ // Use item.context as the context for action execution
265
+ const context = item.context; // Directly use item.context as the context for action execution
266
+ // Iterate through attributeMappings to execute actions
267
+ try {
268
+ // Execute post-import actions for the current attributeMapping
269
+ // Pass item.finalData as the data to be processed along with the context
270
+ await this.importDataActions.executeAfterImportActions(item.finalData, item.importDef.attributeMappings, context);
271
+ }
272
+ catch (error) {
273
+ MessageFormatter.error(`Failed to execute post-import actions for item in collection ${collectionKey}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Import" });
274
+ }
275
+ }
276
+ }
277
+ }
278
+ else {
279
+ MessageFormatter.info(`Skipping collection: ${collectionKey} because it's not valid for post-import actions`, { prefix: "Import" });
280
+ }
281
+ }
282
+ }
283
+ }
@@ -0,0 +1,50 @@
1
+ import { type Databases, type Storage } from "node-appwrite";
2
+ import type { AppwriteConfig } from "appwrite-utils";
3
+ import { type ValidationRules, type AttributeMappings } from "appwrite-utils";
4
+ import { type ConverterFunctions } from "appwrite-utils";
5
+ import { type AfterImportActions } from "appwrite-utils";
6
+ export declare class ImportDataActions {
7
+ private db;
8
+ private storage;
9
+ private config;
10
+ private converterDefinitions;
11
+ private validityRuleDefinitions;
12
+ private afterImportActionsDefinitions;
13
+ constructor(db: Databases, storage: Storage, config: AppwriteConfig, converterDefinitions: ConverterFunctions, validityRuleDefinitions: ValidationRules, afterImportActionsDefinitions: AfterImportActions);
14
+ /**
15
+ * Runs converter functions on the item based on the provided attribute mappings.
16
+ *
17
+ * @param item - The item to be transformed.
18
+ * @param attributeMappings - The mappings that define how each attribute should be transformed.
19
+ * @returns The transformed item.
20
+ */
21
+ runConverterFunctions(item: any, attributeMappings: AttributeMappings): any;
22
+ /**
23
+ * Validates a single data item based on defined validation rules.
24
+ * @param item The data item to validate.
25
+ * @param attributeMap The attribute mappings for the data item.
26
+ * @param context The context for resolving templated parameters in validation rules.
27
+ * @returns A promise that resolves to true if the item is valid, false otherwise.
28
+ */
29
+ validateItem(item: any, attributeMap: AttributeMappings, context: {
30
+ [key: string]: any;
31
+ }): boolean;
32
+ executeAfterImportActions(item: any, attributeMap: AttributeMappings, context: {
33
+ [key: string]: any;
34
+ }): Promise<void>;
35
+ executeAction(actionName: string, params: any[], // Accepts any type, including objects
36
+ context: {
37
+ [key: string]: any;
38
+ }, item: any): Promise<void>;
39
+ /**
40
+ * Resolves a templated string or object using the provided context and current data item.
41
+ * If the template is a string that starts and ends with "{}", it replaces it with the corresponding value from item or context.
42
+ * If the template is an object, it recursively resolves its properties.
43
+ * @param template The templated string or object.
44
+ * @param context The context for resolving the template.
45
+ * @param item The current data item being processed.
46
+ */
47
+ resolveTemplate(template: any, context: {
48
+ [key: string]: any;
49
+ }, item: any): any;
50
+ }
@@ -0,0 +1,230 @@
1
+ import {} from "node-appwrite";
2
+ import { validationRules, } from "appwrite-utils";
3
+ import { converterFunctions } from "appwrite-utils";
4
+ import { convertObjectBySchema } from "appwrite-utils-helpers";
5
+ import {} from "appwrite-utils";
6
+ import { afterImportActions } from "./afterImportActions.js";
7
+ import { logger } from "appwrite-utils-helpers";
8
+ import { tryAwaitWithRetry } from "appwrite-utils-helpers";
9
+ export class ImportDataActions {
10
+ db;
11
+ storage;
12
+ config;
13
+ converterDefinitions;
14
+ validityRuleDefinitions;
15
+ afterImportActionsDefinitions;
16
+ constructor(db, storage, config, converterDefinitions, validityRuleDefinitions, afterImportActionsDefinitions) {
17
+ this.db = db;
18
+ this.storage = storage;
19
+ this.config = config;
20
+ this.converterDefinitions = converterDefinitions;
21
+ this.validityRuleDefinitions = validityRuleDefinitions;
22
+ this.afterImportActionsDefinitions = afterImportActionsDefinitions;
23
+ }
24
+ /**
25
+ * Runs converter functions on the item based on the provided attribute mappings.
26
+ *
27
+ * @param item - The item to be transformed.
28
+ * @param attributeMappings - The mappings that define how each attribute should be transformed.
29
+ * @returns The transformed item.
30
+ */
31
+ runConverterFunctions(item, attributeMappings) {
32
+ const conversionSchema = attributeMappings.reduce((schema, mapping) => {
33
+ schema[mapping.targetKey] = (originalValue) => {
34
+ if (!mapping.converters) {
35
+ return originalValue;
36
+ }
37
+ return mapping.converters?.reduce((value, converterName) => {
38
+ let shouldProcessAsArray = false;
39
+ if ((converterName.includes("[Arr]") ||
40
+ converterName.includes("[arr]")) &&
41
+ Array.isArray(value)) {
42
+ shouldProcessAsArray = true;
43
+ converterName = converterName
44
+ .replace("[Arr]", "")
45
+ .replace("[arr]", "");
46
+ }
47
+ else if ((!Array.isArray(value) && converterName.includes("[Arr]")) ||
48
+ converterName.includes("[arr]")) {
49
+ converterName = converterName
50
+ .replace("[Arr]", "")
51
+ .replace("[arr]", "");
52
+ }
53
+ const converterFunction = converterFunctions[converterName];
54
+ if (converterFunction) {
55
+ if (Array.isArray(value) && !shouldProcessAsArray) {
56
+ return value.map((item) => converterFunction(item));
57
+ }
58
+ else {
59
+ return converterFunction(value);
60
+ }
61
+ }
62
+ else {
63
+ logger.warn(`Converter function '${converterName}' is not defined.`);
64
+ return value;
65
+ }
66
+ }, originalValue);
67
+ };
68
+ return schema;
69
+ }, {});
70
+ // Convert the item using the constructed schema
71
+ const convertedItem = convertObjectBySchema(item, conversionSchema);
72
+ // Merge the converted item back into the original item object
73
+ Object.assign(item, convertedItem);
74
+ return item;
75
+ }
76
+ /**
77
+ * Validates a single data item based on defined validation rules.
78
+ * @param item The data item to validate.
79
+ * @param attributeMap The attribute mappings for the data item.
80
+ * @param context The context for resolving templated parameters in validation rules.
81
+ * @returns A promise that resolves to true if the item is valid, false otherwise.
82
+ */
83
+ validateItem(item, attributeMap, context) {
84
+ for (const mapping of attributeMap) {
85
+ const { validationActions } = mapping;
86
+ if (!validationActions ||
87
+ !Array.isArray(validationActions) ||
88
+ !validationActions.length) {
89
+ return true; // Assume items without validation actions as valid.
90
+ }
91
+ for (const ruleDef of validationActions) {
92
+ const { action, params } = ruleDef;
93
+ const validationRule = validationRules[action];
94
+ if (!validationRule) {
95
+ logger.warn(`Validation rule '${action}' is not defined.`);
96
+ continue; // Optionally, consider undefined rules as a validation failure.
97
+ }
98
+ // Resolve templated parameters
99
+ const resolvedParams = params.map((param) => this.resolveTemplate(param, context, item));
100
+ // Apply the validation rule
101
+ let isValid = false;
102
+ if (Array.isArray(item)) {
103
+ isValid = item.every((item) => validationRule(item, ...resolvedParams));
104
+ }
105
+ else {
106
+ isValid = validationRule(item, ...resolvedParams);
107
+ }
108
+ if (!isValid) {
109
+ logger.error(`Validation failed for rule '${action}' with params ${params.join(", ")}`);
110
+ return false; // Stop validation on first failure
111
+ }
112
+ }
113
+ }
114
+ return true; // The item passed all validations
115
+ }
116
+ async executeAfterImportActions(item, attributeMap, context) {
117
+ for (const mapping of attributeMap) {
118
+ const { postImportActions } = mapping;
119
+ if (!postImportActions || !Array.isArray(postImportActions)) {
120
+ continue; // Skip to the next attribute if no actions are defined
121
+ }
122
+ for (const actionDef of postImportActions) {
123
+ const { action, params } = actionDef;
124
+ console.log(`Executing post-import action '${action}' for attribute '${mapping.targetKey}' with params ${params.join(", ")}...`);
125
+ try {
126
+ await tryAwaitWithRetry(async () => await this.executeAction(action, params, context, item));
127
+ }
128
+ catch (error) {
129
+ logger.error(`Failed to execute post-import action '${action}' for attribute '${mapping.targetKey}':`, error);
130
+ }
131
+ }
132
+ }
133
+ }
134
+ async executeAction(actionName, params, // Accepts any type, including objects
135
+ context, item) {
136
+ const actionMethod = afterImportActions[actionName];
137
+ if (typeof actionMethod === "function") {
138
+ try {
139
+ // Resolve parameters, handling both strings and objects
140
+ const resolvedParams = params.map((param) => {
141
+ // Directly resolve each param, whether it's an object or a string
142
+ return this.resolveTemplate(param, context, item);
143
+ });
144
+ // Execute the action with resolved parameters
145
+ // Parameters are passed as-is, with objects treated as single parameters
146
+ console.log(`Executing action '${actionName}' from context with params:`, resolvedParams);
147
+ logger.info(`Executing action '${actionName}' from context: ${JSON.stringify(context, null, 2)} with params:`, resolvedParams);
148
+ await actionMethod(this.config, ...resolvedParams);
149
+ }
150
+ catch (error) {
151
+ logger.error(`Error executing action '${actionName}' with context:`, context, error);
152
+ throw new Error(`Execution failed for action '${actionName}': ${error.message}`);
153
+ }
154
+ }
155
+ else {
156
+ logger.warn(`Action '${actionName}' is not defined.`);
157
+ throw new Error(`Action '${actionName}' is not defined.`);
158
+ }
159
+ }
160
+ /**
161
+ * Resolves a templated string or object using the provided context and current data item.
162
+ * If the template is a string that starts and ends with "{}", it replaces it with the corresponding value from item or context.
163
+ * If the template is an object, it recursively resolves its properties.
164
+ * @param template The templated string or object.
165
+ * @param context The context for resolving the template.
166
+ * @param item The current data item being processed.
167
+ */
168
+ resolveTemplate(template, context, item) {
169
+ // Function to recursively resolve paths, including handling [any] notation
170
+ const resolvePath = (path, currentContext) => {
171
+ const anyKeyRegex = /\[any\]/g;
172
+ let pathParts = path.split(".").filter(Boolean);
173
+ return pathParts.reduce((acc, part, index) => {
174
+ // Handle [any] part by iterating over all elements if it's an object or an array
175
+ if (part === "[any]") {
176
+ if (Array.isArray(acc)) {
177
+ return acc
178
+ .map((item) => item[pathParts[index + 1]])
179
+ .filter((item) => item !== undefined);
180
+ }
181
+ else if (typeof acc === "object") {
182
+ return Object.values(acc)
183
+ .map((item) => item[pathParts[index + 1]])
184
+ .filter((item) => item !== undefined);
185
+ }
186
+ }
187
+ else {
188
+ return acc?.[part];
189
+ }
190
+ }, currentContext);
191
+ };
192
+ if (typeof template === "string") {
193
+ // Matches placeholders in the template
194
+ const regex = /\{([^}]+)\}/g;
195
+ let match;
196
+ let resolvedString = template;
197
+ while ((match = regex.exec(template)) !== null) {
198
+ const path = match[1];
199
+ // Resolve the path, handling [any] notation and arrays/objects
200
+ const resolvedValue = resolvePath(path, { ...context, ...item });
201
+ if (resolvedValue !== undefined) {
202
+ // If it's an array (from [any] notation), join the values; adjust as needed
203
+ const value = Array.isArray(resolvedValue)
204
+ ? resolvedValue.join(", ")
205
+ : resolvedValue;
206
+ resolvedString = resolvedString.replace(match[0], value);
207
+ }
208
+ else {
209
+ logger.warn(`Failed to resolve ${template} in context: `, JSON.stringify({ ...context, ...item }, null, 2));
210
+ }
211
+ }
212
+ // console.log(`Resolved string: ${resolvedString}`);
213
+ return resolvedString;
214
+ }
215
+ else if (typeof template === "object" && template !== null) {
216
+ // Recursively resolve templates for each property in the object
217
+ const resolvedObject = Array.isArray(template) ? [] : {};
218
+ for (const key in template) {
219
+ const resolvedValue = this.resolveTemplate(template[key], context, item);
220
+ if (resolvedValue !== undefined) {
221
+ // Only assign if resolvedValue is not undefined
222
+ resolvedObject[key] = resolvedValue;
223
+ }
224
+ }
225
+ return resolvedObject;
226
+ }
227
+ // console.log(`Template is not a string or object: ${template}`);
228
+ return template;
229
+ }
230
+ }
@@ -0,0 +1,29 @@
1
+ import { Databases } from "node-appwrite";
2
+ import type { AppwriteConfig } from "appwrite-utils";
3
+ /**
4
+ * Finds collections that have defined relationship attributes.
5
+ */
6
+ export declare const findCollectionsWithRelationships: (config: AppwriteConfig) => Map<string, {
7
+ key: string;
8
+ required: boolean;
9
+ type: "relationship";
10
+ relatedCollection: string;
11
+ relationType: "oneToMany" | "manyToOne" | "oneToOne" | "manyToMany";
12
+ twoWay: boolean;
13
+ onDelete: "setNull" | "cascade" | "restrict";
14
+ array?: boolean | undefined;
15
+ format?: string | undefined;
16
+ status?: string | undefined;
17
+ attributes?: string[] | undefined;
18
+ orders?: string[] | undefined;
19
+ $createdAt?: string | undefined;
20
+ $updatedAt?: string | undefined;
21
+ error?: string | undefined;
22
+ twoWayKey?: string | undefined;
23
+ side?: "parent" | "child" | undefined;
24
+ importMapping?: {
25
+ originalIdField: string;
26
+ targetField?: string | undefined;
27
+ } | undefined;
28
+ }[]>;
29
+ export declare function resolveAndUpdateRelationships(dbId: string, database: Databases, config: AppwriteConfig): Promise<void>;