appwrite-utils-cli 1.9.6 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/CONFIG_TODO.md +1189 -1189
  2. package/README.md +1004 -1004
  3. package/SELECTION_DIALOGS.md +145 -145
  4. package/SERVICE_IMPLEMENTATION_REPORT.md +462 -462
  5. package/package.json +6 -3
  6. package/scripts/copy-templates.ts +23 -23
  7. package/src/adapters/index.ts +11 -37
  8. package/src/backups/operations/bucketBackup.ts +277 -277
  9. package/src/backups/operations/collectionBackup.ts +310 -310
  10. package/src/backups/operations/comprehensiveBackup.ts +342 -342
  11. package/src/backups/schemas/bucketManifest.ts +78 -78
  12. package/src/backups/schemas/comprehensiveManifest.ts +76 -76
  13. package/src/backups/tracking/centralizedTracking.ts +352 -352
  14. package/src/cli/commands/configCommands.ts +265 -201
  15. package/src/cli/commands/databaseCommands.ts +931 -879
  16. package/src/cli/commands/functionCommands.ts +333 -332
  17. package/src/cli/commands/importFileCommands.ts +815 -0
  18. package/src/cli/commands/schemaCommands.ts +141 -141
  19. package/src/cli/commands/storageCommands.ts +2 -3
  20. package/src/cli/commands/transferCommands.ts +454 -457
  21. package/src/collections/attributes.ts.backup +1555 -1555
  22. package/src/collections/{attributes.ts → columns.ts} +15 -44
  23. package/src/collections/indexes.ts +350 -352
  24. package/src/collections/methods.ts +714 -815
  25. package/src/collections/tableOperations.ts +57 -21
  26. package/src/collections/transferOperations.ts +376 -377
  27. package/src/collections/wipeOperations.ts +449 -346
  28. package/src/databases/methods.ts +49 -49
  29. package/src/databases/setup.ts +77 -77
  30. package/src/examples/yamlTerminologyExample.ts +346 -346
  31. package/src/functions/deployments.ts +221 -220
  32. package/src/functions/fnConfigDiscovery.ts +2 -2
  33. package/src/functions/methods.ts +284 -284
  34. package/src/functions/templates/count-docs-in-collection/README.md +53 -53
  35. package/src/functions/templates/count-docs-in-collection/src/main.ts +159 -159
  36. package/src/functions/templates/count-docs-in-collection/src/request.ts +8 -8
  37. package/src/functions/templates/hono-typescript/README.md +285 -285
  38. package/src/functions/templates/hono-typescript/src/adapters/request.ts +73 -73
  39. package/src/functions/templates/hono-typescript/src/adapters/response.ts +105 -105
  40. package/src/functions/templates/hono-typescript/src/app.ts +179 -179
  41. package/src/functions/templates/hono-typescript/src/context.ts +102 -102
  42. package/src/functions/templates/hono-typescript/src/{index.ts → main.ts} +53 -53
  43. package/src/functions/templates/hono-typescript/src/middleware/appwrite.ts +118 -118
  44. package/src/functions/templates/typescript-node/README.md +31 -31
  45. package/src/functions/templates/typescript-node/src/context.ts +102 -102
  46. package/src/functions/templates/typescript-node/src/{index.ts → main.ts} +29 -29
  47. package/src/functions/templates/uv/README.md +30 -30
  48. package/src/functions/templates/uv/pyproject.toml +29 -29
  49. package/src/functions/templates/uv/src/context.py +124 -124
  50. package/src/functions/templates/uv/src/{index.py → main.py} +45 -45
  51. package/src/init.ts +62 -62
  52. package/src/interactiveCLI.ts +1095 -1030
  53. package/src/main.ts +1517 -1670
  54. package/src/migrations/afterImportActions.ts +579 -580
  55. package/src/migrations/appwriteToX.ts +634 -630
  56. package/src/migrations/comprehensiveTransfer.ts +2149 -2149
  57. package/src/migrations/dataLoader.ts +1729 -1702
  58. package/src/migrations/importController.ts +440 -428
  59. package/src/migrations/importDataActions.ts +315 -315
  60. package/src/migrations/relationships.ts +333 -334
  61. package/src/migrations/services/DataTransformationService.ts +195 -195
  62. package/src/migrations/services/FileHandlerService.ts +310 -310
  63. package/src/migrations/services/ImportOrchestrator.ts +674 -665
  64. package/src/migrations/services/RateLimitManager.ts +362 -362
  65. package/src/migrations/services/RelationshipResolver.ts +460 -460
  66. package/src/migrations/services/UserMappingService.ts +344 -344
  67. package/src/migrations/services/ValidationService.ts +333 -333
  68. package/src/migrations/transfer.ts +987 -942
  69. package/src/migrations/yaml/YamlImportConfigLoader.ts +438 -438
  70. package/src/migrations/yaml/YamlImportIntegration.ts +438 -438
  71. package/src/migrations/yaml/generateImportSchemas.ts +1347 -1347
  72. package/src/schemas/authUser.ts +23 -23
  73. package/src/setup.ts +8 -8
  74. package/src/setupCommands.ts +5 -6
  75. package/src/setupController.ts +42 -42
  76. package/src/shared/backupMetadataSchema.ts +93 -93
  77. package/src/shared/backupTracking.ts +211 -211
  78. package/src/shared/confirmationDialogs.ts +326 -326
  79. package/src/shared/migrationHelpers.ts +232 -232
  80. package/src/shared/operationLogger.ts +20 -20
  81. package/src/shared/operationQueue.ts +326 -327
  82. package/src/shared/operationsTable.ts +338 -338
  83. package/src/shared/operationsTableSchema.ts +60 -60
  84. package/src/shared/progressManager.ts +277 -277
  85. package/src/shared/relationshipExtractor.ts +214 -214
  86. package/src/shared/selectionDialogs.ts +775 -722
  87. package/src/storage/backupCompression.ts +88 -88
  88. package/src/storage/methods.ts +695 -682
  89. package/src/storage/schemas.ts +205 -205
  90. package/src/tables/indexManager.ts +409 -0
  91. package/src/types/node-appwrite-tablesdb.d.ts +43 -43
  92. package/src/types.ts +9 -9
  93. package/src/users/methods.ts +358 -359
  94. package/src/utils/configMigration.ts +347 -347
  95. package/src/utils/index.ts +2 -2
  96. package/src/utils/loadConfigs.ts +457 -449
  97. package/src/utils/setupFiles.ts +1236 -1238
  98. package/src/utilsController.ts +1263 -1213
  99. package/tests/README.md +496 -496
  100. package/tests/adapters/AdapterFactory.test.ts +276 -276
  101. package/tests/integration/syncOperations.test.ts +462 -462
  102. package/tests/jest.config.js +24 -24
  103. package/tests/migration/configMigration.test.ts +545 -545
  104. package/tests/setup.ts +61 -61
  105. package/tests/testUtils.ts +339 -339
  106. package/tests/utils/loadConfigs.test.ts +349 -349
  107. package/tests/validation/configValidation.test.ts +411 -411
  108. package/tsconfig.json +44 -44
  109. package/.appwrite/.yaml_schemas/appwrite-config.schema.json +0 -380
  110. package/.appwrite/.yaml_schemas/collection.schema.json +0 -255
  111. package/.appwrite/collections/Categories.yaml +0 -182
  112. package/.appwrite/collections/ExampleCollection.yaml +0 -36
  113. package/.appwrite/collections/Posts.yaml +0 -227
  114. package/.appwrite/collections/Users.yaml +0 -149
  115. package/.appwrite/config.yaml +0 -109
  116. package/.appwrite/import/README.md +0 -148
  117. package/.appwrite/import/categories-import.yaml +0 -129
  118. package/.appwrite/import/posts-import.yaml +0 -208
  119. package/.appwrite/import/users-import.yaml +0 -130
  120. package/.appwrite/importData/categories.json +0 -194
  121. package/.appwrite/importData/posts.json +0 -270
  122. package/.appwrite/importData/users.json +0 -220
  123. package/.appwrite/schemas/categories.json +0 -128
  124. package/.appwrite/schemas/exampleCollection.json +0 -52
  125. package/.appwrite/schemas/posts.json +0 -173
  126. package/.appwrite/schemas/users.json +0 -125
  127. package/dist/adapters/AdapterFactory.d.ts +0 -94
  128. package/dist/adapters/AdapterFactory.js +0 -405
  129. package/dist/adapters/DatabaseAdapter.d.ts +0 -242
  130. package/dist/adapters/DatabaseAdapter.js +0 -50
  131. package/dist/adapters/LegacyAdapter.d.ts +0 -50
  132. package/dist/adapters/LegacyAdapter.js +0 -612
  133. package/dist/adapters/TablesDBAdapter.d.ts +0 -45
  134. package/dist/adapters/TablesDBAdapter.js +0 -596
  135. package/dist/adapters/index.d.ts +0 -11
  136. package/dist/adapters/index.js +0 -12
  137. package/dist/backups/operations/bucketBackup.d.ts +0 -19
  138. package/dist/backups/operations/bucketBackup.js +0 -197
  139. package/dist/backups/operations/collectionBackup.d.ts +0 -30
  140. package/dist/backups/operations/collectionBackup.js +0 -201
  141. package/dist/backups/operations/comprehensiveBackup.d.ts +0 -25
  142. package/dist/backups/operations/comprehensiveBackup.js +0 -238
  143. package/dist/backups/schemas/bucketManifest.d.ts +0 -93
  144. package/dist/backups/schemas/bucketManifest.js +0 -33
  145. package/dist/backups/schemas/comprehensiveManifest.d.ts +0 -108
  146. package/dist/backups/schemas/comprehensiveManifest.js +0 -32
  147. package/dist/backups/tracking/centralizedTracking.d.ts +0 -34
  148. package/dist/backups/tracking/centralizedTracking.js +0 -274
  149. package/dist/cli/commands/configCommands.d.ts +0 -8
  150. package/dist/cli/commands/configCommands.js +0 -166
  151. package/dist/cli/commands/databaseCommands.d.ts +0 -14
  152. package/dist/cli/commands/databaseCommands.js +0 -644
  153. package/dist/cli/commands/functionCommands.d.ts +0 -7
  154. package/dist/cli/commands/functionCommands.js +0 -330
  155. package/dist/cli/commands/schemaCommands.d.ts +0 -7
  156. package/dist/cli/commands/schemaCommands.js +0 -169
  157. package/dist/cli/commands/storageCommands.d.ts +0 -5
  158. package/dist/cli/commands/storageCommands.js +0 -143
  159. package/dist/cli/commands/transferCommands.d.ts +0 -5
  160. package/dist/cli/commands/transferCommands.js +0 -384
  161. package/dist/collections/attributes.d.ts +0 -13
  162. package/dist/collections/attributes.js +0 -1364
  163. package/dist/collections/indexes.d.ts +0 -12
  164. package/dist/collections/indexes.js +0 -217
  165. package/dist/collections/methods.d.ts +0 -19
  166. package/dist/collections/methods.js +0 -734
  167. package/dist/collections/tableOperations.d.ts +0 -86
  168. package/dist/collections/tableOperations.js +0 -434
  169. package/dist/collections/transferOperations.d.ts +0 -8
  170. package/dist/collections/transferOperations.js +0 -412
  171. package/dist/collections/wipeOperations.d.ts +0 -16
  172. package/dist/collections/wipeOperations.js +0 -233
  173. package/dist/config/ConfigManager.d.ts +0 -450
  174. package/dist/config/ConfigManager.js +0 -625
  175. package/dist/config/configMigration.d.ts +0 -87
  176. package/dist/config/configMigration.js +0 -390
  177. package/dist/config/configValidation.d.ts +0 -66
  178. package/dist/config/configValidation.js +0 -358
  179. package/dist/config/index.d.ts +0 -8
  180. package/dist/config/index.js +0 -7
  181. package/dist/config/services/ConfigDiscoveryService.d.ts +0 -122
  182. package/dist/config/services/ConfigDiscoveryService.js +0 -322
  183. package/dist/config/services/ConfigLoaderService.d.ts +0 -129
  184. package/dist/config/services/ConfigLoaderService.js +0 -535
  185. package/dist/config/services/ConfigMergeService.d.ts +0 -208
  186. package/dist/config/services/ConfigMergeService.js +0 -308
  187. package/dist/config/services/ConfigValidationService.d.ts +0 -214
  188. package/dist/config/services/ConfigValidationService.js +0 -310
  189. package/dist/config/services/SessionAuthService.d.ts +0 -225
  190. package/dist/config/services/SessionAuthService.js +0 -456
  191. package/dist/config/services/__tests__/ConfigMergeService.test.d.ts +0 -1
  192. package/dist/config/services/__tests__/ConfigMergeService.test.js +0 -271
  193. package/dist/config/services/index.d.ts +0 -13
  194. package/dist/config/services/index.js +0 -10
  195. package/dist/config/yamlConfig.d.ts +0 -722
  196. package/dist/config/yamlConfig.js +0 -702
  197. package/dist/databases/methods.d.ts +0 -6
  198. package/dist/databases/methods.js +0 -35
  199. package/dist/databases/setup.d.ts +0 -5
  200. package/dist/databases/setup.js +0 -45
  201. package/dist/examples/yamlTerminologyExample.d.ts +0 -42
  202. package/dist/examples/yamlTerminologyExample.js +0 -272
  203. package/dist/functions/deployments.d.ts +0 -4
  204. package/dist/functions/deployments.js +0 -146
  205. package/dist/functions/fnConfigDiscovery.d.ts +0 -3
  206. package/dist/functions/fnConfigDiscovery.js +0 -108
  207. package/dist/functions/methods.d.ts +0 -16
  208. package/dist/functions/methods.js +0 -174
  209. package/dist/functions/pathResolution.d.ts +0 -37
  210. package/dist/functions/pathResolution.js +0 -185
  211. package/dist/functions/templates/count-docs-in-collection/README.md +0 -54
  212. package/dist/functions/templates/count-docs-in-collection/package.json +0 -25
  213. package/dist/functions/templates/count-docs-in-collection/src/main.ts +0 -159
  214. package/dist/functions/templates/count-docs-in-collection/src/request.ts +0 -9
  215. package/dist/functions/templates/count-docs-in-collection/tsconfig.json +0 -28
  216. package/dist/functions/templates/hono-typescript/README.md +0 -286
  217. package/dist/functions/templates/hono-typescript/package.json +0 -26
  218. package/dist/functions/templates/hono-typescript/src/adapters/request.ts +0 -74
  219. package/dist/functions/templates/hono-typescript/src/adapters/response.ts +0 -106
  220. package/dist/functions/templates/hono-typescript/src/app.ts +0 -180
  221. package/dist/functions/templates/hono-typescript/src/context.ts +0 -103
  222. package/dist/functions/templates/hono-typescript/src/index.ts +0 -54
  223. package/dist/functions/templates/hono-typescript/src/middleware/appwrite.ts +0 -119
  224. package/dist/functions/templates/hono-typescript/tsconfig.json +0 -20
  225. package/dist/functions/templates/typescript-node/README.md +0 -32
  226. package/dist/functions/templates/typescript-node/package.json +0 -25
  227. package/dist/functions/templates/typescript-node/src/context.ts +0 -103
  228. package/dist/functions/templates/typescript-node/src/index.ts +0 -29
  229. package/dist/functions/templates/typescript-node/tsconfig.json +0 -28
  230. package/dist/functions/templates/uv/README.md +0 -31
  231. package/dist/functions/templates/uv/pyproject.toml +0 -30
  232. package/dist/functions/templates/uv/src/__init__.py +0 -0
  233. package/dist/functions/templates/uv/src/context.py +0 -125
  234. package/dist/functions/templates/uv/src/index.py +0 -46
  235. package/dist/init.d.ts +0 -2
  236. package/dist/init.js +0 -57
  237. package/dist/interactiveCLI.d.ts +0 -31
  238. package/dist/interactiveCLI.js +0 -898
  239. package/dist/main.d.ts +0 -2
  240. package/dist/main.js +0 -1180
  241. package/dist/migrations/afterImportActions.d.ts +0 -17
  242. package/dist/migrations/afterImportActions.js +0 -306
  243. package/dist/migrations/appwriteToX.d.ts +0 -211
  244. package/dist/migrations/appwriteToX.js +0 -491
  245. package/dist/migrations/comprehensiveTransfer.d.ts +0 -147
  246. package/dist/migrations/comprehensiveTransfer.js +0 -1317
  247. package/dist/migrations/dataLoader.d.ts +0 -753
  248. package/dist/migrations/dataLoader.js +0 -1250
  249. package/dist/migrations/importController.d.ts +0 -23
  250. package/dist/migrations/importController.js +0 -268
  251. package/dist/migrations/importDataActions.d.ts +0 -50
  252. package/dist/migrations/importDataActions.js +0 -230
  253. package/dist/migrations/relationships.d.ts +0 -29
  254. package/dist/migrations/relationships.js +0 -204
  255. package/dist/migrations/services/DataTransformationService.d.ts +0 -55
  256. package/dist/migrations/services/DataTransformationService.js +0 -158
  257. package/dist/migrations/services/FileHandlerService.d.ts +0 -75
  258. package/dist/migrations/services/FileHandlerService.js +0 -236
  259. package/dist/migrations/services/ImportOrchestrator.d.ts +0 -97
  260. package/dist/migrations/services/ImportOrchestrator.js +0 -485
  261. package/dist/migrations/services/RateLimitManager.d.ts +0 -138
  262. package/dist/migrations/services/RateLimitManager.js +0 -279
  263. package/dist/migrations/services/RelationshipResolver.d.ts +0 -120
  264. package/dist/migrations/services/RelationshipResolver.js +0 -332
  265. package/dist/migrations/services/UserMappingService.d.ts +0 -109
  266. package/dist/migrations/services/UserMappingService.js +0 -277
  267. package/dist/migrations/services/ValidationService.d.ts +0 -74
  268. package/dist/migrations/services/ValidationService.js +0 -260
  269. package/dist/migrations/transfer.d.ts +0 -26
  270. package/dist/migrations/transfer.js +0 -608
  271. package/dist/migrations/yaml/YamlImportConfigLoader.d.ts +0 -131
  272. package/dist/migrations/yaml/YamlImportConfigLoader.js +0 -383
  273. package/dist/migrations/yaml/YamlImportIntegration.d.ts +0 -93
  274. package/dist/migrations/yaml/YamlImportIntegration.js +0 -341
  275. package/dist/migrations/yaml/generateImportSchemas.d.ts +0 -30
  276. package/dist/migrations/yaml/generateImportSchemas.js +0 -1327
  277. package/dist/schemas/authUser.d.ts +0 -24
  278. package/dist/schemas/authUser.js +0 -17
  279. package/dist/setup.d.ts +0 -2
  280. package/dist/setup.js +0 -5
  281. package/dist/setupCommands.d.ts +0 -58
  282. package/dist/setupCommands.js +0 -490
  283. package/dist/setupController.d.ts +0 -9
  284. package/dist/setupController.js +0 -34
  285. package/dist/shared/attributeMapper.d.ts +0 -20
  286. package/dist/shared/attributeMapper.js +0 -203
  287. package/dist/shared/backupMetadataSchema.d.ts +0 -94
  288. package/dist/shared/backupMetadataSchema.js +0 -38
  289. package/dist/shared/backupTracking.d.ts +0 -18
  290. package/dist/shared/backupTracking.js +0 -176
  291. package/dist/shared/confirmationDialogs.d.ts +0 -75
  292. package/dist/shared/confirmationDialogs.js +0 -236
  293. package/dist/shared/errorUtils.d.ts +0 -54
  294. package/dist/shared/errorUtils.js +0 -95
  295. package/dist/shared/functionManager.d.ts +0 -48
  296. package/dist/shared/functionManager.js +0 -348
  297. package/dist/shared/indexManager.d.ts +0 -24
  298. package/dist/shared/indexManager.js +0 -151
  299. package/dist/shared/jsonSchemaGenerator.d.ts +0 -50
  300. package/dist/shared/jsonSchemaGenerator.js +0 -290
  301. package/dist/shared/logging.d.ts +0 -61
  302. package/dist/shared/logging.js +0 -116
  303. package/dist/shared/messageFormatter.d.ts +0 -39
  304. package/dist/shared/messageFormatter.js +0 -162
  305. package/dist/shared/migrationHelpers.d.ts +0 -61
  306. package/dist/shared/migrationHelpers.js +0 -145
  307. package/dist/shared/operationLogger.d.ts +0 -10
  308. package/dist/shared/operationLogger.js +0 -12
  309. package/dist/shared/operationQueue.d.ts +0 -40
  310. package/dist/shared/operationQueue.js +0 -311
  311. package/dist/shared/operationsTable.d.ts +0 -26
  312. package/dist/shared/operationsTable.js +0 -286
  313. package/dist/shared/operationsTableSchema.d.ts +0 -48
  314. package/dist/shared/operationsTableSchema.js +0 -35
  315. package/dist/shared/progressManager.d.ts +0 -62
  316. package/dist/shared/progressManager.js +0 -215
  317. package/dist/shared/pydanticModelGenerator.d.ts +0 -17
  318. package/dist/shared/pydanticModelGenerator.js +0 -615
  319. package/dist/shared/relationshipExtractor.d.ts +0 -56
  320. package/dist/shared/relationshipExtractor.js +0 -138
  321. package/dist/shared/schemaGenerator.d.ts +0 -40
  322. package/dist/shared/schemaGenerator.js +0 -556
  323. package/dist/shared/selectionDialogs.d.ts +0 -214
  324. package/dist/shared/selectionDialogs.js +0 -544
  325. package/dist/storage/backupCompression.d.ts +0 -20
  326. package/dist/storage/backupCompression.js +0 -67
  327. package/dist/storage/methods.d.ts +0 -32
  328. package/dist/storage/methods.js +0 -472
  329. package/dist/storage/schemas.d.ts +0 -842
  330. package/dist/storage/schemas.js +0 -175
  331. package/dist/types.d.ts +0 -4
  332. package/dist/types.js +0 -3
  333. package/dist/users/methods.d.ts +0 -16
  334. package/dist/users/methods.js +0 -277
  335. package/dist/utils/ClientFactory.d.ts +0 -87
  336. package/dist/utils/ClientFactory.js +0 -212
  337. package/dist/utils/configDiscovery.d.ts +0 -78
  338. package/dist/utils/configDiscovery.js +0 -472
  339. package/dist/utils/configMigration.d.ts +0 -1
  340. package/dist/utils/configMigration.js +0 -261
  341. package/dist/utils/constantsGenerator.d.ts +0 -31
  342. package/dist/utils/constantsGenerator.js +0 -321
  343. package/dist/utils/dataConverters.d.ts +0 -46
  344. package/dist/utils/dataConverters.js +0 -139
  345. package/dist/utils/directoryUtils.d.ts +0 -22
  346. package/dist/utils/directoryUtils.js +0 -59
  347. package/dist/utils/getClientFromConfig.d.ts +0 -39
  348. package/dist/utils/getClientFromConfig.js +0 -199
  349. package/dist/utils/helperFunctions.d.ts +0 -63
  350. package/dist/utils/helperFunctions.js +0 -156
  351. package/dist/utils/index.d.ts +0 -2
  352. package/dist/utils/index.js +0 -2
  353. package/dist/utils/loadConfigs.d.ts +0 -50
  354. package/dist/utils/loadConfigs.js +0 -358
  355. package/dist/utils/pathResolvers.d.ts +0 -53
  356. package/dist/utils/pathResolvers.js +0 -72
  357. package/dist/utils/projectConfig.d.ts +0 -122
  358. package/dist/utils/projectConfig.js +0 -206
  359. package/dist/utils/retryFailedPromises.d.ts +0 -2
  360. package/dist/utils/retryFailedPromises.js +0 -23
  361. package/dist/utils/sessionAuth.d.ts +0 -48
  362. package/dist/utils/sessionAuth.js +0 -164
  363. package/dist/utils/setupFiles.d.ts +0 -4
  364. package/dist/utils/setupFiles.js +0 -1192
  365. package/dist/utils/typeGuards.d.ts +0 -35
  366. package/dist/utils/typeGuards.js +0 -57
  367. package/dist/utils/validationRules.d.ts +0 -43
  368. package/dist/utils/validationRules.js +0 -42
  369. package/dist/utils/versionDetection.d.ts +0 -58
  370. package/dist/utils/versionDetection.js +0 -251
  371. package/dist/utils/yamlConverter.d.ts +0 -100
  372. package/dist/utils/yamlConverter.js +0 -428
  373. package/dist/utils/yamlLoader.d.ts +0 -70
  374. package/dist/utils/yamlLoader.js +0 -267
  375. package/dist/utilsController.d.ts +0 -107
  376. package/dist/utilsController.js +0 -873
  377. package/src/adapters/AdapterFactory.ts +0 -510
  378. package/src/adapters/DatabaseAdapter.ts +0 -318
  379. package/src/adapters/LegacyAdapter.ts +0 -841
  380. package/src/adapters/TablesDBAdapter.ts +0 -815
  381. package/src/config/ConfigManager.ts +0 -817
  382. package/src/config/README.md +0 -274
  383. package/src/config/configMigration.ts +0 -575
  384. package/src/config/configValidation.ts +0 -445
  385. package/src/config/index.ts +0 -10
  386. package/src/config/services/ConfigDiscoveryService.ts +0 -410
  387. package/src/config/services/ConfigLoaderService.ts +0 -732
  388. package/src/config/services/ConfigMergeService.ts +0 -388
  389. package/src/config/services/ConfigValidationService.ts +0 -394
  390. package/src/config/services/SessionAuthService.ts +0 -565
  391. package/src/config/services/__tests__/ConfigMergeService.test.ts +0 -351
  392. package/src/config/services/index.ts +0 -29
  393. package/src/config/yamlConfig.ts +0 -761
  394. package/src/functions/pathResolution.ts +0 -227
  395. package/src/functions/templates/count-docs-in-collection/package.json +0 -25
  396. package/src/functions/templates/count-docs-in-collection/tsconfig.json +0 -28
  397. package/src/functions/templates/hono-typescript/package.json +0 -26
  398. package/src/functions/templates/hono-typescript/tsconfig.json +0 -20
  399. package/src/functions/templates/typescript-node/package.json +0 -25
  400. package/src/functions/templates/typescript-node/tsconfig.json +0 -28
  401. package/src/shared/attributeMapper.ts +0 -229
  402. package/src/shared/errorUtils.ts +0 -110
  403. package/src/shared/functionManager.ts +0 -537
  404. package/src/shared/indexManager.ts +0 -254
  405. package/src/shared/jsonSchemaGenerator.ts +0 -383
  406. package/src/shared/logging.ts +0 -149
  407. package/src/shared/messageFormatter.ts +0 -208
  408. package/src/shared/pydanticModelGenerator.ts +0 -618
  409. package/src/shared/schemaGenerator.ts +0 -644
  410. package/src/utils/ClientFactory.ts +0 -240
  411. package/src/utils/configDiscovery.ts +0 -557
  412. package/src/utils/constantsGenerator.ts +0 -369
  413. package/src/utils/dataConverters.ts +0 -159
  414. package/src/utils/directoryUtils.ts +0 -61
  415. package/src/utils/getClientFromConfig.ts +0 -257
  416. package/src/utils/helperFunctions.ts +0 -228
  417. package/src/utils/pathResolvers.ts +0 -81
  418. package/src/utils/projectConfig.ts +0 -340
  419. package/src/utils/retryFailedPromises.ts +0 -29
  420. package/src/utils/sessionAuth.ts +0 -230
  421. package/src/utils/typeGuards.ts +0 -65
  422. package/src/utils/validationRules.ts +0 -88
  423. package/src/utils/versionDetection.ts +0 -292
  424. package/src/utils/yamlConverter.ts +0 -542
  425. package/src/utils/yamlLoader.ts +0 -371
  426. package/tmp-sync-test/.appwrite/collections/TestCollection.yaml +0 -7
@@ -1,412 +0,0 @@
1
- import { Client, Databases, ID, Query, } from "node-appwrite";
2
- import { tryAwaitWithRetry, delay, calculateExponentialBackoff } from "../utils/helperFunctions.js";
3
- import { MessageFormatter } from "../shared/messageFormatter.js";
4
- import { chunk } from "es-toolkit";
5
- import { isLegacyDatabases } from "../utils/typeGuards.js";
6
- import { getAdapter } from "../utils/getClientFromConfig.js";
7
- /**
8
- * Transfers all documents from one collection to another in a different database
9
- * within the same Appwrite Project
10
- */
11
- export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
12
- // Use adapter path when available for bulk operations
13
- if (!isLegacyDatabases(db)) {
14
- const adapter = db;
15
- const pageSize = 1000;
16
- let lastId;
17
- let totalTransferred = 0;
18
- while (true) {
19
- const queries = [Query.limit(pageSize)];
20
- if (lastId)
21
- queries.push(Query.cursorAfter(lastId));
22
- const result = await adapter.listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
23
- const rows = result.rows || result.documents || [];
24
- if (!rows.length)
25
- break;
26
- // Prepare rows: strip system fields, keep $id and $permissions
27
- const prepared = rows.map((doc) => {
28
- const data = { ...doc };
29
- delete data.$databaseId;
30
- delete data.$collectionId;
31
- delete data.$createdAt;
32
- delete data.$updatedAt;
33
- return data; // keep $id and $permissions for upsert
34
- });
35
- // Prefer bulk upsert, then bulk create, then individual
36
- if (typeof adapter.bulkUpsertRows === 'function' && adapter.supportsBulkOperations()) {
37
- await adapter.bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
38
- }
39
- else if (typeof adapter.bulkCreateRows === 'function' && adapter.supportsBulkOperations()) {
40
- await adapter.bulkCreateRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
41
- }
42
- else {
43
- for (const row of prepared) {
44
- const id = row.$id || ID.unique();
45
- const permissions = row.$permissions || [];
46
- const { $id, $permissions, ...data } = row;
47
- await adapter.createRow({ databaseId: toDbId, tableId: toCollId, id, data, permissions });
48
- }
49
- }
50
- totalTransferred += rows.length;
51
- if (rows.length < pageSize)
52
- break;
53
- lastId = rows[rows.length - 1].$id;
54
- }
55
- MessageFormatter.success(`Transferred ${totalTransferred} rows from ${fromDbId}/${fromCollId} to ${toDbId}/${toCollId}`, { prefix: "Transfer" });
56
- return;
57
- }
58
- // Legacy path (Databases) – keep existing behavior
59
- const legacyDb = db;
60
- let fromCollDocs = await tryAwaitWithRetry(async () => legacyDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
61
- let totalDocumentsTransferred = 0;
62
- if (fromCollDocs.documents.length === 0) {
63
- MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
64
- return;
65
- }
66
- else if (fromCollDocs.documents.length < 50) {
67
- const batchedPromises = fromCollDocs.documents.map((doc) => {
68
- const toCreateObject = {
69
- ...doc,
70
- };
71
- delete toCreateObject.$databaseId;
72
- delete toCreateObject.$collectionId;
73
- delete toCreateObject.$createdAt;
74
- delete toCreateObject.$updatedAt;
75
- delete toCreateObject.$id;
76
- delete toCreateObject.$permissions;
77
- return tryAwaitWithRetry(async () => await legacyDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
78
- });
79
- await Promise.all(batchedPromises);
80
- totalDocumentsTransferred += fromCollDocs.documents.length;
81
- }
82
- else {
83
- const batchedPromises = fromCollDocs.documents.map((doc) => {
84
- const toCreateObject = {
85
- ...doc,
86
- };
87
- delete toCreateObject.$databaseId;
88
- delete toCreateObject.$collectionId;
89
- delete toCreateObject.$createdAt;
90
- delete toCreateObject.$updatedAt;
91
- delete toCreateObject.$id;
92
- delete toCreateObject.$permissions;
93
- return tryAwaitWithRetry(async () => legacyDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
94
- });
95
- await Promise.all(batchedPromises);
96
- totalDocumentsTransferred += fromCollDocs.documents.length;
97
- while (fromCollDocs.documents.length === 50) {
98
- fromCollDocs = await tryAwaitWithRetry(async () => await legacyDb.listDocuments(fromDbId, fromCollId, [
99
- Query.limit(50),
100
- Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
101
- ]));
102
- const batchedPromises = fromCollDocs.documents.map((doc) => {
103
- const toCreateObject = {
104
- ...doc,
105
- };
106
- delete toCreateObject.$databaseId;
107
- delete toCreateObject.$collectionId;
108
- delete toCreateObject.$createdAt;
109
- delete toCreateObject.$updatedAt;
110
- delete toCreateObject.$id;
111
- delete toCreateObject.$permissions;
112
- return tryAwaitWithRetry(async () => await legacyDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
113
- });
114
- await Promise.all(batchedPromises);
115
- totalDocumentsTransferred += fromCollDocs.documents.length;
116
- }
117
- }
118
- MessageFormatter.success(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`, { prefix: "Transfer" });
119
- };
120
- /**
121
- * Enhanced document transfer with fault tolerance and exponential backoff
122
- */
123
- const transferDocumentWithRetry = async (db, dbId, collectionId, documentId, documentData, permissions, maxRetries = 3, retryCount = 0) => {
124
- try {
125
- await db.createDocument(dbId, collectionId, documentId, documentData, permissions);
126
- return true;
127
- }
128
- catch (error) {
129
- // Check if document already exists
130
- if (error.code === 409 || error.message?.toLowerCase().includes('already exists')) {
131
- await db.updateDocument(dbId, collectionId, documentId, documentData, permissions);
132
- }
133
- if (retryCount < maxRetries) {
134
- // Calculate exponential backoff: 1s, 2s, 4s, max 8s
135
- const exponentialDelay = calculateExponentialBackoff(retryCount, 1000, 8000);
136
- MessageFormatter.progress(`Retrying document ${documentId} (attempt ${retryCount + 1}/${maxRetries}, backoff: ${exponentialDelay}ms)`, { prefix: "Transfer" });
137
- await delay(exponentialDelay);
138
- return await transferDocumentWithRetry(db, dbId, collectionId, documentId, documentData, permissions, maxRetries, retryCount + 1);
139
- }
140
- MessageFormatter.error(`Failed to transfer document ${documentId} after ${maxRetries} retries`, error, { prefix: "Transfer" });
141
- return false;
142
- }
143
- };
144
- /**
145
- * Check if endpoint supports bulk operations (cloud.appwrite.io)
146
- */
147
- const supportsBulkOperations = (endpoint) => {
148
- return endpoint.includes('cloud.appwrite.io');
149
- };
150
- /**
151
- * Direct HTTP implementation of bulk upsert API
152
- */
153
- const bulkUpsertDocuments = async (client, dbId, collectionId, documents) => {
154
- const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
155
- const url = new URL(client.config.endpoint + apiPath);
156
- const headers = {
157
- 'Content-Type': 'application/json',
158
- 'X-Appwrite-Project': client.config.project,
159
- 'X-Appwrite-Key': client.config.key
160
- };
161
- const response = await fetch(url.toString(), {
162
- method: 'PUT',
163
- headers,
164
- body: JSON.stringify({ documents })
165
- });
166
- if (!response.ok) {
167
- const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
168
- throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
169
- }
170
- return await response.json();
171
- };
172
- /**
173
- * Direct HTTP implementation of bulk create API
174
- */
175
- const bulkCreateDocuments = async (client, dbId, collectionId, documents) => {
176
- const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
177
- const url = new URL(client.config.endpoint + apiPath);
178
- const headers = {
179
- 'Content-Type': 'application/json',
180
- 'X-Appwrite-Project': client.config.project,
181
- 'X-Appwrite-Key': client.config.key
182
- };
183
- const response = await fetch(url.toString(), {
184
- method: 'POST',
185
- headers,
186
- body: JSON.stringify({ documents })
187
- });
188
- if (!response.ok) {
189
- const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
190
- throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
191
- }
192
- return await response.json();
193
- };
194
- /**
195
- * Enhanced bulk document creation using direct HTTP calls
196
- */
197
- const transferDocumentsBulkUpsert = async (client, dbId, collectionId, documents, maxBatchSize = 1000) => {
198
- let successful = 0;
199
- let failed = 0;
200
- // Prepare documents for bulk upsert
201
- const preparedDocs = documents.map(doc => {
202
- const toCreateObject = { ...doc };
203
- delete toCreateObject.$databaseId;
204
- delete toCreateObject.$collectionId;
205
- delete toCreateObject.$createdAt;
206
- delete toCreateObject.$updatedAt;
207
- // Keep $id and $permissions for upsert functionality
208
- return toCreateObject;
209
- });
210
- // Process in batches based on plan limits
211
- const documentBatches = chunk(preparedDocs, maxBatchSize);
212
- for (const batch of documentBatches) {
213
- MessageFormatter.progress(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
214
- try {
215
- // Try bulk upsert with direct HTTP call
216
- const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
217
- successful += result.documents?.length || batch.length;
218
- MessageFormatter.success(`Bulk upserted ${result.documents?.length || batch.length} documents`, { prefix: "Transfer" });
219
- }
220
- catch (error) {
221
- MessageFormatter.progress(`Bulk upsert failed, trying smaller batch size...`, { prefix: "Transfer" });
222
- // If bulk upsert fails, try with smaller batch size (Pro plan limit)
223
- if (maxBatchSize > 100) {
224
- const smallerBatches = chunk(batch, 100);
225
- for (const smallBatch of smallerBatches) {
226
- try {
227
- const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
228
- successful += result.documents?.length || smallBatch.length;
229
- MessageFormatter.success(`Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`, { prefix: "Transfer" });
230
- }
231
- catch (smallBatchError) {
232
- MessageFormatter.progress(`Smaller batch failed, falling back to individual transfers...`, { prefix: "Transfer" });
233
- // Fall back to individual document transfer for this batch
234
- const db = new Databases(client);
235
- const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, smallBatch.map((doc, index) => ({
236
- ...doc,
237
- $id: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$id || ID.unique(),
238
- $permissions: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$permissions || []
239
- })));
240
- successful += indivSuccessful;
241
- failed += indivFailed;
242
- }
243
- // Add delay between batches
244
- await delay(200);
245
- }
246
- }
247
- else {
248
- // Fall back to individual document transfer
249
- const db = new Databases(client);
250
- const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, batch.map((doc, index) => ({
251
- ...doc,
252
- $id: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$id || ID.unique(),
253
- $permissions: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$permissions || []
254
- })));
255
- successful += indivSuccessful;
256
- failed += indivFailed;
257
- }
258
- }
259
- // Add delay between major batches
260
- if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
261
- await delay(500);
262
- }
263
- }
264
- return { successful, failed };
265
- };
266
- /**
267
- * Fallback batch document transfer with individual retry logic
268
- */
269
- const transferDocumentBatchWithRetryFallback = async (db, dbId, collectionId, documents, batchSize = 10) => {
270
- let successful = 0;
271
- let failed = 0;
272
- // Process documents in smaller batches to avoid overwhelming the server
273
- const documentBatches = chunk(documents, batchSize);
274
- for (const batch of documentBatches) {
275
- MessageFormatter.progress(`Processing batch of ${batch.length} documents...`, { prefix: "Transfer" });
276
- const batchPromises = batch.map(async (doc) => {
277
- const toCreateObject = { ...doc };
278
- delete toCreateObject.$databaseId;
279
- delete toCreateObject.$collectionId;
280
- delete toCreateObject.$createdAt;
281
- delete toCreateObject.$updatedAt;
282
- delete toCreateObject.$id;
283
- delete toCreateObject.$permissions;
284
- const result = await transferDocumentWithRetry(db, dbId, collectionId, doc.$id, toCreateObject, doc.$permissions || []);
285
- return { docId: doc.$id, success: result };
286
- });
287
- const results = await Promise.allSettled(batchPromises);
288
- results.forEach((result, index) => {
289
- if (result.status === 'fulfilled') {
290
- if (result.value.success) {
291
- successful++;
292
- }
293
- else {
294
- failed++;
295
- }
296
- }
297
- else {
298
- MessageFormatter.error(`Batch promise rejected for document ${batch[index].$id}`, new Error(String(result.reason)), { prefix: "Transfer" });
299
- failed++;
300
- }
301
- });
302
- // Add delay between batches to avoid rate limiting
303
- if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
304
- await delay(500);
305
- }
306
- }
307
- return { successful, failed };
308
- };
309
- /**
310
- * Enhanced batch document transfer with fault tolerance and bulk API support
311
- */
312
- const transferDocumentBatchWithRetry = async (db, client, dbId, collectionId, documents, batchSize = 10) => {
313
- // Check if we can use bulk operations
314
- if (supportsBulkOperations(client.config.endpoint)) {
315
- MessageFormatter.info(`Using bulk upsert API for faster document transfer`, { prefix: "Transfer" });
316
- // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
317
- const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
318
- for (const maxBatchSize of batchSizes) {
319
- try {
320
- return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
321
- }
322
- catch (error) {
323
- MessageFormatter.progress(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`, { prefix: "Transfer" });
324
- continue;
325
- }
326
- }
327
- // If all bulk operations fail, fall back to individual transfers
328
- MessageFormatter.progress(`All bulk operations failed, falling back to individual document transfers`, { prefix: "Transfer" });
329
- }
330
- // Fall back to individual document transfer
331
- return await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, documents, batchSize);
332
- };
333
- export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId, fromCollId, toCollId) => {
334
- MessageFormatter.info(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`, { prefix: "Transfer" });
335
- // Prefer adapter for remote to enable bulk operations
336
- const { adapter: remoteAdapter, client } = await getAdapter(endpoint, projectId, apiKey, 'auto');
337
- const remoteDb = new Databases(client); // Legacy fallback for HTTP/individual
338
- let totalDocumentsProcessed = 0;
339
- let totalSuccessful = 0;
340
- let totalFailed = 0;
341
- // Fetch documents in larger batches (1000 at a time)
342
- let hasMoreDocuments = true;
343
- let lastDocumentId;
344
- while (hasMoreDocuments) {
345
- const queries = [Query.limit(1000)]; // Fetch 1000 documents at a time
346
- if (lastDocumentId) {
347
- queries.push(Query.cursorAfter(lastDocumentId));
348
- }
349
- const fromCollDocs = await tryAwaitWithRetry(async () => {
350
- if (isLegacyDatabases(localDb)) {
351
- return localDb.listDocuments(fromDbId, fromCollId, queries);
352
- }
353
- else {
354
- const res = await localDb.listRows({ databaseId: fromDbId, tableId: fromCollId, queries });
355
- const rows = res.rows || res.documents || [];
356
- return { documents: rows };
357
- }
358
- });
359
- if (fromCollDocs.documents.length === 0) {
360
- hasMoreDocuments = false;
361
- break;
362
- }
363
- MessageFormatter.progress(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`, { prefix: "Transfer" });
364
- // Prefer remote adapter bulk upsert if available
365
- const prepared = fromCollDocs.documents.map((doc) => {
366
- const data = { ...doc };
367
- delete data.$databaseId;
368
- delete data.$collectionId;
369
- delete data.$createdAt;
370
- delete data.$updatedAt;
371
- return data; // Keep $id and $permissions for upsert
372
- });
373
- let successful = 0;
374
- let failed = 0;
375
- if (typeof remoteAdapter.bulkUpsertRows === 'function' && remoteAdapter.supportsBulkOperations()) {
376
- try {
377
- await remoteAdapter.bulkUpsertRows({ databaseId: toDbId, tableId: toCollId, rows: prepared });
378
- successful = prepared.length;
379
- }
380
- catch (e) {
381
- MessageFormatter.warning('Remote adapter bulk upsert failed, falling back to HTTP/individual', { prefix: 'Transfer' });
382
- }
383
- }
384
- if (successful === 0) {
385
- const res = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
386
- successful = res.successful;
387
- failed = res.failed;
388
- }
389
- totalDocumentsProcessed += fromCollDocs.documents.length;
390
- totalSuccessful += successful;
391
- totalFailed += failed;
392
- // Check if we have more documents to process
393
- if (fromCollDocs.documents.length < 1000) {
394
- hasMoreDocuments = false;
395
- }
396
- else {
397
- lastDocumentId = fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
398
- }
399
- MessageFormatter.debug(`Batch complete: ${successful} successful, ${failed} failed`, undefined, { prefix: "Transfer" });
400
- }
401
- if (totalDocumentsProcessed === 0) {
402
- MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
403
- return;
404
- }
405
- const message = `Total documents processed: ${totalDocumentsProcessed}, successful: ${totalSuccessful}, failed: ${totalFailed}`;
406
- if (totalFailed > 0) {
407
- MessageFormatter.warning(message, { prefix: "Transfer" });
408
- }
409
- else {
410
- MessageFormatter.success(message, { prefix: "Transfer" });
411
- }
412
- };
@@ -1,16 +0,0 @@
1
- import { Databases } from "node-appwrite";
2
- import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
3
- export declare const wipeDatabase: (database: Databases, databaseId: string) => Promise<{
4
- collectionId: string;
5
- collectionName: string;
6
- }[]>;
7
- export declare const wipeCollection: (database: Databases, databaseId: string, collectionId: string) => Promise<void>;
8
- export declare const wipeAllTables: (adapter: DatabaseAdapter, databaseId: string) => Promise<{
9
- tableId: string;
10
- tableName: string;
11
- }[]>;
12
- /**
13
- * Optimized deletion of all rows from a table using direct bulk deletion
14
- * Uses Query.limit() to delete rows without fetching IDs first
15
- */
16
- export declare const wipeTableRows: (adapter: DatabaseAdapter, databaseId: string, tableId: string) => Promise<void>;
@@ -1,233 +0,0 @@
1
- import { Databases, Query, } from "node-appwrite";
2
- import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
3
- import { MessageFormatter } from "../shared/messageFormatter.js";
4
- import { ProgressManager } from "../shared/progressManager.js";
5
- import { isRetryableError, isCriticalError } from "../shared/errorUtils.js";
6
- import { delay } from "../utils/helperFunctions.js";
7
- import { chunk } from "es-toolkit";
8
- import pLimit from "p-limit";
9
- import { fetchAllCollections } from "./methods.js";
10
- /**
11
- * Optimized streaming deletion of all documents from a collection
12
- * Uses memory-efficient pagination instead of loading all documents into memory
13
- */
14
- async function wipeDocumentsFromCollection(database, databaseId, collectionId) {
15
- try {
16
- // Use streaming deletion pattern - fetch and delete in batches without accumulating
17
- const FETCH_BATCH_SIZE = 1000; // How many to fetch per query
18
- const DELETE_BATCH_SIZE = 200; // How many to delete concurrently
19
- const MAX_CONCURRENT_DELETIONS = 10; // Concurrent deletion operations
20
- let totalDeleted = 0;
21
- let cursor;
22
- let hasMoreDocuments = true;
23
- MessageFormatter.info("Starting optimized document deletion...", { prefix: "Wipe" });
24
- // Create progress tracker (we'll update the total as we discover more documents)
25
- const progress = ProgressManager.create(`delete-${collectionId}`, 1, // Start with 1, will update as we go
26
- { title: "Deleting documents" });
27
- while (hasMoreDocuments) {
28
- // Fetch next batch of documents
29
- const queries = [Query.limit(FETCH_BATCH_SIZE)];
30
- if (cursor) {
31
- queries.push(Query.cursorAfter(cursor));
32
- }
33
- const response = await database.listDocuments(databaseId, collectionId, queries);
34
- const documents = response.documents;
35
- if (documents.length === 0) {
36
- hasMoreDocuments = false;
37
- break;
38
- }
39
- // Update progress total as we discover more documents
40
- if (documents.length === FETCH_BATCH_SIZE) {
41
- // There might be more documents, update progress total
42
- progress.setTotal(totalDeleted + documents.length + 1000); // Estimate more
43
- }
44
- MessageFormatter.progress(`Processing batch: ${documents.length} documents (${totalDeleted + documents.length} total so far)`, { prefix: "Wipe" });
45
- // Delete this batch using optimized concurrent deletion
46
- const documentBatches = chunk(documents, DELETE_BATCH_SIZE);
47
- const limit = pLimit(MAX_CONCURRENT_DELETIONS);
48
- const deletePromises = documentBatches.map((batch) => limit(async () => {
49
- const batchDeletePromises = batch.map(async (doc) => {
50
- try {
51
- await tryAwaitWithRetry(async () => database.deleteDocument(databaseId, collectionId, doc.$id));
52
- totalDeleted++;
53
- progress.update(totalDeleted);
54
- }
55
- catch (error) {
56
- const errorMessage = error.message || String(error);
57
- // Enhanced error handling for document deletion
58
- if (errorMessage.includes("Document with the requested ID could not be found")) {
59
- // Document already deleted, skip silently
60
- totalDeleted++;
61
- progress.update(totalDeleted);
62
- }
63
- else if (isCriticalError(errorMessage)) {
64
- // Critical error, log and rethrow to stop operation
65
- MessageFormatter.error(`Critical error deleting document ${doc.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
66
- throw error;
67
- }
68
- else if (isRetryableError(errorMessage)) {
69
- // Retryable error, will be handled by tryAwaitWithRetry
70
- MessageFormatter.progress(`Retryable error for document ${doc.$id}, will retry`, { prefix: "Wipe" });
71
- totalDeleted++;
72
- progress.update(totalDeleted);
73
- }
74
- else {
75
- // Other non-critical errors, log but continue
76
- MessageFormatter.error(`Failed to delete document ${doc.$id}: ${errorMessage}`, error, { prefix: "Wipe" });
77
- totalDeleted++;
78
- progress.update(totalDeleted);
79
- }
80
- }
81
- });
82
- await Promise.all(batchDeletePromises);
83
- }));
84
- await Promise.all(deletePromises);
85
- // Set up cursor for next iteration
86
- if (documents.length < FETCH_BATCH_SIZE) {
87
- hasMoreDocuments = false;
88
- }
89
- else {
90
- cursor = documents[documents.length - 1].$id;
91
- }
92
- // Small delay between fetch cycles to be respectful to the API
93
- await delay(10);
94
- }
95
- // Update final progress total
96
- progress.setTotal(totalDeleted);
97
- progress.stop();
98
- if (totalDeleted === 0) {
99
- MessageFormatter.info("No documents found to delete", { prefix: "Wipe" });
100
- }
101
- else {
102
- MessageFormatter.success(`Successfully deleted ${totalDeleted} documents from collection ${collectionId}`, { prefix: "Wipe" });
103
- }
104
- }
105
- catch (error) {
106
- MessageFormatter.error(`Error wiping documents from collection ${collectionId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Wipe" });
107
- throw error;
108
- }
109
- }
110
- export const wipeDatabase = async (database, databaseId) => {
111
- MessageFormatter.info(`Wiping database: ${databaseId}`, { prefix: "Wipe" });
112
- const existingCollections = await fetchAllCollections(databaseId, database);
113
- let collectionsDeleted = [];
114
- if (existingCollections.length === 0) {
115
- MessageFormatter.info("No collections to delete", { prefix: "Wipe" });
116
- return collectionsDeleted;
117
- }
118
- const progress = ProgressManager.create(`wipe-db-${databaseId}`, existingCollections.length, { title: "Deleting collections" });
119
- let processed = 0;
120
- for (const { $id: collectionId, name: name } of existingCollections) {
121
- MessageFormatter.progress(`Deleting collection: ${collectionId}`, { prefix: "Wipe" });
122
- collectionsDeleted.push({
123
- collectionId: collectionId,
124
- collectionName: name,
125
- });
126
- tryAwaitWithRetry(async () => await database.deleteCollection(databaseId, collectionId)); // Try to delete the collection and ignore errors if it doesn't exist or if it's already being deleted
127
- processed++;
128
- progress.update(processed);
129
- await delay(100);
130
- }
131
- progress.stop();
132
- MessageFormatter.success(`Deleted ${collectionsDeleted.length} collections from database`, { prefix: "Wipe" });
133
- return collectionsDeleted;
134
- };
135
- export const wipeCollection = async (database, databaseId, collectionId) => {
136
- const collections = await database.listCollections(databaseId, [
137
- Query.equal("$id", collectionId),
138
- ]);
139
- if (collections.total === 0) {
140
- MessageFormatter.warning(`Collection ${collectionId} not found`, { prefix: "Wipe" });
141
- return;
142
- }
143
- const collection = collections.collections[0];
144
- await wipeDocumentsFromCollection(database, databaseId, collection.$id);
145
- };
146
- // TablesDB helpers for wiping
147
- export const wipeAllTables = async (adapter, databaseId) => {
148
- MessageFormatter.info(`Wiping tables in database: ${databaseId}`, { prefix: 'Wipe' });
149
- const res = await adapter.listTables({ databaseId, queries: [Query.limit(500)] });
150
- const tables = res.tables || [];
151
- const deleted = [];
152
- const progress = ProgressManager.create(`wipe-db-${databaseId}`, tables.length, { title: 'Deleting tables' });
153
- let processed = 0;
154
- for (const t of tables) {
155
- try {
156
- await adapter.deleteTable({ databaseId, tableId: t.$id });
157
- deleted.push({ tableId: t.$id, tableName: t.name });
158
- }
159
- catch (e) {
160
- MessageFormatter.error(`Failed deleting table ${t.$id}`, e instanceof Error ? e : new Error(String(e)), { prefix: 'Wipe' });
161
- }
162
- processed++;
163
- progress.update(processed);
164
- await delay(100);
165
- }
166
- progress.stop();
167
- return deleted;
168
- };
169
- /**
170
- * Optimized deletion of all rows from a table using direct bulk deletion
171
- * Uses Query.limit() to delete rows without fetching IDs first
172
- */
173
- export const wipeTableRows = async (adapter, databaseId, tableId) => {
174
- try {
175
- // Check if bulk deletion is available
176
- if (!adapter.bulkDeleteRows) {
177
- MessageFormatter.error("Bulk deletion not available for this adapter - wipe operation not supported", new Error("bulkDeleteRows not available"), { prefix: "Wipe" });
178
- throw new Error("Bulk deletion required for wipe operations");
179
- }
180
- const DELETE_BATCH_SIZE = 250; // How many rows to delete per batch
181
- let totalDeleted = 0;
182
- let hasMoreRows = true;
183
- MessageFormatter.info("Starting optimized table row deletion...", { prefix: "Wipe" });
184
- const progress = ProgressManager.create(`delete-${tableId}`, 1, // Start with 1, will update as we discover more
185
- { title: "Deleting table rows" });
186
- while (hasMoreRows) {
187
- try {
188
- // Delete next batch using Query.limit() - no fetching needed!
189
- const result = await tryAwaitWithRetry(async () => adapter.bulkDeleteRows({
190
- databaseId,
191
- tableId,
192
- rowIds: [], // Empty array signals we want to use Query.limit instead
193
- batchSize: DELETE_BATCH_SIZE
194
- }));
195
- const deletedCount = result.total || 0;
196
- if (deletedCount === 0) {
197
- hasMoreRows = false;
198
- break;
199
- }
200
- totalDeleted += deletedCount;
201
- progress.setTotal(totalDeleted + 100); // Estimate more rows exist
202
- progress.update(totalDeleted);
203
- MessageFormatter.progress(`Deleted ${deletedCount} rows (${totalDeleted} total so far)`, { prefix: "Wipe" });
204
- // Small delay between batches to be respectful to the API
205
- await delay(10);
206
- }
207
- catch (error) {
208
- const errorMessage = error.message || String(error);
209
- if (isCriticalError(errorMessage)) {
210
- MessageFormatter.error(`Critical error during bulk deletion: ${errorMessage}`, error, { prefix: "Wipe" });
211
- throw error;
212
- }
213
- else {
214
- MessageFormatter.error(`Error during deletion batch: ${errorMessage}`, error, { prefix: "Wipe" });
215
- // Continue trying with next batch
216
- }
217
- }
218
- }
219
- // Update final progress total
220
- progress.setTotal(totalDeleted);
221
- progress.stop();
222
- if (totalDeleted === 0) {
223
- MessageFormatter.info("No rows found to delete", { prefix: "Wipe" });
224
- }
225
- else {
226
- MessageFormatter.success(`Successfully deleted ${totalDeleted} rows from table ${tableId}`, { prefix: "Wipe" });
227
- }
228
- }
229
- catch (error) {
230
- MessageFormatter.error(`Error wiping rows from table ${tableId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Wipe" });
231
- throw error;
232
- }
233
- };