appwrite-utils-cli 1.9.6 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/CONFIG_TODO.md +1189 -1189
  2. package/README.md +1004 -1004
  3. package/SELECTION_DIALOGS.md +145 -145
  4. package/SERVICE_IMPLEMENTATION_REPORT.md +462 -462
  5. package/package.json +6 -3
  6. package/scripts/copy-templates.ts +23 -23
  7. package/src/adapters/index.ts +11 -37
  8. package/src/backups/operations/bucketBackup.ts +277 -277
  9. package/src/backups/operations/collectionBackup.ts +310 -310
  10. package/src/backups/operations/comprehensiveBackup.ts +342 -342
  11. package/src/backups/schemas/bucketManifest.ts +78 -78
  12. package/src/backups/schemas/comprehensiveManifest.ts +76 -76
  13. package/src/backups/tracking/centralizedTracking.ts +352 -352
  14. package/src/cli/commands/configCommands.ts +265 -201
  15. package/src/cli/commands/databaseCommands.ts +931 -879
  16. package/src/cli/commands/functionCommands.ts +333 -332
  17. package/src/cli/commands/importFileCommands.ts +815 -0
  18. package/src/cli/commands/schemaCommands.ts +141 -141
  19. package/src/cli/commands/storageCommands.ts +2 -3
  20. package/src/cli/commands/transferCommands.ts +454 -457
  21. package/src/collections/attributes.ts.backup +1555 -1555
  22. package/src/collections/{attributes.ts → columns.ts} +15 -44
  23. package/src/collections/indexes.ts +350 -352
  24. package/src/collections/methods.ts +714 -815
  25. package/src/collections/tableOperations.ts +57 -21
  26. package/src/collections/transferOperations.ts +376 -377
  27. package/src/collections/wipeOperations.ts +449 -346
  28. package/src/databases/methods.ts +49 -49
  29. package/src/databases/setup.ts +77 -77
  30. package/src/examples/yamlTerminologyExample.ts +346 -346
  31. package/src/functions/deployments.ts +221 -220
  32. package/src/functions/fnConfigDiscovery.ts +2 -2
  33. package/src/functions/methods.ts +284 -284
  34. package/src/functions/templates/count-docs-in-collection/README.md +53 -53
  35. package/src/functions/templates/count-docs-in-collection/src/main.ts +159 -159
  36. package/src/functions/templates/count-docs-in-collection/src/request.ts +8 -8
  37. package/src/functions/templates/hono-typescript/README.md +285 -285
  38. package/src/functions/templates/hono-typescript/src/adapters/request.ts +73 -73
  39. package/src/functions/templates/hono-typescript/src/adapters/response.ts +105 -105
  40. package/src/functions/templates/hono-typescript/src/app.ts +179 -179
  41. package/src/functions/templates/hono-typescript/src/context.ts +102 -102
  42. package/src/functions/templates/hono-typescript/src/{index.ts → main.ts} +53 -53
  43. package/src/functions/templates/hono-typescript/src/middleware/appwrite.ts +118 -118
  44. package/src/functions/templates/typescript-node/README.md +31 -31
  45. package/src/functions/templates/typescript-node/src/context.ts +102 -102
  46. package/src/functions/templates/typescript-node/src/{index.ts → main.ts} +29 -29
  47. package/src/functions/templates/uv/README.md +30 -30
  48. package/src/functions/templates/uv/pyproject.toml +29 -29
  49. package/src/functions/templates/uv/src/context.py +124 -124
  50. package/src/functions/templates/uv/src/{index.py → main.py} +45 -45
  51. package/src/init.ts +62 -62
  52. package/src/interactiveCLI.ts +1095 -1030
  53. package/src/main.ts +1517 -1670
  54. package/src/migrations/afterImportActions.ts +579 -580
  55. package/src/migrations/appwriteToX.ts +634 -630
  56. package/src/migrations/comprehensiveTransfer.ts +2149 -2149
  57. package/src/migrations/dataLoader.ts +1729 -1702
  58. package/src/migrations/importController.ts +440 -428
  59. package/src/migrations/importDataActions.ts +315 -315
  60. package/src/migrations/relationships.ts +333 -334
  61. package/src/migrations/services/DataTransformationService.ts +195 -195
  62. package/src/migrations/services/FileHandlerService.ts +310 -310
  63. package/src/migrations/services/ImportOrchestrator.ts +674 -665
  64. package/src/migrations/services/RateLimitManager.ts +362 -362
  65. package/src/migrations/services/RelationshipResolver.ts +460 -460
  66. package/src/migrations/services/UserMappingService.ts +344 -344
  67. package/src/migrations/services/ValidationService.ts +333 -333
  68. package/src/migrations/transfer.ts +987 -942
  69. package/src/migrations/yaml/YamlImportConfigLoader.ts +438 -438
  70. package/src/migrations/yaml/YamlImportIntegration.ts +438 -438
  71. package/src/migrations/yaml/generateImportSchemas.ts +1347 -1347
  72. package/src/schemas/authUser.ts +23 -23
  73. package/src/setup.ts +8 -8
  74. package/src/setupCommands.ts +5 -6
  75. package/src/setupController.ts +42 -42
  76. package/src/shared/backupMetadataSchema.ts +93 -93
  77. package/src/shared/backupTracking.ts +211 -211
  78. package/src/shared/confirmationDialogs.ts +326 -326
  79. package/src/shared/migrationHelpers.ts +232 -232
  80. package/src/shared/operationLogger.ts +20 -20
  81. package/src/shared/operationQueue.ts +326 -327
  82. package/src/shared/operationsTable.ts +338 -338
  83. package/src/shared/operationsTableSchema.ts +60 -60
  84. package/src/shared/progressManager.ts +277 -277
  85. package/src/shared/relationshipExtractor.ts +214 -214
  86. package/src/shared/selectionDialogs.ts +775 -722
  87. package/src/storage/backupCompression.ts +88 -88
  88. package/src/storage/methods.ts +695 -682
  89. package/src/storage/schemas.ts +205 -205
  90. package/src/tables/indexManager.ts +409 -0
  91. package/src/types/node-appwrite-tablesdb.d.ts +43 -43
  92. package/src/types.ts +9 -9
  93. package/src/users/methods.ts +358 -359
  94. package/src/utils/configMigration.ts +347 -347
  95. package/src/utils/index.ts +2 -2
  96. package/src/utils/loadConfigs.ts +457 -449
  97. package/src/utils/setupFiles.ts +1236 -1238
  98. package/src/utilsController.ts +1263 -1213
  99. package/tests/README.md +496 -496
  100. package/tests/adapters/AdapterFactory.test.ts +276 -276
  101. package/tests/integration/syncOperations.test.ts +462 -462
  102. package/tests/jest.config.js +24 -24
  103. package/tests/migration/configMigration.test.ts +545 -545
  104. package/tests/setup.ts +61 -61
  105. package/tests/testUtils.ts +339 -339
  106. package/tests/utils/loadConfigs.test.ts +349 -349
  107. package/tests/validation/configValidation.test.ts +411 -411
  108. package/tsconfig.json +44 -44
  109. package/.appwrite/.yaml_schemas/appwrite-config.schema.json +0 -380
  110. package/.appwrite/.yaml_schemas/collection.schema.json +0 -255
  111. package/.appwrite/collections/Categories.yaml +0 -182
  112. package/.appwrite/collections/ExampleCollection.yaml +0 -36
  113. package/.appwrite/collections/Posts.yaml +0 -227
  114. package/.appwrite/collections/Users.yaml +0 -149
  115. package/.appwrite/config.yaml +0 -109
  116. package/.appwrite/import/README.md +0 -148
  117. package/.appwrite/import/categories-import.yaml +0 -129
  118. package/.appwrite/import/posts-import.yaml +0 -208
  119. package/.appwrite/import/users-import.yaml +0 -130
  120. package/.appwrite/importData/categories.json +0 -194
  121. package/.appwrite/importData/posts.json +0 -270
  122. package/.appwrite/importData/users.json +0 -220
  123. package/.appwrite/schemas/categories.json +0 -128
  124. package/.appwrite/schemas/exampleCollection.json +0 -52
  125. package/.appwrite/schemas/posts.json +0 -173
  126. package/.appwrite/schemas/users.json +0 -125
  127. package/dist/adapters/AdapterFactory.d.ts +0 -94
  128. package/dist/adapters/AdapterFactory.js +0 -405
  129. package/dist/adapters/DatabaseAdapter.d.ts +0 -242
  130. package/dist/adapters/DatabaseAdapter.js +0 -50
  131. package/dist/adapters/LegacyAdapter.d.ts +0 -50
  132. package/dist/adapters/LegacyAdapter.js +0 -612
  133. package/dist/adapters/TablesDBAdapter.d.ts +0 -45
  134. package/dist/adapters/TablesDBAdapter.js +0 -596
  135. package/dist/adapters/index.d.ts +0 -11
  136. package/dist/adapters/index.js +0 -12
  137. package/dist/backups/operations/bucketBackup.d.ts +0 -19
  138. package/dist/backups/operations/bucketBackup.js +0 -197
  139. package/dist/backups/operations/collectionBackup.d.ts +0 -30
  140. package/dist/backups/operations/collectionBackup.js +0 -201
  141. package/dist/backups/operations/comprehensiveBackup.d.ts +0 -25
  142. package/dist/backups/operations/comprehensiveBackup.js +0 -238
  143. package/dist/backups/schemas/bucketManifest.d.ts +0 -93
  144. package/dist/backups/schemas/bucketManifest.js +0 -33
  145. package/dist/backups/schemas/comprehensiveManifest.d.ts +0 -108
  146. package/dist/backups/schemas/comprehensiveManifest.js +0 -32
  147. package/dist/backups/tracking/centralizedTracking.d.ts +0 -34
  148. package/dist/backups/tracking/centralizedTracking.js +0 -274
  149. package/dist/cli/commands/configCommands.d.ts +0 -8
  150. package/dist/cli/commands/configCommands.js +0 -166
  151. package/dist/cli/commands/databaseCommands.d.ts +0 -14
  152. package/dist/cli/commands/databaseCommands.js +0 -644
  153. package/dist/cli/commands/functionCommands.d.ts +0 -7
  154. package/dist/cli/commands/functionCommands.js +0 -330
  155. package/dist/cli/commands/schemaCommands.d.ts +0 -7
  156. package/dist/cli/commands/schemaCommands.js +0 -169
  157. package/dist/cli/commands/storageCommands.d.ts +0 -5
  158. package/dist/cli/commands/storageCommands.js +0 -143
  159. package/dist/cli/commands/transferCommands.d.ts +0 -5
  160. package/dist/cli/commands/transferCommands.js +0 -384
  161. package/dist/collections/attributes.d.ts +0 -13
  162. package/dist/collections/attributes.js +0 -1364
  163. package/dist/collections/indexes.d.ts +0 -12
  164. package/dist/collections/indexes.js +0 -217
  165. package/dist/collections/methods.d.ts +0 -19
  166. package/dist/collections/methods.js +0 -734
  167. package/dist/collections/tableOperations.d.ts +0 -86
  168. package/dist/collections/tableOperations.js +0 -434
  169. package/dist/collections/transferOperations.d.ts +0 -8
  170. package/dist/collections/transferOperations.js +0 -412
  171. package/dist/collections/wipeOperations.d.ts +0 -16
  172. package/dist/collections/wipeOperations.js +0 -233
  173. package/dist/config/ConfigManager.d.ts +0 -450
  174. package/dist/config/ConfigManager.js +0 -625
  175. package/dist/config/configMigration.d.ts +0 -87
  176. package/dist/config/configMigration.js +0 -390
  177. package/dist/config/configValidation.d.ts +0 -66
  178. package/dist/config/configValidation.js +0 -358
  179. package/dist/config/index.d.ts +0 -8
  180. package/dist/config/index.js +0 -7
  181. package/dist/config/services/ConfigDiscoveryService.d.ts +0 -122
  182. package/dist/config/services/ConfigDiscoveryService.js +0 -322
  183. package/dist/config/services/ConfigLoaderService.d.ts +0 -129
  184. package/dist/config/services/ConfigLoaderService.js +0 -535
  185. package/dist/config/services/ConfigMergeService.d.ts +0 -208
  186. package/dist/config/services/ConfigMergeService.js +0 -308
  187. package/dist/config/services/ConfigValidationService.d.ts +0 -214
  188. package/dist/config/services/ConfigValidationService.js +0 -310
  189. package/dist/config/services/SessionAuthService.d.ts +0 -225
  190. package/dist/config/services/SessionAuthService.js +0 -456
  191. package/dist/config/services/__tests__/ConfigMergeService.test.d.ts +0 -1
  192. package/dist/config/services/__tests__/ConfigMergeService.test.js +0 -271
  193. package/dist/config/services/index.d.ts +0 -13
  194. package/dist/config/services/index.js +0 -10
  195. package/dist/config/yamlConfig.d.ts +0 -722
  196. package/dist/config/yamlConfig.js +0 -702
  197. package/dist/databases/methods.d.ts +0 -6
  198. package/dist/databases/methods.js +0 -35
  199. package/dist/databases/setup.d.ts +0 -5
  200. package/dist/databases/setup.js +0 -45
  201. package/dist/examples/yamlTerminologyExample.d.ts +0 -42
  202. package/dist/examples/yamlTerminologyExample.js +0 -272
  203. package/dist/functions/deployments.d.ts +0 -4
  204. package/dist/functions/deployments.js +0 -146
  205. package/dist/functions/fnConfigDiscovery.d.ts +0 -3
  206. package/dist/functions/fnConfigDiscovery.js +0 -108
  207. package/dist/functions/methods.d.ts +0 -16
  208. package/dist/functions/methods.js +0 -174
  209. package/dist/functions/pathResolution.d.ts +0 -37
  210. package/dist/functions/pathResolution.js +0 -185
  211. package/dist/functions/templates/count-docs-in-collection/README.md +0 -54
  212. package/dist/functions/templates/count-docs-in-collection/package.json +0 -25
  213. package/dist/functions/templates/count-docs-in-collection/src/main.ts +0 -159
  214. package/dist/functions/templates/count-docs-in-collection/src/request.ts +0 -9
  215. package/dist/functions/templates/count-docs-in-collection/tsconfig.json +0 -28
  216. package/dist/functions/templates/hono-typescript/README.md +0 -286
  217. package/dist/functions/templates/hono-typescript/package.json +0 -26
  218. package/dist/functions/templates/hono-typescript/src/adapters/request.ts +0 -74
  219. package/dist/functions/templates/hono-typescript/src/adapters/response.ts +0 -106
  220. package/dist/functions/templates/hono-typescript/src/app.ts +0 -180
  221. package/dist/functions/templates/hono-typescript/src/context.ts +0 -103
  222. package/dist/functions/templates/hono-typescript/src/index.ts +0 -54
  223. package/dist/functions/templates/hono-typescript/src/middleware/appwrite.ts +0 -119
  224. package/dist/functions/templates/hono-typescript/tsconfig.json +0 -20
  225. package/dist/functions/templates/typescript-node/README.md +0 -32
  226. package/dist/functions/templates/typescript-node/package.json +0 -25
  227. package/dist/functions/templates/typescript-node/src/context.ts +0 -103
  228. package/dist/functions/templates/typescript-node/src/index.ts +0 -29
  229. package/dist/functions/templates/typescript-node/tsconfig.json +0 -28
  230. package/dist/functions/templates/uv/README.md +0 -31
  231. package/dist/functions/templates/uv/pyproject.toml +0 -30
  232. package/dist/functions/templates/uv/src/__init__.py +0 -0
  233. package/dist/functions/templates/uv/src/context.py +0 -125
  234. package/dist/functions/templates/uv/src/index.py +0 -46
  235. package/dist/init.d.ts +0 -2
  236. package/dist/init.js +0 -57
  237. package/dist/interactiveCLI.d.ts +0 -31
  238. package/dist/interactiveCLI.js +0 -898
  239. package/dist/main.d.ts +0 -2
  240. package/dist/main.js +0 -1180
  241. package/dist/migrations/afterImportActions.d.ts +0 -17
  242. package/dist/migrations/afterImportActions.js +0 -306
  243. package/dist/migrations/appwriteToX.d.ts +0 -211
  244. package/dist/migrations/appwriteToX.js +0 -491
  245. package/dist/migrations/comprehensiveTransfer.d.ts +0 -147
  246. package/dist/migrations/comprehensiveTransfer.js +0 -1317
  247. package/dist/migrations/dataLoader.d.ts +0 -753
  248. package/dist/migrations/dataLoader.js +0 -1250
  249. package/dist/migrations/importController.d.ts +0 -23
  250. package/dist/migrations/importController.js +0 -268
  251. package/dist/migrations/importDataActions.d.ts +0 -50
  252. package/dist/migrations/importDataActions.js +0 -230
  253. package/dist/migrations/relationships.d.ts +0 -29
  254. package/dist/migrations/relationships.js +0 -204
  255. package/dist/migrations/services/DataTransformationService.d.ts +0 -55
  256. package/dist/migrations/services/DataTransformationService.js +0 -158
  257. package/dist/migrations/services/FileHandlerService.d.ts +0 -75
  258. package/dist/migrations/services/FileHandlerService.js +0 -236
  259. package/dist/migrations/services/ImportOrchestrator.d.ts +0 -97
  260. package/dist/migrations/services/ImportOrchestrator.js +0 -485
  261. package/dist/migrations/services/RateLimitManager.d.ts +0 -138
  262. package/dist/migrations/services/RateLimitManager.js +0 -279
  263. package/dist/migrations/services/RelationshipResolver.d.ts +0 -120
  264. package/dist/migrations/services/RelationshipResolver.js +0 -332
  265. package/dist/migrations/services/UserMappingService.d.ts +0 -109
  266. package/dist/migrations/services/UserMappingService.js +0 -277
  267. package/dist/migrations/services/ValidationService.d.ts +0 -74
  268. package/dist/migrations/services/ValidationService.js +0 -260
  269. package/dist/migrations/transfer.d.ts +0 -26
  270. package/dist/migrations/transfer.js +0 -608
  271. package/dist/migrations/yaml/YamlImportConfigLoader.d.ts +0 -131
  272. package/dist/migrations/yaml/YamlImportConfigLoader.js +0 -383
  273. package/dist/migrations/yaml/YamlImportIntegration.d.ts +0 -93
  274. package/dist/migrations/yaml/YamlImportIntegration.js +0 -341
  275. package/dist/migrations/yaml/generateImportSchemas.d.ts +0 -30
  276. package/dist/migrations/yaml/generateImportSchemas.js +0 -1327
  277. package/dist/schemas/authUser.d.ts +0 -24
  278. package/dist/schemas/authUser.js +0 -17
  279. package/dist/setup.d.ts +0 -2
  280. package/dist/setup.js +0 -5
  281. package/dist/setupCommands.d.ts +0 -58
  282. package/dist/setupCommands.js +0 -490
  283. package/dist/setupController.d.ts +0 -9
  284. package/dist/setupController.js +0 -34
  285. package/dist/shared/attributeMapper.d.ts +0 -20
  286. package/dist/shared/attributeMapper.js +0 -203
  287. package/dist/shared/backupMetadataSchema.d.ts +0 -94
  288. package/dist/shared/backupMetadataSchema.js +0 -38
  289. package/dist/shared/backupTracking.d.ts +0 -18
  290. package/dist/shared/backupTracking.js +0 -176
  291. package/dist/shared/confirmationDialogs.d.ts +0 -75
  292. package/dist/shared/confirmationDialogs.js +0 -236
  293. package/dist/shared/errorUtils.d.ts +0 -54
  294. package/dist/shared/errorUtils.js +0 -95
  295. package/dist/shared/functionManager.d.ts +0 -48
  296. package/dist/shared/functionManager.js +0 -348
  297. package/dist/shared/indexManager.d.ts +0 -24
  298. package/dist/shared/indexManager.js +0 -151
  299. package/dist/shared/jsonSchemaGenerator.d.ts +0 -50
  300. package/dist/shared/jsonSchemaGenerator.js +0 -290
  301. package/dist/shared/logging.d.ts +0 -61
  302. package/dist/shared/logging.js +0 -116
  303. package/dist/shared/messageFormatter.d.ts +0 -39
  304. package/dist/shared/messageFormatter.js +0 -162
  305. package/dist/shared/migrationHelpers.d.ts +0 -61
  306. package/dist/shared/migrationHelpers.js +0 -145
  307. package/dist/shared/operationLogger.d.ts +0 -10
  308. package/dist/shared/operationLogger.js +0 -12
  309. package/dist/shared/operationQueue.d.ts +0 -40
  310. package/dist/shared/operationQueue.js +0 -311
  311. package/dist/shared/operationsTable.d.ts +0 -26
  312. package/dist/shared/operationsTable.js +0 -286
  313. package/dist/shared/operationsTableSchema.d.ts +0 -48
  314. package/dist/shared/operationsTableSchema.js +0 -35
  315. package/dist/shared/progressManager.d.ts +0 -62
  316. package/dist/shared/progressManager.js +0 -215
  317. package/dist/shared/pydanticModelGenerator.d.ts +0 -17
  318. package/dist/shared/pydanticModelGenerator.js +0 -615
  319. package/dist/shared/relationshipExtractor.d.ts +0 -56
  320. package/dist/shared/relationshipExtractor.js +0 -138
  321. package/dist/shared/schemaGenerator.d.ts +0 -40
  322. package/dist/shared/schemaGenerator.js +0 -556
  323. package/dist/shared/selectionDialogs.d.ts +0 -214
  324. package/dist/shared/selectionDialogs.js +0 -544
  325. package/dist/storage/backupCompression.d.ts +0 -20
  326. package/dist/storage/backupCompression.js +0 -67
  327. package/dist/storage/methods.d.ts +0 -32
  328. package/dist/storage/methods.js +0 -472
  329. package/dist/storage/schemas.d.ts +0 -842
  330. package/dist/storage/schemas.js +0 -175
  331. package/dist/types.d.ts +0 -4
  332. package/dist/types.js +0 -3
  333. package/dist/users/methods.d.ts +0 -16
  334. package/dist/users/methods.js +0 -277
  335. package/dist/utils/ClientFactory.d.ts +0 -87
  336. package/dist/utils/ClientFactory.js +0 -212
  337. package/dist/utils/configDiscovery.d.ts +0 -78
  338. package/dist/utils/configDiscovery.js +0 -472
  339. package/dist/utils/configMigration.d.ts +0 -1
  340. package/dist/utils/configMigration.js +0 -261
  341. package/dist/utils/constantsGenerator.d.ts +0 -31
  342. package/dist/utils/constantsGenerator.js +0 -321
  343. package/dist/utils/dataConverters.d.ts +0 -46
  344. package/dist/utils/dataConverters.js +0 -139
  345. package/dist/utils/directoryUtils.d.ts +0 -22
  346. package/dist/utils/directoryUtils.js +0 -59
  347. package/dist/utils/getClientFromConfig.d.ts +0 -39
  348. package/dist/utils/getClientFromConfig.js +0 -199
  349. package/dist/utils/helperFunctions.d.ts +0 -63
  350. package/dist/utils/helperFunctions.js +0 -156
  351. package/dist/utils/index.d.ts +0 -2
  352. package/dist/utils/index.js +0 -2
  353. package/dist/utils/loadConfigs.d.ts +0 -50
  354. package/dist/utils/loadConfigs.js +0 -358
  355. package/dist/utils/pathResolvers.d.ts +0 -53
  356. package/dist/utils/pathResolvers.js +0 -72
  357. package/dist/utils/projectConfig.d.ts +0 -122
  358. package/dist/utils/projectConfig.js +0 -206
  359. package/dist/utils/retryFailedPromises.d.ts +0 -2
  360. package/dist/utils/retryFailedPromises.js +0 -23
  361. package/dist/utils/sessionAuth.d.ts +0 -48
  362. package/dist/utils/sessionAuth.js +0 -164
  363. package/dist/utils/setupFiles.d.ts +0 -4
  364. package/dist/utils/setupFiles.js +0 -1192
  365. package/dist/utils/typeGuards.d.ts +0 -35
  366. package/dist/utils/typeGuards.js +0 -57
  367. package/dist/utils/validationRules.d.ts +0 -43
  368. package/dist/utils/validationRules.js +0 -42
  369. package/dist/utils/versionDetection.d.ts +0 -58
  370. package/dist/utils/versionDetection.js +0 -251
  371. package/dist/utils/yamlConverter.d.ts +0 -100
  372. package/dist/utils/yamlConverter.js +0 -428
  373. package/dist/utils/yamlLoader.d.ts +0 -70
  374. package/dist/utils/yamlLoader.js +0 -267
  375. package/dist/utilsController.d.ts +0 -107
  376. package/dist/utilsController.js +0 -873
  377. package/src/adapters/AdapterFactory.ts +0 -510
  378. package/src/adapters/DatabaseAdapter.ts +0 -318
  379. package/src/adapters/LegacyAdapter.ts +0 -841
  380. package/src/adapters/TablesDBAdapter.ts +0 -815
  381. package/src/config/ConfigManager.ts +0 -817
  382. package/src/config/README.md +0 -274
  383. package/src/config/configMigration.ts +0 -575
  384. package/src/config/configValidation.ts +0 -445
  385. package/src/config/index.ts +0 -10
  386. package/src/config/services/ConfigDiscoveryService.ts +0 -410
  387. package/src/config/services/ConfigLoaderService.ts +0 -732
  388. package/src/config/services/ConfigMergeService.ts +0 -388
  389. package/src/config/services/ConfigValidationService.ts +0 -394
  390. package/src/config/services/SessionAuthService.ts +0 -565
  391. package/src/config/services/__tests__/ConfigMergeService.test.ts +0 -351
  392. package/src/config/services/index.ts +0 -29
  393. package/src/config/yamlConfig.ts +0 -761
  394. package/src/functions/pathResolution.ts +0 -227
  395. package/src/functions/templates/count-docs-in-collection/package.json +0 -25
  396. package/src/functions/templates/count-docs-in-collection/tsconfig.json +0 -28
  397. package/src/functions/templates/hono-typescript/package.json +0 -26
  398. package/src/functions/templates/hono-typescript/tsconfig.json +0 -20
  399. package/src/functions/templates/typescript-node/package.json +0 -25
  400. package/src/functions/templates/typescript-node/tsconfig.json +0 -28
  401. package/src/shared/attributeMapper.ts +0 -229
  402. package/src/shared/errorUtils.ts +0 -110
  403. package/src/shared/functionManager.ts +0 -537
  404. package/src/shared/indexManager.ts +0 -254
  405. package/src/shared/jsonSchemaGenerator.ts +0 -383
  406. package/src/shared/logging.ts +0 -149
  407. package/src/shared/messageFormatter.ts +0 -208
  408. package/src/shared/pydanticModelGenerator.ts +0 -618
  409. package/src/shared/schemaGenerator.ts +0 -644
  410. package/src/utils/ClientFactory.ts +0 -240
  411. package/src/utils/configDiscovery.ts +0 -557
  412. package/src/utils/constantsGenerator.ts +0 -369
  413. package/src/utils/dataConverters.ts +0 -159
  414. package/src/utils/directoryUtils.ts +0 -61
  415. package/src/utils/getClientFromConfig.ts +0 -257
  416. package/src/utils/helperFunctions.ts +0 -228
  417. package/src/utils/pathResolvers.ts +0 -81
  418. package/src/utils/projectConfig.ts +0 -340
  419. package/src/utils/retryFailedPromises.ts +0 -29
  420. package/src/utils/sessionAuth.ts +0 -230
  421. package/src/utils/typeGuards.ts +0 -65
  422. package/src/utils/validationRules.ts +0 -88
  423. package/src/utils/versionDetection.ts +0 -292
  424. package/src/utils/yamlConverter.ts +0 -542
  425. package/src/utils/yamlLoader.ts +0 -371
  426. package/tmp-sync-test/.appwrite/collections/TestCollection.yaml +0 -7
@@ -0,0 +1,815 @@
1
+ import inquirer from "inquirer";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import readline from "node:readline";
5
+ import Papa from "papaparse";
6
+ import { ID, Query } from "node-appwrite";
7
+ import pLimit from "p-limit";
8
+ import {
9
+ MessageFormatter,
10
+ tryAwaitWithRetry,
11
+ delay,
12
+ } from "appwrite-utils-helpers";
13
+ import type { DatabaseAdapter } from "appwrite-utils-helpers";
14
+ import { ProgressManager } from "../../shared/progressManager.js";
15
+ import { fetchAllDatabases } from "../../databases/methods.js";
16
+ import type { InteractiveCLI } from "../../interactiveCLI.js";
17
+
18
+ // ── Schema helpers ──────────────────────────────────────────────────────
19
+
20
+ interface SchemaInfo {
21
+ columnTypeMap: Map<string, string>;
22
+ arrayColumns: Set<string>;
23
+ enumElementsMap: Map<string, Set<string>>;
24
+ hasRelationships: boolean;
25
+ }
26
+
27
+ const SYSTEM_FIELDS = new Set([
28
+ "$databaseId",
29
+ "$collectionId",
30
+ "$tableId",
31
+ "$sequence",
32
+ "$tenant",
33
+ ]);
34
+ const PASSTHROUGH_FIELDS = new Set(["$id", "$permissions", "$createdAt", "$updatedAt"]);
35
+
36
+ async function fetchSchema(
37
+ adapter: DatabaseAdapter,
38
+ databaseId: string,
39
+ tableId: string
40
+ ): Promise<SchemaInfo> {
41
+ MessageFormatter.info(`Fetching schema for database="${databaseId}" table="${tableId}"`, { prefix: "Import" });
42
+ const tableInfo = await adapter.getTable({ databaseId, tableId });
43
+ const columns: any[] = (tableInfo.data as any)?.columns || (tableInfo.data as any)?.attributes || [];
44
+ const availableColumns = columns.filter((col: any) => !col.status || col.status === "available");
45
+ MessageFormatter.info(
46
+ `Schema columns: ${availableColumns.map((c: any) => c.key || c.$id).join(", ")}`,
47
+ { prefix: "Import" }
48
+ );
49
+
50
+ const columnTypeMap = new Map<string, string>();
51
+ const enumElementsMap = new Map<string, Set<string>>();
52
+ const arrayColumns = new Set<string>();
53
+ const hasRelationships = availableColumns.some((col: any) => col.type === "relationship");
54
+
55
+ for (const col of availableColumns) {
56
+ const key = col.key || col.$id;
57
+ if (!key) continue;
58
+ columnTypeMap.set(key, col.type);
59
+ if (col.array) arrayColumns.add(key);
60
+ if (col.type === "enum" && Array.isArray(col.elements)) {
61
+ enumElementsMap.set(key, new Set(col.elements as string[]));
62
+ }
63
+ }
64
+
65
+ if (columnTypeMap.size > 0) {
66
+ MessageFormatter.info(
67
+ `Table schema: ${columns.length} columns detected, will cast values to match types`,
68
+ { prefix: "Import" }
69
+ );
70
+ }
71
+
72
+ return { columnTypeMap, arrayColumns, enumElementsMap, hasRelationships };
73
+ }
74
+
75
+ // ── Row cleaning ────────────────────────────────────────────────────────
76
+
77
+ function cleanRow(row: any, schema: SchemaInfo): any {
78
+ const { columnTypeMap, arrayColumns } = schema;
79
+ const clean: any = {};
80
+
81
+ for (const [key, value] of Object.entries(row)) {
82
+ if (SYSTEM_FIELDS.has(key)) continue;
83
+
84
+ if (PASSTHROUGH_FIELDS.has(key)) {
85
+ if (value === null || value === undefined || value === "null" || value === "undefined" || value === "") continue;
86
+
87
+ if (key === "$permissions") {
88
+ if (Array.isArray(value)) {
89
+ clean[key] = value;
90
+ } else if (typeof value === "string" && value.trim()) {
91
+ clean[key] = value.match(/[a-z]+\([^)]*\)/g) || [];
92
+ }
93
+ } else if (key === "$createdAt" || key === "$updatedAt") {
94
+ const str = value instanceof Date ? value.toISOString() : String(value).trim();
95
+ // Only set if it looks like a valid ISO datetime
96
+ if (str && !isNaN(Date.parse(str))) {
97
+ clean[key] = str;
98
+ }
99
+ } else {
100
+ clean[key] = value;
101
+ }
102
+ continue;
103
+ }
104
+
105
+ if (columnTypeMap.size > 0 && !columnTypeMap.has(key)) continue;
106
+ if (value === null || value === undefined || value === "null") continue;
107
+
108
+ const colType = columnTypeMap.get(key);
109
+ if (colType === "relationship") continue;
110
+
111
+ // Array columns
112
+ if (arrayColumns.has(key)) {
113
+ let arr: any[];
114
+ if (Array.isArray(value)) {
115
+ arr = value;
116
+ } else if (typeof value === "string") {
117
+ const trimmed = value.trim();
118
+ if (!trimmed || trimmed === "[]") {
119
+ clean[key] = [];
120
+ continue;
121
+ }
122
+ if (trimmed.startsWith("[")) {
123
+ try {
124
+ arr = JSON.parse(trimmed);
125
+ } catch {
126
+ arr = trimmed.split(",").map((s) => s.trim()).filter(Boolean);
127
+ }
128
+ } else {
129
+ arr = trimmed.split(",").map((s) => s.trim()).filter(Boolean);
130
+ }
131
+ } else {
132
+ arr = [value];
133
+ }
134
+ // Cast elements to scalar type
135
+ if (colType === "integer") {
136
+ arr = arr.map((v) => typeof v === "number" ? Math.round(v) : parseInt(String(v), 10)).filter((v) => !isNaN(v));
137
+ } else if (colType === "float" || colType === "double") {
138
+ arr = arr.map((v) => typeof v === "number" ? v : parseFloat(String(v))).filter((v) => !isNaN(v));
139
+ } else if (colType === "boolean") {
140
+ arr = arr.map((v) => {
141
+ if (typeof v === "boolean") return v;
142
+ const s = String(v).toLowerCase().trim();
143
+ return s === "true" || s === "1" || s === "yes";
144
+ });
145
+ } else {
146
+ arr = arr.map((v) => String(v));
147
+ }
148
+ clean[key] = arr;
149
+ continue;
150
+ }
151
+
152
+ // Scalar casting
153
+ if (colType === "enum") {
154
+ clean[key] = String(value);
155
+ } else if (colType === "boolean") {
156
+ if (typeof value === "boolean") {
157
+ clean[key] = value;
158
+ } else if (typeof value === "string") {
159
+ const lower = value.toLowerCase().trim();
160
+ if (lower === "true" || lower === "1" || lower === "yes") {
161
+ clean[key] = true;
162
+ } else if (lower === "false" || lower === "0" || lower === "no" || lower === "") {
163
+ clean[key] = false;
164
+ } else {
165
+ clean[key] = Boolean(value);
166
+ }
167
+ } else if (typeof value === "number") {
168
+ clean[key] = value !== 0;
169
+ } else {
170
+ clean[key] = Boolean(value);
171
+ }
172
+ } else if (colType === "integer") {
173
+ if (typeof value === "number") {
174
+ clean[key] = Math.round(value);
175
+ } else if (typeof value === "string" && value.trim() !== "") {
176
+ const parsed = parseInt(value, 10);
177
+ if (!isNaN(parsed)) clean[key] = parsed;
178
+ }
179
+ } else if (colType === "float" || colType === "double") {
180
+ if (typeof value === "number") {
181
+ clean[key] = value;
182
+ } else if (typeof value === "string" && value.trim() !== "") {
183
+ const parsed = parseFloat(value);
184
+ if (!isNaN(parsed)) clean[key] = parsed;
185
+ }
186
+ } else if (typeof value === "string" && value === "") {
187
+ // Skip empty strings — let database use defaults
188
+ continue;
189
+ } else {
190
+ clean[key] = value;
191
+ }
192
+ }
193
+ return clean;
194
+ }
195
+
196
+ // ── Upsert helpers ──────────────────────────────────────────────────────
197
+
198
+ function makeUpsertRow(
199
+ adapter: DatabaseAdapter,
200
+ databaseId: string,
201
+ tableId: string
202
+ ) {
203
+ return async (row: any) => {
204
+ const { $id, $permissions, ...data } = row;
205
+ const rowId = $id || ID.unique();
206
+ const permissions = Array.isArray($permissions) ? $permissions : undefined;
207
+ try {
208
+ await adapter.createRow({ databaseId, tableId, id: rowId, data, permissions });
209
+ } catch (error: any) {
210
+ const code = error?.originalError?.code || error?.code;
211
+ if (code === 409) {
212
+ await adapter.updateRow({ databaseId, tableId, id: rowId, data, permissions });
213
+ } else {
214
+ throw error;
215
+ }
216
+ }
217
+ };
218
+ }
219
+
220
+ interface FlushStats {
221
+ created: number;
222
+ errors: number;
223
+ firstError: string | null;
224
+ }
225
+
226
+ // Track the max working bulk size across calls so we don't retry sizes that already failed
227
+ let effectiveBulkMax = 2500;
228
+
229
+ async function flushBatch(
230
+ batch: any[],
231
+ adapter: DatabaseAdapter,
232
+ databaseId: string,
233
+ tableId: string,
234
+ schema: SchemaInfo,
235
+ useBulk: boolean,
236
+ upsertRow: (row: any) => Promise<void>
237
+ ): Promise<FlushStats> {
238
+ let created = 0;
239
+ let errors = 0;
240
+ let firstError: string | null = null;
241
+
242
+ if (useBulk) {
243
+ // Deduplicate by $id within the batch — keep last occurrence
244
+ const deduped = new Map<string, any>();
245
+ for (const row of batch) {
246
+ const id = row.$id || ID.unique();
247
+ if (!row.$id) row.$id = id;
248
+ deduped.set(id, row);
249
+ }
250
+ const dedupedRows = Array.from(deduped.values());
251
+
252
+ // Split into sub-batches of effectiveBulkMax and run up to 5 concurrently
253
+ const subBatches: any[][] = [];
254
+ for (let i = 0; i < dedupedRows.length; i += effectiveBulkMax) {
255
+ subBatches.push(dedupedRows.slice(i, i + effectiveBulkMax));
256
+ }
257
+ const bulkLimit = pLimit(500);
258
+ const results = await Promise.all(
259
+ subBatches.map((sub) =>
260
+ bulkLimit(async () => {
261
+ try {
262
+ const result = await tryAwaitWithRetry(async () =>
263
+ adapter.bulkUpsertRows!({ databaseId, tableId, rows: sub })
264
+ );
265
+ return { created: (result as any)?.total || (result as any)?.rows?.length || sub.length, errors: 0, firstError: null as string | null };
266
+ } catch (error: any) {
267
+ const msg = error?.message || error?.originalError?.message || String(error);
268
+ const isSizeError = /too many|payload too large|batch.*size|limit|no longer than \d+ items|array.*\d+ items/i.test(msg)
269
+ || error?.originalError?.code === 413
270
+ || error?.code === 413
271
+ || error?.originalError?.type === "general_argument_invalid";
272
+
273
+ if (isSizeError && sub.length > 50) {
274
+ const newMax = Math.floor(sub.length / 2);
275
+ if (newMax < effectiveBulkMax) {
276
+ effectiveBulkMax = newMax;
277
+ MessageFormatter.warning(`Bulk batch too large, reducing to ${effectiveBulkMax} per call`, { prefix: "Import" });
278
+ }
279
+ // Retry halved sequentially within this slot
280
+ const mid = Math.ceil(sub.length / 2);
281
+ const halves = [sub.slice(0, mid), sub.slice(mid)];
282
+ let hCreated = 0, hErrors = 0;
283
+ let hFirst: string | null = null;
284
+ for (const half of halves) {
285
+ try {
286
+ const r = await tryAwaitWithRetry(async () =>
287
+ adapter.bulkUpsertRows!({ databaseId, tableId, rows: half })
288
+ );
289
+ hCreated += (r as any)?.total || (r as any)?.rows?.length || half.length;
290
+ } catch (e: any) {
291
+ hErrors += half.length;
292
+ if (!hFirst) hFirst = e?.message || String(e);
293
+ }
294
+ }
295
+ return { created: hCreated, errors: hErrors, firstError: hFirst };
296
+ }
297
+ return { created: 0, errors: sub.length, firstError: msg };
298
+ }
299
+ })
300
+ )
301
+ );
302
+ for (const r of results) {
303
+ created += r.created;
304
+ errors += r.errors;
305
+ if (r.firstError && !firstError) firstError = r.firstError;
306
+ }
307
+ } else {
308
+ const limit = pLimit(500);
309
+ const promises = batch.map((row) =>
310
+ limit(async () => {
311
+ try {
312
+ await upsertRow(row);
313
+ created++;
314
+ } catch (error: any) {
315
+ errors++;
316
+ const msg = error?.message || error?.originalError?.message || String(error);
317
+ if (!firstError) firstError = msg;
318
+ }
319
+ })
320
+ );
321
+ await Promise.all(promises);
322
+ }
323
+
324
+ return { created, errors, firstError };
325
+ }
326
+
327
+ // ── Streaming CSV/TSV importer ──────────────────────────────────────────
328
+
329
+ async function countLines(filePath: string): Promise<number> {
330
+ return new Promise((resolve, reject) => {
331
+ let count = 0;
332
+ const stream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
333
+ stream.on("data", (chunk: string | Buffer) => {
334
+ const buf = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
335
+ for (let i = 0; i < buf.length; i++) {
336
+ if (buf[i] === 0x0a) count++;
337
+ }
338
+ });
339
+ stream.on("end", () => resolve(count)); // header line means count ≈ data rows
340
+ stream.on("error", reject);
341
+ });
342
+ }
343
+
344
+ async function importCsvStream(
345
+ adapter: DatabaseAdapter,
346
+ filePath: string,
347
+ databaseId: string,
348
+ tableId: string,
349
+ schema: SchemaInfo,
350
+ delimiter?: string
351
+ ): Promise<void> {
352
+ const useBulk = false; // individual upserts with high concurrency
353
+ const BATCH_SIZE = 500;
354
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
355
+
356
+ if (schema.hasRelationships) {
357
+ MessageFormatter.info("Table has relationships — using individual upserts (25 concurrent)", { prefix: "Import" });
358
+ }
359
+ if (useBulk) {
360
+ MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
361
+ }
362
+
363
+ // Get approximate row count for progress
364
+ const lineCount = await countLines(filePath);
365
+ MessageFormatter.info(`~${lineCount} rows detected (streaming)`, { prefix: "Import" });
366
+
367
+ const progress = ProgressManager.create(`import-${tableId}`, lineCount, {
368
+ title: `Importing into ${tableId}`,
369
+ });
370
+
371
+ let totalCreated = 0;
372
+ let totalErrors = 0;
373
+ let firstError: string | null = null;
374
+ let buffer: any[] = [];
375
+ let headerLogged = false;
376
+
377
+ await new Promise<void>((resolve, reject) => {
378
+ const stream = fs.createReadStream(filePath, { encoding: "utf8" });
379
+
380
+ Papa.parse(stream, {
381
+ header: true,
382
+ dynamicTyping: false,
383
+ skipEmptyLines: true,
384
+ delimiter,
385
+ step: async (result: any, parser: any) => {
386
+ if (result.errors?.length > 0) {
387
+ const critical = result.errors.filter((e: any) => e.type !== "FieldMismatch");
388
+ if (critical.length > 0 && !firstError) {
389
+ firstError = critical[0].message;
390
+ }
391
+ }
392
+
393
+ const cleaned = cleanRow(result.data, schema);
394
+
395
+ if (!headerLogged && cleaned) {
396
+ MessageFormatter.info(`Columns: ${Object.keys(cleaned).join(", ")}`, { prefix: "Import" });
397
+ headerLogged = true;
398
+ }
399
+
400
+ buffer.push(cleaned);
401
+
402
+ if (buffer.length >= BATCH_SIZE) {
403
+ parser.pause();
404
+ const batch = buffer.splice(0, buffer.length);
405
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
406
+ totalCreated += stats.created;
407
+ totalErrors += stats.errors;
408
+ if (stats.firstError && !firstError) firstError = stats.firstError;
409
+ progress.update(totalCreated + totalErrors);
410
+ parser.resume();
411
+ }
412
+ },
413
+ complete: async () => {
414
+ // Flush remaining rows
415
+ if (buffer.length > 0) {
416
+ const stats = await flushBatch(buffer, adapter, databaseId, tableId, schema, useBulk, upsertRow);
417
+ totalCreated += stats.created;
418
+ totalErrors += stats.errors;
419
+ if (stats.firstError && !firstError) firstError = stats.firstError;
420
+ }
421
+ resolve();
422
+ },
423
+ error: (error: any) => {
424
+ reject(error);
425
+ },
426
+ });
427
+ });
428
+
429
+ progress.setTotal(totalCreated + totalErrors);
430
+ progress.stop();
431
+
432
+ if (firstError) {
433
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
434
+ }
435
+ if (totalErrors > 0) {
436
+ MessageFormatter.warning(
437
+ `Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`,
438
+ { prefix: "Import" }
439
+ );
440
+ } else {
441
+ MessageFormatter.success(
442
+ `Successfully imported ${totalCreated} rows into ${tableId}`,
443
+ { prefix: "Import" }
444
+ );
445
+ }
446
+ }
447
+
448
+ // ── Streaming JSONL importer ────────────────────────────────────────────
449
+
450
+ async function importJsonlStream(
451
+ adapter: DatabaseAdapter,
452
+ filePath: string,
453
+ databaseId: string,
454
+ tableId: string,
455
+ schema: SchemaInfo
456
+ ): Promise<void> {
457
+ const useBulk = false; // individual upserts with high concurrency
458
+ const BATCH_SIZE = 500;
459
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
460
+
461
+ if (useBulk) MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
462
+
463
+ const lineCount = await countLines(filePath);
464
+ MessageFormatter.info(`~${lineCount} rows detected (streaming)`, { prefix: "Import" });
465
+
466
+ const progress = ProgressManager.create(`import-${tableId}`, lineCount, {
467
+ title: `Importing into ${tableId}`,
468
+ });
469
+
470
+ let totalCreated = 0;
471
+ let totalErrors = 0;
472
+ let firstError: string | null = null;
473
+ let buffer: any[] = [];
474
+ let headerLogged = false;
475
+
476
+ const rl = readline.createInterface({
477
+ input: fs.createReadStream(filePath, { encoding: "utf8" }),
478
+ crlfDelay: Infinity,
479
+ });
480
+
481
+ for await (const line of rl) {
482
+ const trimmed = line.trim();
483
+ if (!trimmed) continue;
484
+
485
+ let row: any;
486
+ try {
487
+ row = JSON.parse(trimmed);
488
+ } catch {
489
+ totalErrors++;
490
+ if (!firstError) firstError = `Invalid JSON line: ${trimmed.slice(0, 80)}`;
491
+ continue;
492
+ }
493
+
494
+ const cleaned = cleanRow(row, schema);
495
+ if (!headerLogged && cleaned) {
496
+ MessageFormatter.info(`Columns: ${Object.keys(cleaned).join(", ")}`, { prefix: "Import" });
497
+ headerLogged = true;
498
+ }
499
+
500
+ buffer.push(cleaned);
501
+
502
+ if (buffer.length >= BATCH_SIZE) {
503
+ const batch = buffer.splice(0, buffer.length);
504
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
505
+ totalCreated += stats.created;
506
+ totalErrors += stats.errors;
507
+ if (stats.firstError && !firstError) firstError = stats.firstError;
508
+ progress.update(totalCreated + totalErrors);
509
+ }
510
+ }
511
+
512
+ // Flush remaining
513
+ if (buffer.length > 0) {
514
+ const stats = await flushBatch(buffer, adapter, databaseId, tableId, schema, useBulk, upsertRow);
515
+ totalCreated += stats.created;
516
+ totalErrors += stats.errors;
517
+ if (stats.firstError && !firstError) firstError = stats.firstError;
518
+ }
519
+
520
+ progress.setTotal(totalCreated + totalErrors);
521
+ progress.stop();
522
+
523
+ if (firstError) {
524
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
525
+ }
526
+ if (totalErrors > 0) {
527
+ MessageFormatter.warning(
528
+ `Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`,
529
+ { prefix: "Import" }
530
+ );
531
+ } else {
532
+ MessageFormatter.success(
533
+ `Successfully imported ${totalCreated} rows into ${tableId}`,
534
+ { prefix: "Import" }
535
+ );
536
+ }
537
+ }
538
+
539
+ // ── JSON importer (non-streaming, for .json files) ──────────────────────
540
+
541
+ async function importJsonFile(
542
+ adapter: DatabaseAdapter,
543
+ filePath: string,
544
+ databaseId: string,
545
+ tableId: string,
546
+ schema: SchemaInfo
547
+ ): Promise<void> {
548
+ const raw = fs.readFileSync(filePath, "utf8");
549
+ const parsed = JSON.parse(raw);
550
+
551
+ let rows: any[];
552
+ if (Array.isArray(parsed)) {
553
+ rows = parsed;
554
+ } else if (typeof parsed === "object" && parsed !== null) {
555
+ const arrayKey = Object.keys(parsed).find((k) => Array.isArray(parsed[k]));
556
+ if (arrayKey) {
557
+ MessageFormatter.info(
558
+ `Using "${arrayKey}" array from JSON (${parsed[arrayKey].length} items)`,
559
+ { prefix: "Import" }
560
+ );
561
+ rows = parsed[arrayKey];
562
+ } else {
563
+ throw new Error("JSON file must be an array or an object containing an array");
564
+ }
565
+ } else {
566
+ throw new Error("JSON file must be an array or an object containing an array");
567
+ }
568
+
569
+ if (!rows.length) {
570
+ MessageFormatter.warning("No rows found in file", { prefix: "Import" });
571
+ return;
572
+ }
573
+
574
+ MessageFormatter.info(`Parsed ${rows.length} rows`, { prefix: "Import" });
575
+
576
+ const useBulk = false; // individual upserts with high concurrency
577
+ const BATCH_SIZE = 500;
578
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
579
+
580
+ if (useBulk) MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
581
+ if (schema.hasRelationships) {
582
+ MessageFormatter.info("Table has relationships — using individual upserts (25 concurrent)", { prefix: "Import" });
583
+ }
584
+
585
+ const cleanedRows = rows.map((row) => cleanRow(row, schema));
586
+
587
+ MessageFormatter.info(`Columns: ${Object.keys(cleanedRows[0]).join(", ")}`, { prefix: "Import" });
588
+
589
+ const progress = ProgressManager.create(`import-${tableId}`, cleanedRows.length, {
590
+ title: `Importing into ${tableId}`,
591
+ });
592
+
593
+ let totalCreated = 0;
594
+ let totalErrors = 0;
595
+ let firstError: string | null = null;
596
+
597
+ // Process in batches
598
+ for (let i = 0; i < cleanedRows.length; i += BATCH_SIZE) {
599
+ const batch = cleanedRows.slice(i, i + BATCH_SIZE);
600
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
601
+ totalCreated += stats.created;
602
+ totalErrors += stats.errors;
603
+ if (stats.firstError && !firstError) firstError = stats.firstError;
604
+ progress.update(totalCreated + totalErrors);
605
+ }
606
+
607
+ progress.setTotal(totalCreated + totalErrors);
608
+ progress.stop();
609
+
610
+ if (firstError) {
611
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
612
+ }
613
+ if (totalErrors > 0) {
614
+ MessageFormatter.warning(
615
+ `Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`,
616
+ { prefix: "Import" }
617
+ );
618
+ } else {
619
+ MessageFormatter.success(
620
+ `Successfully imported ${totalCreated} rows into ${tableId}`,
621
+ { prefix: "Import" }
622
+ );
623
+ }
624
+ }
625
+
626
+ // ── Main entry point ────────────────────────────────────────────────────
627
+
628
+ export async function importFileFromPath(
629
+ adapter: DatabaseAdapter,
630
+ filePath: string,
631
+ databaseId: string,
632
+ tableId: string
633
+ ): Promise<void> {
634
+ const resolvedPath = path.resolve(filePath);
635
+
636
+ if (!fs.existsSync(resolvedPath)) {
637
+ MessageFormatter.error(`File not found: ${resolvedPath}`, undefined, {
638
+ prefix: "Import",
639
+ });
640
+ return;
641
+ }
642
+
643
+ const ext = path.extname(resolvedPath).toLowerCase();
644
+ const supported = new Set([".csv", ".tsv", ".json", ".jsonl"]);
645
+ if (!supported.has(ext)) {
646
+ MessageFormatter.error(
647
+ `Unsupported file format: ${ext}. Use .csv, .tsv, .json, or .jsonl`,
648
+ undefined,
649
+ { prefix: "Import" }
650
+ );
651
+ return;
652
+ }
653
+
654
+ const stat = fs.statSync(resolvedPath);
655
+ const sizeMB = (stat.size / 1024 / 1024).toFixed(1);
656
+ MessageFormatter.info(`Importing ${path.basename(resolvedPath)} (${sizeMB} MB)`, { prefix: "Import" });
657
+
658
+ const schema = await fetchSchema(adapter, databaseId, tableId);
659
+
660
+ try {
661
+ if (ext === ".csv" || ext === ".tsv") {
662
+ await importCsvStream(adapter, resolvedPath, databaseId, tableId, schema, ext === ".tsv" ? "\t" : undefined);
663
+ } else if (ext === ".jsonl") {
664
+ await importJsonlStream(adapter, resolvedPath, databaseId, tableId, schema);
665
+ } else {
666
+ await importJsonFile(adapter, resolvedPath, databaseId, tableId, schema);
667
+ }
668
+ } catch (error: any) {
669
+ MessageFormatter.error(
670
+ `Failed to import file: ${error.message}`,
671
+ error,
672
+ { prefix: "Import" }
673
+ );
674
+ }
675
+ }
676
+
677
+ // ── Prompt for missing targetDb / targetTable ───────────────────────────
678
+
679
+ export async function importFilePromptMissing(
680
+ adapter: DatabaseAdapter,
681
+ database: any,
682
+ filePath: string,
683
+ targetDb?: string,
684
+ targetTable?: string
685
+ ): Promise<void> {
686
+ let databaseId = targetDb;
687
+ let tableId = targetTable;
688
+
689
+ if (!databaseId) {
690
+ const allDatabases = await fetchAllDatabases(database);
691
+ if (allDatabases.length === 0) {
692
+ MessageFormatter.error("No databases found", undefined, { prefix: "Import" });
693
+ return;
694
+ }
695
+ const { selectedDb } = await inquirer.prompt([
696
+ {
697
+ type: "list",
698
+ name: "selectedDb",
699
+ message: "Select target database:",
700
+ choices: allDatabases.map((db) => ({
701
+ name: `${db.name} (${db.$id})`,
702
+ value: db.$id,
703
+ })),
704
+ },
705
+ ]);
706
+ databaseId = selectedDb;
707
+ }
708
+
709
+ if (!tableId) {
710
+ const tablesResponse = await adapter.listTables({
711
+ databaseId: databaseId!,
712
+ queries: [Query.limit(500)],
713
+ });
714
+ const tables: any[] = tablesResponse.tables || tablesResponse.data || [];
715
+ if (tables.length === 0) {
716
+ MessageFormatter.error("No tables found in selected database", undefined, { prefix: "Import" });
717
+ return;
718
+ }
719
+ const { selectedTable } = await inquirer.prompt([
720
+ {
721
+ type: "list",
722
+ name: "selectedTable",
723
+ message: "Select target table:",
724
+ choices: tables.map((t: any) => ({
725
+ name: `${t.name} (${t.$id})`,
726
+ value: t.$id,
727
+ })),
728
+ },
729
+ ]);
730
+ tableId = selectedTable;
731
+ }
732
+
733
+ await importFileFromPath(adapter, filePath, databaseId!, tableId!);
734
+ }
735
+
736
+ // ── Interactive wrapper ─────────────────────────────────────────────────
737
+
738
+ export const importFileCommands = {
739
+ async importFile(cli: InteractiveCLI): Promise<void> {
740
+ const controller = (cli as any).controller!;
741
+ const adapter = controller.adapter as DatabaseAdapter;
742
+ const database = controller.database!;
743
+
744
+ // 1. Prompt for file path
745
+ const { filePath } = await inquirer.prompt([
746
+ {
747
+ type: "input",
748
+ name: "filePath",
749
+ message: "Path to CSV or JSON file:",
750
+ validate: (input: string) => {
751
+ if (!input || !input.trim()) return "Please provide a file path";
752
+ const resolved = path.resolve(input.trim());
753
+ if (!fs.existsSync(resolved))
754
+ return `File not found: ${resolved}`;
755
+ const ext = path.extname(resolved).toLowerCase();
756
+ if (![".csv", ".tsv", ".json", ".jsonl"].includes(ext))
757
+ return "Supported formats: .csv, .tsv, .json, .jsonl";
758
+ return true;
759
+ },
760
+ },
761
+ ]);
762
+
763
+ // 2. Select database
764
+ const allDatabases = await fetchAllDatabases(database);
765
+ if (allDatabases.length === 0) {
766
+ MessageFormatter.error("No databases found", undefined, {
767
+ prefix: "Import",
768
+ });
769
+ return;
770
+ }
771
+
772
+ const { selectedDb } = await inquirer.prompt([
773
+ {
774
+ type: "list",
775
+ name: "selectedDb",
776
+ message: "Select target database:",
777
+ choices: allDatabases.map((db) => ({
778
+ name: `${db.name} (${db.$id})`,
779
+ value: db.$id,
780
+ })),
781
+ },
782
+ ]);
783
+
784
+ // 3. Select table
785
+ const tablesResponse = await adapter.listTables({
786
+ databaseId: selectedDb,
787
+ queries: [Query.limit(500)],
788
+ });
789
+ const tables: any[] = tablesResponse.tables || tablesResponse.data || [];
790
+
791
+ if (tables.length === 0) {
792
+ MessageFormatter.error(
793
+ "No tables found in selected database",
794
+ undefined,
795
+ { prefix: "Import" }
796
+ );
797
+ return;
798
+ }
799
+
800
+ const { selectedTable } = await inquirer.prompt([
801
+ {
802
+ type: "list",
803
+ name: "selectedTable",
804
+ message: "Select target table:",
805
+ choices: tables.map((t: any) => ({
806
+ name: `${t.name} (${t.$id})`,
807
+ value: t.$id,
808
+ })),
809
+ },
810
+ ]);
811
+
812
+ // 4. Import
813
+ await importFileFromPath(adapter, filePath.trim(), selectedDb, selectedTable);
814
+ },
815
+ };