@njdamstra/appwrite-utils-cli 1.8.9 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (285) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/CONFIG_TODO.md +1189 -0
  3. package/SELECTION_DIALOGS.md +146 -0
  4. package/SERVICE_IMPLEMENTATION_REPORT.md +462 -0
  5. package/dist/adapters/index.d.ts +7 -8
  6. package/dist/adapters/index.js +7 -9
  7. package/dist/backups/operations/bucketBackup.js +2 -2
  8. package/dist/backups/operations/collectionBackup.d.ts +1 -1
  9. package/dist/backups/operations/collectionBackup.js +3 -3
  10. package/dist/backups/operations/comprehensiveBackup.d.ts +1 -1
  11. package/dist/backups/operations/comprehensiveBackup.js +2 -2
  12. package/dist/backups/tracking/centralizedTracking.d.ts +1 -1
  13. package/dist/backups/tracking/centralizedTracking.js +2 -2
  14. package/dist/cli/commands/configCommands.js +51 -7
  15. package/dist/cli/commands/databaseCommands.d.ts +1 -0
  16. package/dist/cli/commands/databaseCommands.js +119 -9
  17. package/dist/cli/commands/functionCommands.js +3 -3
  18. package/dist/cli/commands/importFileCommands.d.ts +7 -0
  19. package/dist/cli/commands/importFileCommands.js +674 -0
  20. package/dist/cli/commands/schemaCommands.js +3 -3
  21. package/dist/cli/commands/storageCommands.js +2 -3
  22. package/dist/cli/commands/transferCommands.js +3 -5
  23. package/dist/collections/attributes.d.ts +1 -1
  24. package/dist/collections/attributes.js +85 -35
  25. package/dist/collections/indexes.js +2 -4
  26. package/dist/collections/methods.d.ts +1 -1
  27. package/dist/collections/methods.js +111 -192
  28. package/dist/collections/tableOperations.d.ts +1 -0
  29. package/dist/collections/tableOperations.js +90 -23
  30. package/dist/collections/transferOperations.d.ts +1 -1
  31. package/dist/collections/transferOperations.js +3 -4
  32. package/dist/collections/wipeOperations.d.ts +4 -3
  33. package/dist/collections/wipeOperations.js +112 -39
  34. package/dist/databases/methods.js +2 -2
  35. package/dist/databases/setup.js +2 -2
  36. package/dist/examples/yamlTerminologyExample.js +2 -2
  37. package/dist/functions/deployments.d.ts +1 -1
  38. package/dist/functions/deployments.js +5 -5
  39. package/dist/functions/fnConfigDiscovery.js +2 -2
  40. package/dist/functions/methods.js +16 -4
  41. package/dist/init.js +1 -1
  42. package/dist/interactiveCLI.d.ts +6 -1
  43. package/dist/interactiveCLI.js +64 -10
  44. package/dist/main.js +130 -177
  45. package/dist/migrations/afterImportActions.js +2 -3
  46. package/dist/migrations/appwriteToX.d.ts +97 -1
  47. package/dist/migrations/appwriteToX.js +9 -7
  48. package/dist/migrations/comprehensiveTransfer.js +3 -5
  49. package/dist/migrations/dataLoader.d.ts +194 -2
  50. package/dist/migrations/dataLoader.js +2 -5
  51. package/dist/migrations/importController.js +3 -4
  52. package/dist/migrations/importDataActions.js +3 -3
  53. package/dist/migrations/relationships.js +1 -2
  54. package/dist/migrations/services/DataTransformationService.js +2 -2
  55. package/dist/migrations/services/FileHandlerService.js +1 -1
  56. package/dist/migrations/services/ImportOrchestrator.js +4 -4
  57. package/dist/migrations/services/RateLimitManager.js +1 -1
  58. package/dist/migrations/services/RelationshipResolver.js +1 -1
  59. package/dist/migrations/services/UserMappingService.js +1 -1
  60. package/dist/migrations/services/ValidationService.js +1 -1
  61. package/dist/migrations/transfer.d.ts +8 -4
  62. package/dist/migrations/transfer.js +106 -55
  63. package/dist/migrations/yaml/YamlImportConfigLoader.js +1 -1
  64. package/dist/migrations/yaml/YamlImportIntegration.js +2 -2
  65. package/dist/migrations/yaml/generateImportSchemas.js +1 -1
  66. package/dist/setupCommands.d.ts +1 -1
  67. package/dist/setupCommands.js +5 -6
  68. package/dist/setupController.js +1 -1
  69. package/dist/shared/backupTracking.d.ts +1 -1
  70. package/dist/shared/backupTracking.js +2 -2
  71. package/dist/shared/confirmationDialogs.js +1 -1
  72. package/dist/shared/migrationHelpers.d.ts +1 -1
  73. package/dist/shared/migrationHelpers.js +3 -3
  74. package/dist/shared/operationQueue.d.ts +1 -1
  75. package/dist/shared/operationQueue.js +2 -3
  76. package/dist/shared/operationsTable.d.ts +1 -1
  77. package/dist/shared/operationsTable.js +2 -2
  78. package/dist/shared/progressManager.js +1 -1
  79. package/dist/shared/selectionDialogs.js +9 -8
  80. package/dist/storage/methods.js +4 -4
  81. package/dist/storage/schemas.d.ts +386 -2
  82. package/dist/tables/indexManager.d.ts +65 -0
  83. package/dist/tables/indexManager.js +294 -0
  84. package/dist/types.d.ts +2 -2
  85. package/dist/types.js +1 -1
  86. package/dist/users/methods.js +2 -3
  87. package/dist/utils/configMigration.js +1 -1
  88. package/dist/utils/index.d.ts +1 -1
  89. package/dist/utils/index.js +1 -1
  90. package/dist/utils/loadConfigs.d.ts +2 -2
  91. package/dist/utils/loadConfigs.js +6 -7
  92. package/dist/utils/setupFiles.js +5 -7
  93. package/dist/utilsController.d.ts +15 -8
  94. package/dist/utilsController.js +57 -28
  95. package/package.json +8 -4
  96. package/src/adapters/index.ts +8 -34
  97. package/src/backups/operations/bucketBackup.ts +2 -2
  98. package/src/backups/operations/collectionBackup.ts +4 -4
  99. package/src/backups/operations/comprehensiveBackup.ts +3 -3
  100. package/src/backups/tracking/centralizedTracking.ts +3 -3
  101. package/src/cli/commands/configCommands.ts +72 -8
  102. package/src/cli/commands/databaseCommands.ts +161 -9
  103. package/src/cli/commands/functionCommands.ts +4 -3
  104. package/src/cli/commands/importFileCommands.ts +815 -0
  105. package/src/cli/commands/schemaCommands.ts +3 -3
  106. package/src/cli/commands/storageCommands.ts +2 -3
  107. package/src/cli/commands/transferCommands.ts +3 -6
  108. package/src/collections/attributes.ts +155 -39
  109. package/src/collections/indexes.ts +5 -7
  110. package/src/collections/methods.ts +115 -150
  111. package/src/collections/tableOperations.ts +92 -21
  112. package/src/collections/transferOperations.ts +4 -5
  113. package/src/collections/wipeOperations.ts +154 -51
  114. package/src/databases/methods.ts +2 -2
  115. package/src/databases/setup.ts +2 -2
  116. package/src/examples/yamlTerminologyExample.ts +2 -2
  117. package/src/functions/deployments.ts +6 -5
  118. package/src/functions/fnConfigDiscovery.ts +2 -2
  119. package/src/functions/methods.ts +19 -6
  120. package/src/init.ts +1 -1
  121. package/src/interactiveCLI.ts +78 -13
  122. package/src/main.ts +143 -287
  123. package/src/migrations/afterImportActions.ts +2 -3
  124. package/src/migrations/appwriteToX.ts +12 -8
  125. package/src/migrations/comprehensiveTransfer.ts +6 -6
  126. package/src/migrations/dataLoader.ts +2 -5
  127. package/src/migrations/importController.ts +3 -4
  128. package/src/migrations/importDataActions.ts +3 -3
  129. package/src/migrations/relationships.ts +1 -2
  130. package/src/migrations/services/DataTransformationService.ts +2 -2
  131. package/src/migrations/services/FileHandlerService.ts +1 -1
  132. package/src/migrations/services/ImportOrchestrator.ts +4 -4
  133. package/src/migrations/services/RateLimitManager.ts +1 -1
  134. package/src/migrations/services/RelationshipResolver.ts +1 -1
  135. package/src/migrations/services/UserMappingService.ts +1 -1
  136. package/src/migrations/services/ValidationService.ts +1 -1
  137. package/src/migrations/transfer.ts +126 -83
  138. package/src/migrations/yaml/YamlImportConfigLoader.ts +1 -1
  139. package/src/migrations/yaml/YamlImportIntegration.ts +2 -2
  140. package/src/migrations/yaml/generateImportSchemas.ts +1 -1
  141. package/src/setupCommands.ts +5 -6
  142. package/src/setupController.ts +1 -1
  143. package/src/shared/backupTracking.ts +3 -3
  144. package/src/shared/confirmationDialogs.ts +1 -1
  145. package/src/shared/migrationHelpers.ts +4 -4
  146. package/src/shared/operationQueue.ts +3 -4
  147. package/src/shared/operationsTable.ts +3 -3
  148. package/src/shared/progressManager.ts +1 -1
  149. package/src/shared/selectionDialogs.ts +9 -8
  150. package/src/storage/methods.ts +4 -4
  151. package/src/tables/indexManager.ts +409 -0
  152. package/src/types.ts +2 -2
  153. package/src/users/methods.ts +2 -3
  154. package/src/utils/configMigration.ts +1 -1
  155. package/src/utils/index.ts +1 -1
  156. package/src/utils/loadConfigs.ts +15 -7
  157. package/src/utils/setupFiles.ts +5 -7
  158. package/src/utilsController.ts +86 -32
  159. package/dist/adapters/AdapterFactory.d.ts +0 -94
  160. package/dist/adapters/AdapterFactory.js +0 -405
  161. package/dist/adapters/DatabaseAdapter.d.ts +0 -233
  162. package/dist/adapters/DatabaseAdapter.js +0 -50
  163. package/dist/adapters/LegacyAdapter.d.ts +0 -50
  164. package/dist/adapters/LegacyAdapter.js +0 -612
  165. package/dist/adapters/TablesDBAdapter.d.ts +0 -45
  166. package/dist/adapters/TablesDBAdapter.js +0 -571
  167. package/dist/config/ConfigManager.d.ts +0 -445
  168. package/dist/config/ConfigManager.js +0 -625
  169. package/dist/config/configMigration.d.ts +0 -87
  170. package/dist/config/configMigration.js +0 -390
  171. package/dist/config/configValidation.d.ts +0 -66
  172. package/dist/config/configValidation.js +0 -358
  173. package/dist/config/index.d.ts +0 -8
  174. package/dist/config/index.js +0 -7
  175. package/dist/config/services/ConfigDiscoveryService.d.ts +0 -126
  176. package/dist/config/services/ConfigDiscoveryService.js +0 -374
  177. package/dist/config/services/ConfigLoaderService.d.ts +0 -129
  178. package/dist/config/services/ConfigLoaderService.js +0 -540
  179. package/dist/config/services/ConfigMergeService.d.ts +0 -208
  180. package/dist/config/services/ConfigMergeService.js +0 -308
  181. package/dist/config/services/ConfigValidationService.d.ts +0 -214
  182. package/dist/config/services/ConfigValidationService.js +0 -310
  183. package/dist/config/services/SessionAuthService.d.ts +0 -225
  184. package/dist/config/services/SessionAuthService.js +0 -456
  185. package/dist/config/services/__tests__/ConfigMergeService.test.d.ts +0 -1
  186. package/dist/config/services/__tests__/ConfigMergeService.test.js +0 -271
  187. package/dist/config/services/index.d.ts +0 -13
  188. package/dist/config/services/index.js +0 -10
  189. package/dist/config/yamlConfig.d.ts +0 -722
  190. package/dist/config/yamlConfig.js +0 -702
  191. package/dist/functions/pathResolution.d.ts +0 -37
  192. package/dist/functions/pathResolution.js +0 -185
  193. package/dist/shared/attributeMapper.d.ts +0 -20
  194. package/dist/shared/attributeMapper.js +0 -203
  195. package/dist/shared/errorUtils.d.ts +0 -54
  196. package/dist/shared/errorUtils.js +0 -95
  197. package/dist/shared/functionManager.d.ts +0 -48
  198. package/dist/shared/functionManager.js +0 -336
  199. package/dist/shared/indexManager.d.ts +0 -24
  200. package/dist/shared/indexManager.js +0 -151
  201. package/dist/shared/jsonSchemaGenerator.d.ts +0 -50
  202. package/dist/shared/jsonSchemaGenerator.js +0 -290
  203. package/dist/shared/logging.d.ts +0 -61
  204. package/dist/shared/logging.js +0 -116
  205. package/dist/shared/messageFormatter.d.ts +0 -39
  206. package/dist/shared/messageFormatter.js +0 -162
  207. package/dist/shared/pydanticModelGenerator.d.ts +0 -17
  208. package/dist/shared/pydanticModelGenerator.js +0 -615
  209. package/dist/shared/schemaGenerator.d.ts +0 -40
  210. package/dist/shared/schemaGenerator.js +0 -556
  211. package/dist/utils/ClientFactory.d.ts +0 -87
  212. package/dist/utils/ClientFactory.js +0 -212
  213. package/dist/utils/configDiscovery.d.ts +0 -78
  214. package/dist/utils/configDiscovery.js +0 -472
  215. package/dist/utils/constantsGenerator.d.ts +0 -31
  216. package/dist/utils/constantsGenerator.js +0 -321
  217. package/dist/utils/dataConverters.d.ts +0 -46
  218. package/dist/utils/dataConverters.js +0 -139
  219. package/dist/utils/directoryUtils.d.ts +0 -22
  220. package/dist/utils/directoryUtils.js +0 -59
  221. package/dist/utils/getClientFromConfig.d.ts +0 -39
  222. package/dist/utils/getClientFromConfig.js +0 -199
  223. package/dist/utils/helperFunctions.d.ts +0 -63
  224. package/dist/utils/helperFunctions.js +0 -156
  225. package/dist/utils/pathResolvers.d.ts +0 -53
  226. package/dist/utils/pathResolvers.js +0 -72
  227. package/dist/utils/projectConfig.d.ts +0 -119
  228. package/dist/utils/projectConfig.js +0 -171
  229. package/dist/utils/retryFailedPromises.d.ts +0 -2
  230. package/dist/utils/retryFailedPromises.js +0 -23
  231. package/dist/utils/sessionAuth.d.ts +0 -48
  232. package/dist/utils/sessionAuth.js +0 -164
  233. package/dist/utils/typeGuards.d.ts +0 -35
  234. package/dist/utils/typeGuards.js +0 -57
  235. package/dist/utils/validationRules.d.ts +0 -43
  236. package/dist/utils/validationRules.js +0 -42
  237. package/dist/utils/versionDetection.d.ts +0 -58
  238. package/dist/utils/versionDetection.js +0 -251
  239. package/dist/utils/yamlConverter.d.ts +0 -100
  240. package/dist/utils/yamlConverter.js +0 -428
  241. package/dist/utils/yamlLoader.d.ts +0 -70
  242. package/dist/utils/yamlLoader.js +0 -267
  243. package/src/adapters/AdapterFactory.ts +0 -510
  244. package/src/adapters/DatabaseAdapter.ts +0 -306
  245. package/src/adapters/LegacyAdapter.ts +0 -841
  246. package/src/adapters/TablesDBAdapter.ts +0 -773
  247. package/src/config/ConfigManager.ts +0 -808
  248. package/src/config/README.md +0 -274
  249. package/src/config/configMigration.ts +0 -575
  250. package/src/config/configValidation.ts +0 -445
  251. package/src/config/index.ts +0 -10
  252. package/src/config/services/ConfigDiscoveryService.ts +0 -463
  253. package/src/config/services/ConfigLoaderService.ts +0 -740
  254. package/src/config/services/ConfigMergeService.ts +0 -388
  255. package/src/config/services/ConfigValidationService.ts +0 -394
  256. package/src/config/services/SessionAuthService.ts +0 -565
  257. package/src/config/services/__tests__/ConfigMergeService.test.ts +0 -351
  258. package/src/config/services/index.ts +0 -29
  259. package/src/config/yamlConfig.ts +0 -761
  260. package/src/functions/pathResolution.ts +0 -227
  261. package/src/shared/attributeMapper.ts +0 -229
  262. package/src/shared/errorUtils.ts +0 -110
  263. package/src/shared/functionManager.ts +0 -525
  264. package/src/shared/indexManager.ts +0 -254
  265. package/src/shared/jsonSchemaGenerator.ts +0 -383
  266. package/src/shared/logging.ts +0 -149
  267. package/src/shared/messageFormatter.ts +0 -208
  268. package/src/shared/pydanticModelGenerator.ts +0 -618
  269. package/src/shared/schemaGenerator.ts +0 -644
  270. package/src/utils/ClientFactory.ts +0 -240
  271. package/src/utils/configDiscovery.ts +0 -557
  272. package/src/utils/constantsGenerator.ts +0 -369
  273. package/src/utils/dataConverters.ts +0 -159
  274. package/src/utils/directoryUtils.ts +0 -61
  275. package/src/utils/getClientFromConfig.ts +0 -257
  276. package/src/utils/helperFunctions.ts +0 -228
  277. package/src/utils/pathResolvers.ts +0 -81
  278. package/src/utils/projectConfig.ts +0 -299
  279. package/src/utils/retryFailedPromises.ts +0 -29
  280. package/src/utils/sessionAuth.ts +0 -230
  281. package/src/utils/typeGuards.ts +0 -65
  282. package/src/utils/validationRules.ts +0 -88
  283. package/src/utils/versionDetection.ts +0 -292
  284. package/src/utils/yamlConverter.ts +0 -542
  285. package/src/utils/yamlLoader.ts +0 -371
@@ -0,0 +1,674 @@
1
+ import inquirer from "inquirer";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import readline from "node:readline";
5
+ import Papa from "papaparse";
6
+ import { ID, Query } from "node-appwrite";
7
+ import pLimit from "p-limit";
8
+ import { MessageFormatter, tryAwaitWithRetry, delay, } from "@njdamstra/appwrite-utils-helpers";
9
+ import { ProgressManager } from "../../shared/progressManager.js";
10
+ import { fetchAllDatabases } from "../../databases/methods.js";
11
+ const SYSTEM_FIELDS = new Set([
12
+ "$databaseId",
13
+ "$collectionId",
14
+ "$tableId",
15
+ "$sequence",
16
+ "$tenant",
17
+ ]);
18
+ const PASSTHROUGH_FIELDS = new Set(["$id", "$permissions", "$createdAt", "$updatedAt"]);
19
+ async function fetchSchema(adapter, databaseId, tableId) {
20
+ MessageFormatter.info(`Fetching schema for database="${databaseId}" table="${tableId}"`, { prefix: "Import" });
21
+ const tableInfo = await adapter.getTable({ databaseId, tableId });
22
+ const columns = tableInfo.data?.columns || tableInfo.data?.attributes || [];
23
+ const availableColumns = columns.filter((col) => !col.status || col.status === "available");
24
+ MessageFormatter.info(`Schema columns: ${availableColumns.map((c) => c.key || c.$id).join(", ")}`, { prefix: "Import" });
25
+ const columnTypeMap = new Map();
26
+ const enumElementsMap = new Map();
27
+ const arrayColumns = new Set();
28
+ const hasRelationships = availableColumns.some((col) => col.type === "relationship");
29
+ for (const col of availableColumns) {
30
+ const key = col.key || col.$id;
31
+ if (!key)
32
+ continue;
33
+ columnTypeMap.set(key, col.type);
34
+ if (col.array)
35
+ arrayColumns.add(key);
36
+ if (col.type === "enum" && Array.isArray(col.elements)) {
37
+ enumElementsMap.set(key, new Set(col.elements));
38
+ }
39
+ }
40
+ if (columnTypeMap.size > 0) {
41
+ MessageFormatter.info(`Table schema: ${columns.length} columns detected, will cast values to match types`, { prefix: "Import" });
42
+ }
43
+ return { columnTypeMap, arrayColumns, enumElementsMap, hasRelationships };
44
+ }
45
+ // ── Row cleaning ────────────────────────────────────────────────────────
46
+ function cleanRow(row, schema) {
47
+ const { columnTypeMap, arrayColumns } = schema;
48
+ const clean = {};
49
+ for (const [key, value] of Object.entries(row)) {
50
+ if (SYSTEM_FIELDS.has(key))
51
+ continue;
52
+ if (PASSTHROUGH_FIELDS.has(key)) {
53
+ if (value === null || value === undefined || value === "null" || value === "undefined" || value === "")
54
+ continue;
55
+ if (key === "$permissions") {
56
+ if (Array.isArray(value)) {
57
+ clean[key] = value;
58
+ }
59
+ else if (typeof value === "string" && value.trim()) {
60
+ clean[key] = value.match(/[a-z]+\([^)]*\)/g) || [];
61
+ }
62
+ }
63
+ else if (key === "$createdAt" || key === "$updatedAt") {
64
+ const str = value instanceof Date ? value.toISOString() : String(value).trim();
65
+ // Only set if it looks like a valid ISO datetime
66
+ if (str && !isNaN(Date.parse(str))) {
67
+ clean[key] = str;
68
+ }
69
+ }
70
+ else {
71
+ clean[key] = value;
72
+ }
73
+ continue;
74
+ }
75
+ if (columnTypeMap.size > 0 && !columnTypeMap.has(key))
76
+ continue;
77
+ if (value === null || value === undefined || value === "null")
78
+ continue;
79
+ const colType = columnTypeMap.get(key);
80
+ if (colType === "relationship")
81
+ continue;
82
+ // Array columns
83
+ if (arrayColumns.has(key)) {
84
+ let arr;
85
+ if (Array.isArray(value)) {
86
+ arr = value;
87
+ }
88
+ else if (typeof value === "string") {
89
+ const trimmed = value.trim();
90
+ if (!trimmed || trimmed === "[]") {
91
+ clean[key] = [];
92
+ continue;
93
+ }
94
+ if (trimmed.startsWith("[")) {
95
+ try {
96
+ arr = JSON.parse(trimmed);
97
+ }
98
+ catch {
99
+ arr = trimmed.split(",").map((s) => s.trim()).filter(Boolean);
100
+ }
101
+ }
102
+ else {
103
+ arr = trimmed.split(",").map((s) => s.trim()).filter(Boolean);
104
+ }
105
+ }
106
+ else {
107
+ arr = [value];
108
+ }
109
+ // Cast elements to scalar type
110
+ if (colType === "integer") {
111
+ arr = arr.map((v) => typeof v === "number" ? Math.round(v) : parseInt(String(v), 10)).filter((v) => !isNaN(v));
112
+ }
113
+ else if (colType === "float" || colType === "double") {
114
+ arr = arr.map((v) => typeof v === "number" ? v : parseFloat(String(v))).filter((v) => !isNaN(v));
115
+ }
116
+ else if (colType === "boolean") {
117
+ arr = arr.map((v) => {
118
+ if (typeof v === "boolean")
119
+ return v;
120
+ const s = String(v).toLowerCase().trim();
121
+ return s === "true" || s === "1" || s === "yes";
122
+ });
123
+ }
124
+ else {
125
+ arr = arr.map((v) => String(v));
126
+ }
127
+ clean[key] = arr;
128
+ continue;
129
+ }
130
+ // Scalar casting
131
+ if (colType === "enum") {
132
+ clean[key] = String(value);
133
+ }
134
+ else if (colType === "boolean") {
135
+ if (typeof value === "boolean") {
136
+ clean[key] = value;
137
+ }
138
+ else if (typeof value === "string") {
139
+ const lower = value.toLowerCase().trim();
140
+ if (lower === "true" || lower === "1" || lower === "yes") {
141
+ clean[key] = true;
142
+ }
143
+ else if (lower === "false" || lower === "0" || lower === "no" || lower === "") {
144
+ clean[key] = false;
145
+ }
146
+ else {
147
+ clean[key] = Boolean(value);
148
+ }
149
+ }
150
+ else if (typeof value === "number") {
151
+ clean[key] = value !== 0;
152
+ }
153
+ else {
154
+ clean[key] = Boolean(value);
155
+ }
156
+ }
157
+ else if (colType === "integer") {
158
+ if (typeof value === "number") {
159
+ clean[key] = Math.round(value);
160
+ }
161
+ else if (typeof value === "string" && value.trim() !== "") {
162
+ const parsed = parseInt(value, 10);
163
+ if (!isNaN(parsed))
164
+ clean[key] = parsed;
165
+ }
166
+ }
167
+ else if (colType === "float" || colType === "double") {
168
+ if (typeof value === "number") {
169
+ clean[key] = value;
170
+ }
171
+ else if (typeof value === "string" && value.trim() !== "") {
172
+ const parsed = parseFloat(value);
173
+ if (!isNaN(parsed))
174
+ clean[key] = parsed;
175
+ }
176
+ }
177
+ else if (typeof value === "string" && value === "") {
178
+ // Skip empty strings — let database use defaults
179
+ continue;
180
+ }
181
+ else {
182
+ clean[key] = value;
183
+ }
184
+ }
185
+ return clean;
186
+ }
187
+ // ── Upsert helpers ──────────────────────────────────────────────────────
188
+ function makeUpsertRow(adapter, databaseId, tableId) {
189
+ return async (row) => {
190
+ const { $id, $permissions, ...data } = row;
191
+ const rowId = $id || ID.unique();
192
+ const permissions = Array.isArray($permissions) ? $permissions : undefined;
193
+ try {
194
+ await adapter.createRow({ databaseId, tableId, id: rowId, data, permissions });
195
+ }
196
+ catch (error) {
197
+ const code = error?.originalError?.code || error?.code;
198
+ if (code === 409) {
199
+ await adapter.updateRow({ databaseId, tableId, id: rowId, data, permissions });
200
+ }
201
+ else {
202
+ throw error;
203
+ }
204
+ }
205
+ };
206
+ }
207
+ // Track the max working bulk size across calls so we don't retry sizes that already failed
208
+ let effectiveBulkMax = 2500;
209
+ async function flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow) {
210
+ let created = 0;
211
+ let errors = 0;
212
+ let firstError = null;
213
+ if (useBulk) {
214
+ // Deduplicate by $id within the batch — keep last occurrence
215
+ const deduped = new Map();
216
+ for (const row of batch) {
217
+ const id = row.$id || ID.unique();
218
+ if (!row.$id)
219
+ row.$id = id;
220
+ deduped.set(id, row);
221
+ }
222
+ const dedupedRows = Array.from(deduped.values());
223
+ // Split into sub-batches of effectiveBulkMax and run up to 5 concurrently
224
+ const subBatches = [];
225
+ for (let i = 0; i < dedupedRows.length; i += effectiveBulkMax) {
226
+ subBatches.push(dedupedRows.slice(i, i + effectiveBulkMax));
227
+ }
228
+ const bulkLimit = pLimit(500);
229
+ const results = await Promise.all(subBatches.map((sub) => bulkLimit(async () => {
230
+ try {
231
+ const result = await tryAwaitWithRetry(async () => adapter.bulkUpsertRows({ databaseId, tableId, rows: sub }));
232
+ return { created: result?.total || result?.rows?.length || sub.length, errors: 0, firstError: null };
233
+ }
234
+ catch (error) {
235
+ const msg = error?.message || error?.originalError?.message || String(error);
236
+ const isSizeError = /too many|payload too large|batch.*size|limit|no longer than \d+ items|array.*\d+ items/i.test(msg)
237
+ || error?.originalError?.code === 413
238
+ || error?.code === 413
239
+ || error?.originalError?.type === "general_argument_invalid";
240
+ if (isSizeError && sub.length > 50) {
241
+ const newMax = Math.floor(sub.length / 2);
242
+ if (newMax < effectiveBulkMax) {
243
+ effectiveBulkMax = newMax;
244
+ MessageFormatter.warning(`Bulk batch too large, reducing to ${effectiveBulkMax} per call`, { prefix: "Import" });
245
+ }
246
+ // Retry halved sequentially within this slot
247
+ const mid = Math.ceil(sub.length / 2);
248
+ const halves = [sub.slice(0, mid), sub.slice(mid)];
249
+ let hCreated = 0, hErrors = 0;
250
+ let hFirst = null;
251
+ for (const half of halves) {
252
+ try {
253
+ const r = await tryAwaitWithRetry(async () => adapter.bulkUpsertRows({ databaseId, tableId, rows: half }));
254
+ hCreated += r?.total || r?.rows?.length || half.length;
255
+ }
256
+ catch (e) {
257
+ hErrors += half.length;
258
+ if (!hFirst)
259
+ hFirst = e?.message || String(e);
260
+ }
261
+ }
262
+ return { created: hCreated, errors: hErrors, firstError: hFirst };
263
+ }
264
+ return { created: 0, errors: sub.length, firstError: msg };
265
+ }
266
+ })));
267
+ for (const r of results) {
268
+ created += r.created;
269
+ errors += r.errors;
270
+ if (r.firstError && !firstError)
271
+ firstError = r.firstError;
272
+ }
273
+ }
274
+ else {
275
+ const limit = pLimit(500);
276
+ const promises = batch.map((row) => limit(async () => {
277
+ try {
278
+ await upsertRow(row);
279
+ created++;
280
+ }
281
+ catch (error) {
282
+ errors++;
283
+ const msg = error?.message || error?.originalError?.message || String(error);
284
+ if (!firstError)
285
+ firstError = msg;
286
+ }
287
+ }));
288
+ await Promise.all(promises);
289
+ }
290
+ return { created, errors, firstError };
291
+ }
292
+ // ── Streaming CSV/TSV importer ──────────────────────────────────────────
293
+ async function countLines(filePath) {
294
+ return new Promise((resolve, reject) => {
295
+ let count = 0;
296
+ const stream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
297
+ stream.on("data", (chunk) => {
298
+ const buf = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
299
+ for (let i = 0; i < buf.length; i++) {
300
+ if (buf[i] === 0x0a)
301
+ count++;
302
+ }
303
+ });
304
+ stream.on("end", () => resolve(count)); // header line means count ≈ data rows
305
+ stream.on("error", reject);
306
+ });
307
+ }
308
+ async function importCsvStream(adapter, filePath, databaseId, tableId, schema, delimiter) {
309
+ const useBulk = false; // individual upserts with high concurrency
310
+ const BATCH_SIZE = 500;
311
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
312
+ if (schema.hasRelationships) {
313
+ MessageFormatter.info("Table has relationships — using individual upserts (25 concurrent)", { prefix: "Import" });
314
+ }
315
+ if (useBulk) {
316
+ MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
317
+ }
318
+ // Get approximate row count for progress
319
+ const lineCount = await countLines(filePath);
320
+ MessageFormatter.info(`~${lineCount} rows detected (streaming)`, { prefix: "Import" });
321
+ const progress = ProgressManager.create(`import-${tableId}`, lineCount, {
322
+ title: `Importing into ${tableId}`,
323
+ });
324
+ let totalCreated = 0;
325
+ let totalErrors = 0;
326
+ let firstError = null;
327
+ let buffer = [];
328
+ let headerLogged = false;
329
+ await new Promise((resolve, reject) => {
330
+ const stream = fs.createReadStream(filePath, { encoding: "utf8" });
331
+ Papa.parse(stream, {
332
+ header: true,
333
+ dynamicTyping: false,
334
+ skipEmptyLines: true,
335
+ delimiter,
336
+ step: async (result, parser) => {
337
+ if (result.errors?.length > 0) {
338
+ const critical = result.errors.filter((e) => e.type !== "FieldMismatch");
339
+ if (critical.length > 0 && !firstError) {
340
+ firstError = critical[0].message;
341
+ }
342
+ }
343
+ const cleaned = cleanRow(result.data, schema);
344
+ if (!headerLogged && cleaned) {
345
+ MessageFormatter.info(`Columns: ${Object.keys(cleaned).join(", ")}`, { prefix: "Import" });
346
+ headerLogged = true;
347
+ }
348
+ buffer.push(cleaned);
349
+ if (buffer.length >= BATCH_SIZE) {
350
+ parser.pause();
351
+ const batch = buffer.splice(0, buffer.length);
352
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
353
+ totalCreated += stats.created;
354
+ totalErrors += stats.errors;
355
+ if (stats.firstError && !firstError)
356
+ firstError = stats.firstError;
357
+ progress.update(totalCreated + totalErrors);
358
+ parser.resume();
359
+ }
360
+ },
361
+ complete: async () => {
362
+ // Flush remaining rows
363
+ if (buffer.length > 0) {
364
+ const stats = await flushBatch(buffer, adapter, databaseId, tableId, schema, useBulk, upsertRow);
365
+ totalCreated += stats.created;
366
+ totalErrors += stats.errors;
367
+ if (stats.firstError && !firstError)
368
+ firstError = stats.firstError;
369
+ }
370
+ resolve();
371
+ },
372
+ error: (error) => {
373
+ reject(error);
374
+ },
375
+ });
376
+ });
377
+ progress.setTotal(totalCreated + totalErrors);
378
+ progress.stop();
379
+ if (firstError) {
380
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
381
+ }
382
+ if (totalErrors > 0) {
383
+ MessageFormatter.warning(`Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`, { prefix: "Import" });
384
+ }
385
+ else {
386
+ MessageFormatter.success(`Successfully imported ${totalCreated} rows into ${tableId}`, { prefix: "Import" });
387
+ }
388
+ }
389
+ // ── Streaming JSONL importer ────────────────────────────────────────────
390
+ async function importJsonlStream(adapter, filePath, databaseId, tableId, schema) {
391
+ const useBulk = false; // individual upserts with high concurrency
392
+ const BATCH_SIZE = 500;
393
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
394
+ if (useBulk)
395
+ MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
396
+ const lineCount = await countLines(filePath);
397
+ MessageFormatter.info(`~${lineCount} rows detected (streaming)`, { prefix: "Import" });
398
+ const progress = ProgressManager.create(`import-${tableId}`, lineCount, {
399
+ title: `Importing into ${tableId}`,
400
+ });
401
+ let totalCreated = 0;
402
+ let totalErrors = 0;
403
+ let firstError = null;
404
+ let buffer = [];
405
+ let headerLogged = false;
406
+ const rl = readline.createInterface({
407
+ input: fs.createReadStream(filePath, { encoding: "utf8" }),
408
+ crlfDelay: Infinity,
409
+ });
410
+ for await (const line of rl) {
411
+ const trimmed = line.trim();
412
+ if (!trimmed)
413
+ continue;
414
+ let row;
415
+ try {
416
+ row = JSON.parse(trimmed);
417
+ }
418
+ catch {
419
+ totalErrors++;
420
+ if (!firstError)
421
+ firstError = `Invalid JSON line: ${trimmed.slice(0, 80)}`;
422
+ continue;
423
+ }
424
+ const cleaned = cleanRow(row, schema);
425
+ if (!headerLogged && cleaned) {
426
+ MessageFormatter.info(`Columns: ${Object.keys(cleaned).join(", ")}`, { prefix: "Import" });
427
+ headerLogged = true;
428
+ }
429
+ buffer.push(cleaned);
430
+ if (buffer.length >= BATCH_SIZE) {
431
+ const batch = buffer.splice(0, buffer.length);
432
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
433
+ totalCreated += stats.created;
434
+ totalErrors += stats.errors;
435
+ if (stats.firstError && !firstError)
436
+ firstError = stats.firstError;
437
+ progress.update(totalCreated + totalErrors);
438
+ }
439
+ }
440
+ // Flush remaining
441
+ if (buffer.length > 0) {
442
+ const stats = await flushBatch(buffer, adapter, databaseId, tableId, schema, useBulk, upsertRow);
443
+ totalCreated += stats.created;
444
+ totalErrors += stats.errors;
445
+ if (stats.firstError && !firstError)
446
+ firstError = stats.firstError;
447
+ }
448
+ progress.setTotal(totalCreated + totalErrors);
449
+ progress.stop();
450
+ if (firstError) {
451
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
452
+ }
453
+ if (totalErrors > 0) {
454
+ MessageFormatter.warning(`Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`, { prefix: "Import" });
455
+ }
456
+ else {
457
+ MessageFormatter.success(`Successfully imported ${totalCreated} rows into ${tableId}`, { prefix: "Import" });
458
+ }
459
+ }
460
+ // ── JSON importer (non-streaming, for .json files) ──────────────────────
461
+ async function importJsonFile(adapter, filePath, databaseId, tableId, schema) {
462
+ const raw = fs.readFileSync(filePath, "utf8");
463
+ const parsed = JSON.parse(raw);
464
+ let rows;
465
+ if (Array.isArray(parsed)) {
466
+ rows = parsed;
467
+ }
468
+ else if (typeof parsed === "object" && parsed !== null) {
469
+ const arrayKey = Object.keys(parsed).find((k) => Array.isArray(parsed[k]));
470
+ if (arrayKey) {
471
+ MessageFormatter.info(`Using "${arrayKey}" array from JSON (${parsed[arrayKey].length} items)`, { prefix: "Import" });
472
+ rows = parsed[arrayKey];
473
+ }
474
+ else {
475
+ throw new Error("JSON file must be an array or an object containing an array");
476
+ }
477
+ }
478
+ else {
479
+ throw new Error("JSON file must be an array or an object containing an array");
480
+ }
481
+ if (!rows.length) {
482
+ MessageFormatter.warning("No rows found in file", { prefix: "Import" });
483
+ return;
484
+ }
485
+ MessageFormatter.info(`Parsed ${rows.length} rows`, { prefix: "Import" });
486
+ const useBulk = false; // individual upserts with high concurrency
487
+ const BATCH_SIZE = 500;
488
+ const upsertRow = makeUpsertRow(adapter, databaseId, tableId);
489
+ if (useBulk)
490
+ MessageFormatter.info("Using bulk upsert", { prefix: "Import" });
491
+ if (schema.hasRelationships) {
492
+ MessageFormatter.info("Table has relationships — using individual upserts (25 concurrent)", { prefix: "Import" });
493
+ }
494
+ const cleanedRows = rows.map((row) => cleanRow(row, schema));
495
+ MessageFormatter.info(`Columns: ${Object.keys(cleanedRows[0]).join(", ")}`, { prefix: "Import" });
496
+ const progress = ProgressManager.create(`import-${tableId}`, cleanedRows.length, {
497
+ title: `Importing into ${tableId}`,
498
+ });
499
+ let totalCreated = 0;
500
+ let totalErrors = 0;
501
+ let firstError = null;
502
+ // Process in batches
503
+ for (let i = 0; i < cleanedRows.length; i += BATCH_SIZE) {
504
+ const batch = cleanedRows.slice(i, i + BATCH_SIZE);
505
+ const stats = await flushBatch(batch, adapter, databaseId, tableId, schema, useBulk, upsertRow);
506
+ totalCreated += stats.created;
507
+ totalErrors += stats.errors;
508
+ if (stats.firstError && !firstError)
509
+ firstError = stats.firstError;
510
+ progress.update(totalCreated + totalErrors);
511
+ }
512
+ progress.setTotal(totalCreated + totalErrors);
513
+ progress.stop();
514
+ if (firstError) {
515
+ MessageFormatter.error(`Sample error: ${firstError}`, undefined, { prefix: "Import" });
516
+ }
517
+ if (totalErrors > 0) {
518
+ MessageFormatter.warning(`Imported ${totalCreated} rows with ${totalErrors} errors into ${tableId}`, { prefix: "Import" });
519
+ }
520
+ else {
521
+ MessageFormatter.success(`Successfully imported ${totalCreated} rows into ${tableId}`, { prefix: "Import" });
522
+ }
523
+ }
524
+ // ── Main entry point ────────────────────────────────────────────────────
525
+ export async function importFileFromPath(adapter, filePath, databaseId, tableId) {
526
+ const resolvedPath = path.resolve(filePath);
527
+ if (!fs.existsSync(resolvedPath)) {
528
+ MessageFormatter.error(`File not found: ${resolvedPath}`, undefined, {
529
+ prefix: "Import",
530
+ });
531
+ return;
532
+ }
533
+ const ext = path.extname(resolvedPath).toLowerCase();
534
+ const supported = new Set([".csv", ".tsv", ".json", ".jsonl"]);
535
+ if (!supported.has(ext)) {
536
+ MessageFormatter.error(`Unsupported file format: ${ext}. Use .csv, .tsv, .json, or .jsonl`, undefined, { prefix: "Import" });
537
+ return;
538
+ }
539
+ const stat = fs.statSync(resolvedPath);
540
+ const sizeMB = (stat.size / 1024 / 1024).toFixed(1);
541
+ MessageFormatter.info(`Importing ${path.basename(resolvedPath)} (${sizeMB} MB)`, { prefix: "Import" });
542
+ const schema = await fetchSchema(adapter, databaseId, tableId);
543
+ try {
544
+ if (ext === ".csv" || ext === ".tsv") {
545
+ await importCsvStream(adapter, resolvedPath, databaseId, tableId, schema, ext === ".tsv" ? "\t" : undefined);
546
+ }
547
+ else if (ext === ".jsonl") {
548
+ await importJsonlStream(adapter, resolvedPath, databaseId, tableId, schema);
549
+ }
550
+ else {
551
+ await importJsonFile(adapter, resolvedPath, databaseId, tableId, schema);
552
+ }
553
+ }
554
+ catch (error) {
555
+ MessageFormatter.error(`Failed to import file: ${error.message}`, error, { prefix: "Import" });
556
+ }
557
+ }
558
+ // ── Prompt for missing targetDb / targetTable ───────────────────────────
559
+ export async function importFilePromptMissing(adapter, database, filePath, targetDb, targetTable) {
560
+ let databaseId = targetDb;
561
+ let tableId = targetTable;
562
+ if (!databaseId) {
563
+ const allDatabases = await fetchAllDatabases(database);
564
+ if (allDatabases.length === 0) {
565
+ MessageFormatter.error("No databases found", undefined, { prefix: "Import" });
566
+ return;
567
+ }
568
+ const { selectedDb } = await inquirer.prompt([
569
+ {
570
+ type: "list",
571
+ name: "selectedDb",
572
+ message: "Select target database:",
573
+ choices: allDatabases.map((db) => ({
574
+ name: `${db.name} (${db.$id})`,
575
+ value: db.$id,
576
+ })),
577
+ },
578
+ ]);
579
+ databaseId = selectedDb;
580
+ }
581
+ if (!tableId) {
582
+ const tablesResponse = await adapter.listTables({
583
+ databaseId: databaseId,
584
+ queries: [Query.limit(500)],
585
+ });
586
+ const tables = tablesResponse.tables || tablesResponse.data || [];
587
+ if (tables.length === 0) {
588
+ MessageFormatter.error("No tables found in selected database", undefined, { prefix: "Import" });
589
+ return;
590
+ }
591
+ const { selectedTable } = await inquirer.prompt([
592
+ {
593
+ type: "list",
594
+ name: "selectedTable",
595
+ message: "Select target table:",
596
+ choices: tables.map((t) => ({
597
+ name: `${t.name} (${t.$id})`,
598
+ value: t.$id,
599
+ })),
600
+ },
601
+ ]);
602
+ tableId = selectedTable;
603
+ }
604
+ await importFileFromPath(adapter, filePath, databaseId, tableId);
605
+ }
606
+ // ── Interactive wrapper ─────────────────────────────────────────────────
607
+ export const importFileCommands = {
608
+ async importFile(cli) {
609
+ const controller = cli.controller;
610
+ const adapter = controller.adapter;
611
+ const database = controller.database;
612
+ // 1. Prompt for file path
613
+ const { filePath } = await inquirer.prompt([
614
+ {
615
+ type: "input",
616
+ name: "filePath",
617
+ message: "Path to CSV or JSON file:",
618
+ validate: (input) => {
619
+ if (!input || !input.trim())
620
+ return "Please provide a file path";
621
+ const resolved = path.resolve(input.trim());
622
+ if (!fs.existsSync(resolved))
623
+ return `File not found: ${resolved}`;
624
+ const ext = path.extname(resolved).toLowerCase();
625
+ if (![".csv", ".tsv", ".json", ".jsonl"].includes(ext))
626
+ return "Supported formats: .csv, .tsv, .json, .jsonl";
627
+ return true;
628
+ },
629
+ },
630
+ ]);
631
+ // 2. Select database
632
+ const allDatabases = await fetchAllDatabases(database);
633
+ if (allDatabases.length === 0) {
634
+ MessageFormatter.error("No databases found", undefined, {
635
+ prefix: "Import",
636
+ });
637
+ return;
638
+ }
639
+ const { selectedDb } = await inquirer.prompt([
640
+ {
641
+ type: "list",
642
+ name: "selectedDb",
643
+ message: "Select target database:",
644
+ choices: allDatabases.map((db) => ({
645
+ name: `${db.name} (${db.$id})`,
646
+ value: db.$id,
647
+ })),
648
+ },
649
+ ]);
650
+ // 3. Select table
651
+ const tablesResponse = await adapter.listTables({
652
+ databaseId: selectedDb,
653
+ queries: [Query.limit(500)],
654
+ });
655
+ const tables = tablesResponse.tables || tablesResponse.data || [];
656
+ if (tables.length === 0) {
657
+ MessageFormatter.error("No tables found in selected database", undefined, { prefix: "Import" });
658
+ return;
659
+ }
660
+ const { selectedTable } = await inquirer.prompt([
661
+ {
662
+ type: "list",
663
+ name: "selectedTable",
664
+ message: "Select target table:",
665
+ choices: tables.map((t) => ({
666
+ name: `${t.name} (${t.$id})`,
667
+ value: t.$id,
668
+ })),
669
+ },
670
+ ]);
671
+ // 4. Import
672
+ await importFileFromPath(adapter, filePath.trim(), selectedDb, selectedTable);
673
+ },
674
+ };
@@ -1,8 +1,8 @@
1
1
  import inquirer from "inquirer";
2
2
  import path from "path";
3
3
  import chalk from "chalk";
4
- import { MessageFormatter } from "../../shared/messageFormatter.js";
5
- import { SchemaGenerator } from "../../shared/schemaGenerator.js";
4
+ import { MessageFormatter } from '@njdamstra/appwrite-utils-helpers';
5
+ import { SchemaGenerator } from '@njdamstra/appwrite-utils-helpers';
6
6
  import { setupDirsFiles } from "../../utils/setupFiles.js";
7
7
  import { fetchAllDatabases } from "../../databases/methods.js";
8
8
  export const schemaCommands = {
@@ -112,7 +112,7 @@ export const schemaCommands = {
112
112
  },
113
113
  ]);
114
114
  try {
115
- const { ConstantsGenerator } = await import("../../utils/constantsGenerator.js");
115
+ const { ConstantsGenerator } = await import("@njdamstra/appwrite-utils-helpers");
116
116
  const generator = new ConstantsGenerator(cli.controller.config);
117
117
  const include = {
118
118
  databases: includeWhat.includes('databases'),