appwrite-utils-cli 1.11.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/{src/adapters/index.ts → dist/adapters/index.d.ts} +0 -1
  2. package/dist/adapters/index.js +10 -0
  3. package/dist/backups/operations/bucketBackup.d.ts +19 -0
  4. package/dist/backups/operations/bucketBackup.js +197 -0
  5. package/dist/backups/operations/collectionBackup.d.ts +30 -0
  6. package/dist/backups/operations/collectionBackup.js +201 -0
  7. package/dist/backups/operations/comprehensiveBackup.d.ts +25 -0
  8. package/dist/backups/operations/comprehensiveBackup.js +238 -0
  9. package/dist/backups/schemas/bucketManifest.d.ts +93 -0
  10. package/dist/backups/schemas/bucketManifest.js +33 -0
  11. package/dist/backups/schemas/comprehensiveManifest.d.ts +108 -0
  12. package/dist/backups/schemas/comprehensiveManifest.js +32 -0
  13. package/dist/backups/tracking/centralizedTracking.d.ts +34 -0
  14. package/dist/backups/tracking/centralizedTracking.js +274 -0
  15. package/dist/cli/commands/configCommands.d.ts +8 -0
  16. package/dist/cli/commands/configCommands.js +210 -0
  17. package/dist/cli/commands/databaseCommands.d.ts +14 -0
  18. package/dist/cli/commands/databaseCommands.js +696 -0
  19. package/dist/cli/commands/functionCommands.d.ts +7 -0
  20. package/dist/cli/commands/functionCommands.js +330 -0
  21. package/dist/cli/commands/importFileCommands.d.ts +7 -0
  22. package/dist/cli/commands/importFileCommands.js +674 -0
  23. package/dist/cli/commands/schemaCommands.d.ts +7 -0
  24. package/dist/cli/commands/schemaCommands.js +169 -0
  25. package/dist/cli/commands/storageCommands.d.ts +5 -0
  26. package/dist/cli/commands/storageCommands.js +142 -0
  27. package/dist/cli/commands/transferCommands.d.ts +5 -0
  28. package/dist/cli/commands/transferCommands.js +382 -0
  29. package/dist/collections/columns.d.ts +13 -0
  30. package/dist/collections/columns.js +1339 -0
  31. package/dist/collections/indexes.d.ts +12 -0
  32. package/dist/collections/indexes.js +215 -0
  33. package/dist/collections/methods.d.ts +19 -0
  34. package/dist/collections/methods.js +605 -0
  35. package/dist/collections/tableOperations.d.ts +87 -0
  36. package/dist/collections/tableOperations.js +466 -0
  37. package/dist/collections/transferOperations.d.ts +8 -0
  38. package/dist/collections/transferOperations.js +411 -0
  39. package/dist/collections/wipeOperations.d.ts +17 -0
  40. package/dist/collections/wipeOperations.js +306 -0
  41. package/dist/databases/methods.d.ts +6 -0
  42. package/dist/databases/methods.js +35 -0
  43. package/dist/databases/setup.d.ts +5 -0
  44. package/dist/databases/setup.js +45 -0
  45. package/dist/examples/yamlTerminologyExample.d.ts +42 -0
  46. package/dist/examples/yamlTerminologyExample.js +272 -0
  47. package/dist/functions/deployments.d.ts +4 -0
  48. package/dist/functions/deployments.js +146 -0
  49. package/dist/functions/fnConfigDiscovery.d.ts +3 -0
  50. package/dist/functions/fnConfigDiscovery.js +108 -0
  51. package/dist/functions/methods.d.ts +16 -0
  52. package/dist/functions/methods.js +174 -0
  53. package/dist/init.d.ts +2 -0
  54. package/dist/init.js +57 -0
  55. package/dist/interactiveCLI.d.ts +36 -0
  56. package/dist/interactiveCLI.js +952 -0
  57. package/dist/main.d.ts +2 -0
  58. package/dist/main.js +1125 -0
  59. package/dist/migrations/afterImportActions.d.ts +17 -0
  60. package/dist/migrations/afterImportActions.js +305 -0
  61. package/dist/migrations/appwriteToX.d.ts +211 -0
  62. package/dist/migrations/appwriteToX.js +493 -0
  63. package/dist/migrations/comprehensiveTransfer.d.ts +147 -0
  64. package/dist/migrations/comprehensiveTransfer.js +1315 -0
  65. package/dist/migrations/dataLoader.d.ts +755 -0
  66. package/dist/migrations/dataLoader.js +1272 -0
  67. package/dist/migrations/importController.d.ts +25 -0
  68. package/dist/migrations/importController.js +283 -0
  69. package/dist/migrations/importDataActions.d.ts +50 -0
  70. package/dist/migrations/importDataActions.js +230 -0
  71. package/dist/migrations/relationships.d.ts +29 -0
  72. package/dist/migrations/relationships.js +203 -0
  73. package/dist/migrations/services/DataTransformationService.d.ts +55 -0
  74. package/dist/migrations/services/DataTransformationService.js +158 -0
  75. package/dist/migrations/services/FileHandlerService.d.ts +75 -0
  76. package/dist/migrations/services/FileHandlerService.js +236 -0
  77. package/dist/migrations/services/ImportOrchestrator.d.ts +99 -0
  78. package/dist/migrations/services/ImportOrchestrator.js +493 -0
  79. package/dist/migrations/services/RateLimitManager.d.ts +138 -0
  80. package/dist/migrations/services/RateLimitManager.js +279 -0
  81. package/dist/migrations/services/RelationshipResolver.d.ts +120 -0
  82. package/dist/migrations/services/RelationshipResolver.js +332 -0
  83. package/dist/migrations/services/UserMappingService.d.ts +109 -0
  84. package/dist/migrations/services/UserMappingService.js +277 -0
  85. package/dist/migrations/services/ValidationService.d.ts +74 -0
  86. package/dist/migrations/services/ValidationService.js +260 -0
  87. package/dist/migrations/transfer.d.ts +30 -0
  88. package/dist/migrations/transfer.js +661 -0
  89. package/dist/migrations/yaml/YamlImportConfigLoader.d.ts +131 -0
  90. package/dist/migrations/yaml/YamlImportConfigLoader.js +383 -0
  91. package/dist/migrations/yaml/YamlImportIntegration.d.ts +93 -0
  92. package/dist/migrations/yaml/YamlImportIntegration.js +341 -0
  93. package/dist/migrations/yaml/generateImportSchemas.d.ts +30 -0
  94. package/dist/migrations/yaml/generateImportSchemas.js +1327 -0
  95. package/dist/schemas/authUser.d.ts +24 -0
  96. package/dist/schemas/authUser.js +17 -0
  97. package/dist/setup.d.ts +2 -0
  98. package/{src/setup.ts → dist/setup.js} +0 -3
  99. package/dist/setupCommands.d.ts +58 -0
  100. package/dist/setupCommands.js +489 -0
  101. package/dist/setupController.d.ts +9 -0
  102. package/dist/setupController.js +34 -0
  103. package/dist/shared/backupMetadataSchema.d.ts +94 -0
  104. package/dist/shared/backupMetadataSchema.js +38 -0
  105. package/dist/shared/backupTracking.d.ts +18 -0
  106. package/dist/shared/backupTracking.js +176 -0
  107. package/dist/shared/confirmationDialogs.d.ts +75 -0
  108. package/dist/shared/confirmationDialogs.js +236 -0
  109. package/dist/shared/migrationHelpers.d.ts +61 -0
  110. package/dist/shared/migrationHelpers.js +145 -0
  111. package/{src/shared/operationLogger.ts → dist/shared/operationLogger.d.ts} +1 -11
  112. package/dist/shared/operationLogger.js +12 -0
  113. package/dist/shared/operationQueue.d.ts +40 -0
  114. package/dist/shared/operationQueue.js +310 -0
  115. package/dist/shared/operationsTable.d.ts +26 -0
  116. package/dist/shared/operationsTable.js +287 -0
  117. package/dist/shared/operationsTableSchema.d.ts +48 -0
  118. package/dist/shared/operationsTableSchema.js +35 -0
  119. package/dist/shared/progressManager.d.ts +62 -0
  120. package/dist/shared/progressManager.js +215 -0
  121. package/dist/shared/relationshipExtractor.d.ts +56 -0
  122. package/dist/shared/relationshipExtractor.js +138 -0
  123. package/dist/shared/selectionDialogs.d.ts +220 -0
  124. package/dist/shared/selectionDialogs.js +588 -0
  125. package/dist/storage/backupCompression.d.ts +20 -0
  126. package/dist/storage/backupCompression.js +67 -0
  127. package/dist/storage/methods.d.ts +44 -0
  128. package/dist/storage/methods.js +475 -0
  129. package/dist/storage/schemas.d.ts +842 -0
  130. package/dist/storage/schemas.js +175 -0
  131. package/dist/tables/indexManager.d.ts +65 -0
  132. package/dist/tables/indexManager.js +294 -0
  133. package/{src/types.ts → dist/types.d.ts} +1 -6
  134. package/dist/types.js +3 -0
  135. package/dist/users/methods.d.ts +16 -0
  136. package/dist/users/methods.js +276 -0
  137. package/dist/utils/configMigration.d.ts +1 -0
  138. package/dist/utils/configMigration.js +261 -0
  139. package/dist/utils/index.js +2 -0
  140. package/dist/utils/loadConfigs.d.ts +50 -0
  141. package/dist/utils/loadConfigs.js +357 -0
  142. package/dist/utils/setupFiles.d.ts +4 -0
  143. package/dist/utils/setupFiles.js +1190 -0
  144. package/dist/utilsController.d.ts +114 -0
  145. package/dist/utilsController.js +898 -0
  146. package/package.json +6 -3
  147. package/CHANGELOG.md +0 -35
  148. package/CONFIG_TODO.md +0 -1189
  149. package/SELECTION_DIALOGS.md +0 -146
  150. package/SERVICE_IMPLEMENTATION_REPORT.md +0 -462
  151. package/scripts/copy-templates.ts +0 -23
  152. package/src/backups/operations/bucketBackup.ts +0 -277
  153. package/src/backups/operations/collectionBackup.ts +0 -310
  154. package/src/backups/operations/comprehensiveBackup.ts +0 -342
  155. package/src/backups/schemas/bucketManifest.ts +0 -78
  156. package/src/backups/schemas/comprehensiveManifest.ts +0 -76
  157. package/src/backups/tracking/centralizedTracking.ts +0 -352
  158. package/src/cli/commands/configCommands.ts +0 -265
  159. package/src/cli/commands/databaseCommands.ts +0 -931
  160. package/src/cli/commands/functionCommands.ts +0 -419
  161. package/src/cli/commands/importFileCommands.ts +0 -815
  162. package/src/cli/commands/schemaCommands.ts +0 -200
  163. package/src/cli/commands/storageCommands.ts +0 -151
  164. package/src/cli/commands/transferCommands.ts +0 -454
  165. package/src/collections/attributes.ts.backup +0 -1555
  166. package/src/collections/columns.ts +0 -2025
  167. package/src/collections/indexes.ts +0 -350
  168. package/src/collections/methods.ts +0 -714
  169. package/src/collections/tableOperations.ts +0 -542
  170. package/src/collections/transferOperations.ts +0 -589
  171. package/src/collections/wipeOperations.ts +0 -449
  172. package/src/databases/methods.ts +0 -49
  173. package/src/databases/setup.ts +0 -77
  174. package/src/examples/yamlTerminologyExample.ts +0 -346
  175. package/src/functions/deployments.ts +0 -221
  176. package/src/functions/fnConfigDiscovery.ts +0 -103
  177. package/src/functions/methods.ts +0 -284
  178. package/src/init.ts +0 -62
  179. package/src/interactiveCLI.ts +0 -1201
  180. package/src/main.ts +0 -1517
  181. package/src/migrations/afterImportActions.ts +0 -579
  182. package/src/migrations/appwriteToX.ts +0 -668
  183. package/src/migrations/comprehensiveTransfer.ts +0 -2285
  184. package/src/migrations/dataLoader.ts +0 -1729
  185. package/src/migrations/importController.ts +0 -440
  186. package/src/migrations/importDataActions.ts +0 -315
  187. package/src/migrations/relationships.ts +0 -333
  188. package/src/migrations/services/DataTransformationService.ts +0 -196
  189. package/src/migrations/services/FileHandlerService.ts +0 -311
  190. package/src/migrations/services/ImportOrchestrator.ts +0 -675
  191. package/src/migrations/services/RateLimitManager.ts +0 -363
  192. package/src/migrations/services/RelationshipResolver.ts +0 -461
  193. package/src/migrations/services/UserMappingService.ts +0 -345
  194. package/src/migrations/services/ValidationService.ts +0 -349
  195. package/src/migrations/transfer.ts +0 -1113
  196. package/src/migrations/yaml/YamlImportConfigLoader.ts +0 -439
  197. package/src/migrations/yaml/YamlImportIntegration.ts +0 -446
  198. package/src/migrations/yaml/generateImportSchemas.ts +0 -1354
  199. package/src/schemas/authUser.ts +0 -23
  200. package/src/setupCommands.ts +0 -602
  201. package/src/setupController.ts +0 -43
  202. package/src/shared/backupMetadataSchema.ts +0 -93
  203. package/src/shared/backupTracking.ts +0 -211
  204. package/src/shared/confirmationDialogs.ts +0 -327
  205. package/src/shared/migrationHelpers.ts +0 -232
  206. package/src/shared/operationQueue.ts +0 -376
  207. package/src/shared/operationsTable.ts +0 -338
  208. package/src/shared/operationsTableSchema.ts +0 -60
  209. package/src/shared/progressManager.ts +0 -278
  210. package/src/shared/relationshipExtractor.ts +0 -214
  211. package/src/shared/selectionDialogs.ts +0 -802
  212. package/src/storage/backupCompression.ts +0 -88
  213. package/src/storage/methods.ts +0 -711
  214. package/src/storage/schemas.ts +0 -205
  215. package/src/tables/indexManager.ts +0 -409
  216. package/src/types/node-appwrite-tablesdb.d.ts +0 -44
  217. package/src/users/methods.ts +0 -358
  218. package/src/utils/configMigration.ts +0 -348
  219. package/src/utils/loadConfigs.ts +0 -457
  220. package/src/utils/setupFiles.ts +0 -1236
  221. package/src/utilsController.ts +0 -1263
  222. package/tests/README.md +0 -497
  223. package/tests/adapters/AdapterFactory.test.ts +0 -277
  224. package/tests/integration/syncOperations.test.ts +0 -463
  225. package/tests/jest.config.js +0 -25
  226. package/tests/migration/configMigration.test.ts +0 -546
  227. package/tests/setup.ts +0 -62
  228. package/tests/testUtils.ts +0 -340
  229. package/tests/utils/loadConfigs.test.ts +0 -350
  230. package/tests/validation/configValidation.test.ts +0 -412
  231. package/tsconfig.json +0 -44
  232. /package/{src → dist}/functions/templates/count-docs-in-collection/README.md +0 -0
  233. /package/{src → dist}/functions/templates/count-docs-in-collection/src/main.ts +0 -0
  234. /package/{src → dist}/functions/templates/count-docs-in-collection/src/request.ts +0 -0
  235. /package/{src → dist}/functions/templates/hono-typescript/README.md +0 -0
  236. /package/{src → dist}/functions/templates/hono-typescript/src/adapters/request.ts +0 -0
  237. /package/{src → dist}/functions/templates/hono-typescript/src/adapters/response.ts +0 -0
  238. /package/{src → dist}/functions/templates/hono-typescript/src/app.ts +0 -0
  239. /package/{src → dist}/functions/templates/hono-typescript/src/context.ts +0 -0
  240. /package/{src → dist}/functions/templates/hono-typescript/src/main.ts +0 -0
  241. /package/{src → dist}/functions/templates/hono-typescript/src/middleware/appwrite.ts +0 -0
  242. /package/{src → dist}/functions/templates/typescript-node/README.md +0 -0
  243. /package/{src → dist}/functions/templates/typescript-node/src/context.ts +0 -0
  244. /package/{src → dist}/functions/templates/typescript-node/src/main.ts +0 -0
  245. /package/{src → dist}/functions/templates/uv/README.md +0 -0
  246. /package/{src → dist}/functions/templates/uv/pyproject.toml +0 -0
  247. /package/{src → dist}/functions/templates/uv/src/__init__.py +0 -0
  248. /package/{src → dist}/functions/templates/uv/src/context.py +0 -0
  249. /package/{src → dist}/functions/templates/uv/src/main.py +0 -0
  250. /package/{src/utils/index.ts → dist/utils/index.d.ts} +0 -0
@@ -0,0 +1,61 @@
1
+ import { type Databases } from "node-appwrite";
2
+ import { z } from "zod";
3
+ import type { DatabaseAdapter } from "appwrite-utils-helpers";
4
+ /**
5
+ * Object that contains the context for an action that needs to be executed after import
6
+ * Used in the afterImportActionsDefinitions
7
+ * @type {ContextObject}
8
+ * @typedef {Object} ContextObject
9
+ * @property {string} collectionId - The ID of the collection
10
+ * @property {any} finalItem - The final item that was imported
11
+ * @property {string} action - The name of the action
12
+ * @property {string[]} params - The parameters for the action
13
+ * @property {Object} context - The context object for the action (all the data of this specific item)
14
+ */
15
+ export declare const ContextObject: z.ZodObject<{
16
+ dbId: z.ZodString;
17
+ collectionId: z.ZodString;
18
+ finalItem: z.ZodAny;
19
+ attributeMappings: z.ZodArray<z.ZodObject<{
20
+ oldKey: z.ZodOptional<z.ZodString>;
21
+ oldKeys: z.ZodOptional<z.ZodArray<z.ZodString>>;
22
+ targetKey: z.ZodString;
23
+ valueToSet: z.ZodOptional<z.ZodAny>;
24
+ fileData: z.ZodOptional<z.ZodObject<{
25
+ name: z.ZodString;
26
+ path: z.ZodString;
27
+ }, z.core.$strip>>;
28
+ converters: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodString>>>;
29
+ validationActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
30
+ action: z.ZodString;
31
+ params: z.ZodArray<z.ZodString>;
32
+ }, z.core.$strip>>>>;
33
+ postImportActions: z.ZodOptional<z.ZodDefault<z.ZodArray<z.ZodObject<{
34
+ action: z.ZodString;
35
+ params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>>;
36
+ }, z.core.$strip>>>>;
37
+ }, z.core.$strip>>;
38
+ context: z.ZodAny;
39
+ }, z.core.$strip>;
40
+ export type ContextObject = z.infer<typeof ContextObject>;
41
+ export declare const createOrFindAfterImportOperation: (database: Databases, collectionId: string, context: ContextObject) => Promise<void>;
42
+ export declare const addBatch: (database: Databases, data: string) => Promise<string>;
43
+ export declare const getAfterImportOperations: (database: Databases, collectionId: string) => Promise<{
44
+ $id: string;
45
+ $createdAt: string;
46
+ $updatedAt: string;
47
+ operationType: string;
48
+ collectionId: string;
49
+ data: any;
50
+ progress: number;
51
+ total: number;
52
+ error: string;
53
+ status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
54
+ batches?: string[] | undefined;
55
+ }[]>;
56
+ export declare const findOrCreateOperation: (db: DatabaseAdapter, databaseId: string, operationType: string, collectionId?: string, data?: any) => Promise<any>;
57
+ export declare const updateOperation: (db: DatabaseAdapter, databaseId: string, operationId: string, updates: any) => Promise<any>;
58
+ export declare const getOperation: (db: DatabaseAdapter, databaseId: string, operationId: string) => Promise<any>;
59
+ export declare const maxDataLength = 1073741820;
60
+ export declare const maxBatchItems = 25;
61
+ export declare const splitIntoBatches: (data: any[]) => any[][];
@@ -0,0 +1,145 @@
1
+ import { ID, Query } from "node-appwrite";
2
+ import { BatchSchema, OperationSchema } from "../storage/schemas.js";
3
+ import { AttributeMappingsSchema } from "appwrite-utils";
4
+ import { z } from "zod";
5
+ import { logger } from 'appwrite-utils-helpers';
6
+ import { tryAwaitWithRetry } from "appwrite-utils-helpers";
7
+ import { findOrCreateOperation as findOrCreateOp, updateOperation as updateOp, getOperation as getOp } from "./operationsTable.js";
8
+ import { MessageFormatter } from 'appwrite-utils-helpers';
9
+ /**
10
+ * Object that contains the context for an action that needs to be executed after import
11
+ * Used in the afterImportActionsDefinitions
12
+ * @type {ContextObject}
13
+ * @typedef {Object} ContextObject
14
+ * @property {string} collectionId - The ID of the collection
15
+ * @property {any} finalItem - The final item that was imported
16
+ * @property {string} action - The name of the action
17
+ * @property {string[]} params - The parameters for the action
18
+ * @property {Object} context - The context object for the action (all the data of this specific item)
19
+ */
20
+ export const ContextObject = z.object({
21
+ dbId: z.string(),
22
+ collectionId: z.string(),
23
+ finalItem: z.any(),
24
+ attributeMappings: AttributeMappingsSchema,
25
+ context: z.any(),
26
+ });
27
+ export const createOrFindAfterImportOperation = async (database, collectionId, context) => {
28
+ let operation = await findOrCreateOperationLegacy(database, collectionId, "afterImportAction");
29
+ if (!operation.batches) {
30
+ operation.batches = [];
31
+ }
32
+ // Directly create a new batch for the context without checking for an existing batch
33
+ const contextData = JSON.stringify(context);
34
+ // Create a new batch with the contextData
35
+ const newBatchId = await addBatch(database, contextData);
36
+ // Update the operation with the new batch's $id
37
+ operation.batches = [...operation.batches, newBatchId];
38
+ await database.updateDocument("migrations", "currentOperations", operation.$id, { batches: operation.batches });
39
+ };
40
+ export const addBatch = async (database, data) => {
41
+ const batch = await database.createDocument("migrations", "batches", ID.unique(), {
42
+ data,
43
+ processed: false,
44
+ });
45
+ return batch.$id;
46
+ };
47
+ export const getAfterImportOperations = async (database, collectionId) => {
48
+ let lastDocumentId;
49
+ const allOperations = [];
50
+ let total = 0;
51
+ do {
52
+ const query = [
53
+ Query.equal("collectionId", collectionId),
54
+ Query.equal("operationType", "afterImportAction"),
55
+ Query.limit(100),
56
+ ];
57
+ if (lastDocumentId) {
58
+ query.push(Query.cursorAfter(lastDocumentId));
59
+ }
60
+ const operations = await database.listDocuments("migrations", "currentOperations", query);
61
+ total = operations.total; // Update total with the latest fetch
62
+ allOperations.push(...operations.documents);
63
+ if (operations.documents.length > 0 && operations.documents.length >= 100) {
64
+ lastDocumentId =
65
+ operations.documents[operations.documents.length - 1].$id;
66
+ }
67
+ } while (allOperations.length < total);
68
+ const allOps = allOperations.map((op) => OperationSchema.parse(op));
69
+ return allOps;
70
+ };
71
+ // Legacy function for backward compatibility with old migrations database
72
+ const findOrCreateOperationLegacy = async (database, collectionId, operationType, additionalQueries) => {
73
+ const operations = await tryAwaitWithRetry(async () => await database.listDocuments("migrations", "currentOperations", [
74
+ Query.equal("collectionId", collectionId),
75
+ Query.equal("operationType", operationType),
76
+ Query.equal("status", "pending"),
77
+ ...(additionalQueries || []),
78
+ ]));
79
+ if (operations.documents.length > 0) {
80
+ return OperationSchema.parse(operations.documents[0]);
81
+ }
82
+ else {
83
+ const op = await tryAwaitWithRetry(async () => await database.createDocument("migrations", "currentOperations", ID.unique(), {
84
+ operationType,
85
+ collectionId,
86
+ status: "pending",
87
+ batches: [],
88
+ progress: 0,
89
+ total: 0,
90
+ error: "",
91
+ }));
92
+ return OperationSchema.parse(op);
93
+ }
94
+ };
95
+ export const findOrCreateOperation = async (db, databaseId, operationType, collectionId, data) => {
96
+ // Use new operations table system
97
+ return await findOrCreateOp(db, databaseId, operationType, {
98
+ targetTable: collectionId,
99
+ data: data
100
+ });
101
+ };
102
+ export const updateOperation = async (db, databaseId, operationId, updates) => {
103
+ // Use new operations table system
104
+ return await updateOp(db, databaseId, operationId, updates);
105
+ };
106
+ export const getOperation = async (db, databaseId, operationId) => {
107
+ // Use new operations table system
108
+ return await getOp(db, databaseId, operationId);
109
+ };
110
+ // Actual max 1073741824
111
+ export const maxDataLength = 1073741820;
112
+ export const maxBatchItems = 25;
113
+ export const splitIntoBatches = (data) => {
114
+ let batches = [];
115
+ let currentBatch = [];
116
+ let currentBatchLength = 0;
117
+ let currentBatchItemCount = 0;
118
+ data.forEach((item, index) => {
119
+ const itemLength = JSON.stringify(item).length;
120
+ if (itemLength > maxDataLength) {
121
+ MessageFormatter.warning(`Large item found at index ${index} with length ${itemLength}`, { prefix: "Batch Splitter" });
122
+ logger.debug("Large item data:", item);
123
+ }
124
+ // Check if adding the current item would exceed the max length or max items per batch
125
+ if (currentBatchLength + itemLength >= maxDataLength ||
126
+ currentBatchItemCount >= maxBatchItems) {
127
+ // If so, start a new batch
128
+ batches.push(currentBatch);
129
+ currentBatch = [item];
130
+ currentBatchLength = itemLength;
131
+ currentBatchItemCount = 1; // Reset item count for the new batch
132
+ }
133
+ else {
134
+ // Otherwise, add the item to the current batch
135
+ currentBatch.push(item);
136
+ currentBatchLength += itemLength;
137
+ currentBatchItemCount++;
138
+ }
139
+ });
140
+ // Don't forget to add the last batch if it's not empty
141
+ if (currentBatch.length > 0) {
142
+ batches.push(currentBatch);
143
+ }
144
+ return batches;
145
+ };
@@ -1,6 +1,5 @@
1
1
  import type { Databases, Models } from "node-appwrite";
2
2
  import type { OperationCreate } from "../storage/schemas.js";
3
-
4
3
  /**
5
4
  * Legacy operation logger - deprecated
6
5
  * This function is maintained for backward compatibility but no longer performs any logging.
@@ -8,13 +7,4 @@ import type { OperationCreate } from "../storage/schemas.js";
8
7
  *
9
8
  * @deprecated This function will be removed in a future version
10
9
  */
11
- export const logOperation = async (
12
- db: Databases,
13
- dbId: string,
14
- operationDetails: OperationCreate,
15
- operationId?: string
16
- ): Promise<Models.Document | null> => {
17
- // No-op: Operations logging has been moved to the new operations table system
18
- // Callers should migrate to using operationsTable.ts functions directly
19
- return null;
20
- };
10
+ export declare const logOperation: (db: Databases, dbId: string, operationDetails: OperationCreate, operationId?: string) => Promise<Models.Document | null>;
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Legacy operation logger - deprecated
3
+ * This function is maintained for backward compatibility but no longer performs any logging.
4
+ * The operations table system has been refactored to use the dynamic adapter pattern.
5
+ *
6
+ * @deprecated This function will be removed in a future version
7
+ */
8
+ export const logOperation = async (db, dbId, operationDetails, operationId) => {
9
+ // No-op: Operations logging has been moved to the new operations table system
10
+ // Callers should migrate to using operationsTable.ts functions directly
11
+ return null;
12
+ };
@@ -0,0 +1,40 @@
1
+ import { type Databases, type Models } from "node-appwrite";
2
+ import type { Attribute } from "appwrite-utils";
3
+ import type { DatabaseAdapter } from "appwrite-utils-helpers";
4
+ export interface QueuedOperation {
5
+ type: "attribute";
6
+ collectionId?: string;
7
+ attribute?: Attribute;
8
+ collection?: Models.Collection;
9
+ dependencies?: string[];
10
+ }
11
+ export declare const queuedOperations: QueuedOperation[];
12
+ export declare const nameToIdMapping: Map<string, string>;
13
+ export declare const processedCollections: Set<string>;
14
+ export declare const processedAttributes: Set<string>;
15
+ export declare const enqueueOperation: (operation: QueuedOperation) => void;
16
+ /**
17
+ * Clear all caches and processing state - use between operations
18
+ */
19
+ export declare const clearProcessingState: () => void;
20
+ /**
21
+ * Check if a collection has already been fully processed
22
+ */
23
+ export declare const isCollectionProcessed: (collectionId: string, databaseId: string) => boolean;
24
+ /**
25
+ * Mark a collection as fully processed
26
+ */
27
+ export declare const markCollectionProcessed: (collectionId: string, collectionName: string | undefined, databaseId: string) => void;
28
+ /**
29
+ * Check if a specific attribute has been processed
30
+ */
31
+ export declare const isAttributeProcessed: (databaseId: string, collectionId: string, attributeKey: string) => boolean;
32
+ /**
33
+ * Mark a specific attribute as processed
34
+ */
35
+ export declare const markAttributeProcessed: (databaseId: string, collectionId: string, attributeKey: string) => void;
36
+ /**
37
+ * Process only specific attributes in the queue, not entire collections
38
+ * This prevents triggering full collection re-processing cycles
39
+ */
40
+ export declare const processQueue: (db: Databases | DatabaseAdapter, dbId: string) => Promise<void>;
@@ -0,0 +1,310 @@
1
+ import { Query } from "node-appwrite";
2
+ import { createOrUpdateAttributeWithStatusCheck } from "../collections/columns.js";
3
+ import { fetchAndCacheCollectionByName } from "../collections/methods.js";
4
+ import { tryAwaitWithRetry } from "appwrite-utils-helpers";
5
+ import { logger, MessageFormatter } from "appwrite-utils-helpers";
6
+ // Global state management
7
+ export const queuedOperations = [];
8
+ export const nameToIdMapping = new Map();
9
+ // Keys are scoped per database to avoid cross-database collisions
10
+ // Collections key format: `${databaseId}::${collectionId}`
11
+ // Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
12
+ export const processedCollections = new Set();
13
+ export const processedAttributes = new Set();
14
+ // Helpers to build scoped keys
15
+ const collectionKey = (databaseId, collectionId) => `${databaseId}::${collectionId}`;
16
+ const attributeKeyScoped = (databaseId, collectionId, key) => `${databaseId}::${collectionId}::${key}`;
17
+ export const enqueueOperation = (operation) => {
18
+ // Avoid duplicate queue entries for same attribute
19
+ const attributeKey = operation.attribute?.key;
20
+ const collectionId = operation.collectionId;
21
+ logger.info('Enqueueing operation', {
22
+ type: operation.type,
23
+ attributeKey,
24
+ collectionId,
25
+ dependencies: operation.dependencies,
26
+ queueSizeBefore: queuedOperations.length,
27
+ operation: 'enqueueOperation'
28
+ });
29
+ if (attributeKey && collectionId) {
30
+ const duplicateIndex = queuedOperations.findIndex((op) => op.collectionId === collectionId && op.attribute?.key === attributeKey);
31
+ if (duplicateIndex !== -1) {
32
+ MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
33
+ logger.info('Replacing duplicate queue entry', {
34
+ attributeKey,
35
+ collectionId,
36
+ duplicateIndex,
37
+ operation: 'enqueueOperation'
38
+ });
39
+ queuedOperations[duplicateIndex] = operation;
40
+ return;
41
+ }
42
+ }
43
+ queuedOperations.push(operation);
44
+ logger.debug('Operation enqueued successfully', {
45
+ attributeKey,
46
+ collectionId,
47
+ queueSizeAfter: queuedOperations.length,
48
+ operation: 'enqueueOperation'
49
+ });
50
+ };
51
+ /**
52
+ * Clear all caches and processing state - use between operations
53
+ */
54
+ export const clearProcessingState = () => {
55
+ const sizeBefore = {
56
+ collections: processedCollections.size,
57
+ attributes: processedAttributes.size,
58
+ nameMapping: nameToIdMapping.size
59
+ };
60
+ processedCollections.clear();
61
+ processedAttributes.clear();
62
+ nameToIdMapping.clear();
63
+ logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
64
+ logger.info('Processing state cleared', {
65
+ sizeBefore,
66
+ operation: 'clearProcessingState'
67
+ });
68
+ };
69
+ /**
70
+ * Check if a collection has already been fully processed
71
+ */
72
+ export const isCollectionProcessed = (collectionId, databaseId) => {
73
+ return processedCollections.has(collectionKey(databaseId, collectionId));
74
+ };
75
+ /**
76
+ * Mark a collection as fully processed
77
+ */
78
+ export const markCollectionProcessed = (collectionId, collectionName, databaseId) => {
79
+ processedCollections.add(collectionKey(databaseId, collectionId));
80
+ const logData = {
81
+ databaseId,
82
+ collectionId,
83
+ collectionName,
84
+ totalProcessedCollections: processedCollections.size,
85
+ operation: 'markCollectionProcessed'
86
+ };
87
+ if (collectionName) {
88
+ MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
89
+ }
90
+ logger.info('Collection marked as processed', logData);
91
+ };
92
+ /**
93
+ * Check if a specific attribute has been processed
94
+ */
95
+ export const isAttributeProcessed = (databaseId, collectionId, attributeKey) => {
96
+ return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
97
+ };
98
+ /**
99
+ * Mark a specific attribute as processed
100
+ */
101
+ export const markAttributeProcessed = (databaseId, collectionId, attributeKey) => {
102
+ const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
103
+ processedAttributes.add(identifier);
104
+ logger.debug('Attribute marked as processed', {
105
+ databaseId,
106
+ collectionId,
107
+ attributeKey,
108
+ identifier,
109
+ totalProcessedAttributes: processedAttributes.size,
110
+ operation: 'markAttributeProcessed'
111
+ });
112
+ };
113
+ /**
114
+ * Process only specific attributes in the queue, not entire collections
115
+ * This prevents triggering full collection re-processing cycles
116
+ */
117
+ export const processQueue = async (db, dbId) => {
118
+ const startTime = Date.now();
119
+ if (queuedOperations.length === 0) {
120
+ MessageFormatter.info("No queued operations to process");
121
+ logger.info('Queue processing skipped - no operations', {
122
+ dbId,
123
+ operation: 'processQueue'
124
+ });
125
+ return;
126
+ }
127
+ MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
128
+ logger.info('Starting queue processing', {
129
+ dbId,
130
+ queueSize: queuedOperations.length,
131
+ operations: queuedOperations.map(op => ({
132
+ type: op.type,
133
+ attributeKey: op.attribute?.key,
134
+ collectionId: op.collectionId,
135
+ dependencies: op.dependencies
136
+ })),
137
+ operation: 'processQueue'
138
+ });
139
+ let progress = true;
140
+ let attempts = 0;
141
+ const maxAttempts = 3; // Prevent infinite loops
142
+ while (progress && attempts < maxAttempts) {
143
+ progress = false;
144
+ attempts++;
145
+ MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
146
+ logger.info('Queue processing attempt started', {
147
+ attempt: attempts,
148
+ maxAttempts,
149
+ remainingOperations: queuedOperations.length,
150
+ dbId,
151
+ operation: 'processQueue'
152
+ });
153
+ for (let i = queuedOperations.length - 1; i >= 0; i--) {
154
+ const operation = queuedOperations[i];
155
+ if (!operation.attribute || !operation.collectionId) {
156
+ MessageFormatter.warning("Invalid operation, removing from queue");
157
+ queuedOperations.splice(i, 1);
158
+ continue;
159
+ }
160
+ const attributeKey = operation.attribute.key;
161
+ const collectionId = operation.collectionId;
162
+ // Skip if this specific attribute was already processed (per database)
163
+ if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
164
+ MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
165
+ logger.debug('Removing already processed attribute from queue', {
166
+ attributeKey,
167
+ collectionId,
168
+ queueIndex: i,
169
+ operation: 'processQueue'
170
+ });
171
+ queuedOperations.splice(i, 1);
172
+ continue;
173
+ }
174
+ let targetCollection;
175
+ // Resolve the target collection (where the attribute will be created)
176
+ try {
177
+ targetCollection = await tryAwaitWithRetry(async () => {
178
+ if ('getMetadata' in db && typeof db.getMetadata === 'function') {
179
+ // DatabaseAdapter
180
+ return (await db.getTable({ databaseId: dbId, tableId: collectionId })).data;
181
+ }
182
+ else {
183
+ // Legacy Databases
184
+ return await db.getCollection(dbId, collectionId);
185
+ }
186
+ });
187
+ }
188
+ catch (e) {
189
+ const errorMessage = e instanceof Error ? e.message : String(e);
190
+ MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
191
+ logger.error('Target collection not found during queue processing', {
192
+ collectionId,
193
+ attributeKey,
194
+ error: errorMessage,
195
+ operation: 'processQueue'
196
+ });
197
+ queuedOperations.splice(i, 1);
198
+ continue;
199
+ }
200
+ // For relationship attributes, ensure the related collection exists
201
+ let canProcess = true;
202
+ if (operation.attribute.type === "relationship") {
203
+ const relatedCollection = operation.attribute.relatedCollection;
204
+ if (relatedCollection) {
205
+ // Try to resolve related collection by ID first, then by name
206
+ let relatedFound = false;
207
+ try {
208
+ await tryAwaitWithRetry(async () => {
209
+ if ('getMetadata' in db && typeof db.getMetadata === 'function') {
210
+ // DatabaseAdapter
211
+ return (await db.getTable({ databaseId: dbId, tableId: relatedCollection })).data;
212
+ }
213
+ else {
214
+ // Legacy Databases
215
+ return await db.getCollection(dbId, relatedCollection);
216
+ }
217
+ });
218
+ relatedFound = true;
219
+ nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
220
+ }
221
+ catch (_) {
222
+ // Try by name lookup
223
+ const cachedId = nameToIdMapping.get(relatedCollection);
224
+ if (cachedId) {
225
+ try {
226
+ await tryAwaitWithRetry(async () => {
227
+ if ('getMetadata' in db && typeof db.getMetadata === 'function') {
228
+ // DatabaseAdapter
229
+ return (await db.getTable({ databaseId: dbId, tableId: cachedId })).data;
230
+ }
231
+ else {
232
+ // Legacy Databases
233
+ return await db.getCollection(dbId, cachedId);
234
+ }
235
+ });
236
+ relatedFound = true;
237
+ }
238
+ catch (_) {
239
+ nameToIdMapping.delete(relatedCollection); // Remove stale cache
240
+ }
241
+ }
242
+ if (!relatedFound) {
243
+ // Final attempt: search by name
244
+ try {
245
+ const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
246
+ ? await db.listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
247
+ : await db.listCollections(dbId, [Query.equal("name", relatedCollection)]);
248
+ if (collections.total && collections.total > 0) {
249
+ const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
250
+ ? collections.tables?.[0]
251
+ : collections.collections?.[0];
252
+ nameToIdMapping.set(relatedCollection, firstCollection.$id);
253
+ relatedFound = true;
254
+ }
255
+ }
256
+ catch (_) {
257
+ // Related collection truly doesn't exist yet
258
+ }
259
+ }
260
+ }
261
+ if (!relatedFound) {
262
+ MessageFormatter.warning(`Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`);
263
+ canProcess = false;
264
+ }
265
+ }
266
+ }
267
+ if (canProcess && targetCollection) {
268
+ MessageFormatter.progress(`Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`);
269
+ const success = await createOrUpdateAttributeWithStatusCheck(db, dbId, targetCollection, operation.attribute);
270
+ if (success) {
271
+ MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
272
+ logger.info('Queued attribute processed successfully', {
273
+ attributeKey,
274
+ collectionId,
275
+ targetCollectionName: targetCollection.name,
276
+ operation: 'processQueue'
277
+ });
278
+ markAttributeProcessed(dbId, collectionId, attributeKey);
279
+ queuedOperations.splice(i, 1);
280
+ progress = true;
281
+ }
282
+ else {
283
+ MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
284
+ logger.error('Failed to process queued attribute', {
285
+ attributeKey,
286
+ collectionId,
287
+ targetCollectionName: targetCollection.name,
288
+ operation: 'processQueue'
289
+ });
290
+ queuedOperations.splice(i, 1);
291
+ }
292
+ }
293
+ }
294
+ if (queuedOperations.length === 0) {
295
+ break;
296
+ }
297
+ MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
298
+ }
299
+ if (queuedOperations.length > 0) {
300
+ MessageFormatter.warning(`${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`);
301
+ queuedOperations.forEach((op, index) => {
302
+ MessageFormatter.warning(` ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`);
303
+ });
304
+ MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
305
+ }
306
+ else {
307
+ MessageFormatter.success("All queued operations processed successfully");
308
+ }
309
+ MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
310
+ };
@@ -0,0 +1,26 @@
1
+ import type { DatabaseAdapter } from "appwrite-utils-helpers";
2
+ import { type OperationRecord } from "./operationsTableSchema.js";
3
+ /**
4
+ * Creates the operations tracking table in the specified database
5
+ * Table is created with underscore prefix to indicate it's a system table
6
+ */
7
+ export declare function createOperationsTable(db: DatabaseAdapter, databaseId: string): Promise<void>;
8
+ /**
9
+ * Finds an existing operation or creates a new one
10
+ * Useful for resuming interrupted operations
11
+ */
12
+ export declare function findOrCreateOperation(db: DatabaseAdapter, databaseId: string, operationType: string, params?: Partial<OperationRecord>): Promise<OperationRecord>;
13
+ /**
14
+ * Updates an existing operation record
15
+ */
16
+ export declare function updateOperation(db: DatabaseAdapter, databaseId: string, operationId: string, updates: Partial<OperationRecord>): Promise<OperationRecord>;
17
+ /**
18
+ * Gets a single operation by ID
19
+ */
20
+ export declare function getOperation(db: DatabaseAdapter, databaseId: string, operationId: string): Promise<OperationRecord | null>;
21
+ /**
22
+ * Cleans up old completed operations
23
+ * @param olderThan - Optional date, defaults to operations older than 7 days
24
+ * @returns Number of operations deleted
25
+ */
26
+ export declare function cleanupOperations(db: DatabaseAdapter, databaseId: string, olderThan?: Date): Promise<number>;