appwrite-utils-cli 0.0.1

package/src/migrations/storage.ts

@@ -0,0 +1,351 @@
+import {
+  Storage,
+  Databases,
+  Query,
+  InputFile,
+  type Models,
+  ID,
+} from "node-appwrite";
+import { type OperationCreate, type BackupCreate } from "./backup.js";
+import { splitIntoBatches } from "./migrationHelper.js";
+import type { AppwriteConfig } from "./schema.js";
+
+export const logOperation = async (
+  db: Databases,
+  dbId: string,
+  operationDetails: OperationCreate,
+  operationId?: string
+): Promise<Models.Document> => {
+  try {
+    let operation;
+    if (operationId) {
+      // Update existing operation log
+      operation = await db.updateDocument(
+        "migrations",
+        "currentOperations",
+        operationId,
+        operationDetails
+      );
+    } else {
+      // Create new operation log
+      operation = await db.createDocument(
+        "migrations",
+        "currentOperations",
+        ID.unique(),
+        operationDetails
+      );
+    }
+    console.log(`Operation logged: ${operation.$id}`);
+    return operation;
+  } catch (error) {
+    console.error(`Error logging operation: ${error}`);
+    throw error;
+  }
+};
+
+export const initOrGetBackupStorage = async (storage: Storage) => {
+  try {
+    const backupStorage = await storage.getBucket("backupStorage");
+    return backupStorage;
+  } catch (e) {
+    // ID backupStorage
+    // Name Backups Storage
+    const backupStorage = await storage.createBucket(
+      "backupStorage",
+      "Backups Storage"
+    );
+    return backupStorage;
+  }
+};
+
+export const initOrGetDocumentStorage = async (
+  storage: Storage,
+  config: AppwriteConfig,
+  dbName: string
+) => {
+  try {
+    await storage.getBucket(
+      `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`
+    );
+  } catch (e) {
+    // ID documentStorage
+    // Name Document Storage
+    const documentStorage = await storage.createBucket(
+      `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`,
+      "Document Storage"
+    );
+    return documentStorage;
+  }
+};
+
+export const wipeDocumentStorage = async (
+  storage: Storage,
+  config: AppwriteConfig,
+  dbName: string
+): Promise<void> => {
+  const bucketId = `${config.documentBucketId
+    .toLowerCase()
+    .replace(" ", "")}_${dbName.toLowerCase().replace(" ", "")}`;
+  console.log(`Wiping storage for bucket ID: ${bucketId}`);
+  let moreFiles = true;
+  let lastFileId: string | undefined;
+  const allFiles: string[] = [];
+  while (moreFiles) {
+    const queries = [Query.limit(100)]; // Adjust the limit as needed
+    if (lastFileId) {
+      queries.push(Query.cursorAfter(lastFileId));
+    }
+    const filesPulled = await storage.listFiles(bucketId, queries);
+    if (filesPulled.files.length === 0) {
+      console.log("No files found, done!");
+      moreFiles = false;
+      break;
+    } else if (filesPulled.files.length > 0) {
+      const fileIds = filesPulled.files.map((file) => file.$id);
+      allFiles.push(...fileIds);
+    }
+    moreFiles = filesPulled.files.length > 100; // Adjust based on the limit
+    if (moreFiles) {
+      lastFileId = filesPulled.files[filesPulled.files.length - 1].$id;
+    }
+  }
+
+  for (const fileId of allFiles) {
+    console.log(`Deleting file: ${fileId}`);
+    await storage.deleteFile(bucketId, fileId);
+  }
+  console.log(`All files in bucket ${bucketId} have been deleted.`);
+};
+
+async function retryFailedPromises(
+  batch: Promise<Models.Document>[],
+  maxRetries = 3
+): Promise<PromiseSettledResult<Models.Document>[]> {
+  const results = await Promise.allSettled(batch);
+  const toRetry: Promise<any>[] = [];
+
+  results.forEach((result, index) => {
+    if (result.status === "rejected") {
+      console.error("Promise rejected with reason:", result.reason);
+      if (maxRetries > 0) {
+        toRetry.push(batch[index]);
+      }
+    }
+  });
+
+  if (toRetry.length > 0) {
+    console.log(`Retrying ${toRetry.length} promises`);
+    return retryFailedPromises(toRetry, maxRetries - 1);
+  } else {
+    return results
+      .filter((result) => result.status === "fulfilled")
+      .map((result) => result);
+  }
+}
+
+export const backupDatabase = async (
+  database: Databases,
+  databaseId: string,
+  storage: Storage
+): Promise<void> => {
+  console.log("---------------------------------");
+  console.log("Starting Database Backup of " + databaseId);
+  console.log("---------------------------------");
+  let data: BackupCreate = {
+    database: "",
+    collections: [],
+    documents: [],
+  };
+
+  const backupOperation = await logOperation(database, databaseId, {
+    operationType: "backup",
+    collectionId: "",
+    data: "Starting backup...",
+    progress: 0,
+    total: 100, // This will be dynamically updated later
+    error: "",
+    status: "in_progress",
+  });
+
+  // Fetch and backup the database details
+  let db: Models.Database;
+  try {
+    db = await database.get(databaseId);
+  } catch (e) {
+    console.error(`Error fetching database: ${e}`);
+    await logOperation(
+      database,
+      databaseId,
+      {
+        operationType: "backup",
+        collectionId: "",
+        data: "Error fetching database, skipping...",
+        progress: 0,
+        total: 100, // This will be dynamically updated later
+        error: `Error fetching database: ${e}`,
+        status: "error",
+      },
+      backupOperation.$id
+    );
+    return;
+  }
+  data.database = JSON.stringify(db);
+
+  // Initialize pagination for collections
+  let lastCollectionId = "";
+  let moreCollections = true;
+  let progress = 0;
+  let total = 0; // Initialize total to 0, will be updated dynamically
+
+  while (moreCollections) {
+    const collectionResponse = await database.listCollections(databaseId, [
+      Query.limit(500), // Adjust the limit as needed
+      ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
+    ]);
+
+    total += collectionResponse.collections.length; // Update total with number of collections
+
+    for (const {
+      $id: collectionId,
+      name: collectionName,
+    } of collectionResponse.collections) {
+      let collectionDocumentCount = 0; // Initialize document count for the current collection
+      try {
+        const collection = await database.getCollection(
+          databaseId,
+          collectionId
+        );
+        progress++;
+        data.collections.push(JSON.stringify(collection));
+
+        // Initialize pagination for documents within the current collection
+        let lastDocumentId = "";
+        let moreDocuments = true;
+
+        while (moreDocuments) {
+          const documentResponse = await database.listDocuments(
+            databaseId,
+            collectionId,
+            [
+              Query.limit(500), // Adjust the limit as needed
+              ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
+            ]
+          );
+
+          total += documentResponse.documents.length; // Update total with number of documents
+          collectionDocumentCount += documentResponse.documents.length; // Update document count for the current collection
+          let documentPromises: Promise<Models.Document>[] = [];
+          for (const { $id: documentId } of documentResponse.documents) {
+            documentPromises.push(
+              database.getDocument(databaseId, collectionId, documentId)
+            );
+          }
+          const promiseBatches = splitIntoBatches(documentPromises);
+          const documentsPulled = [];
+          for (const batch of promiseBatches) {
+            const successfulDocuments = await retryFailedPromises(batch);
+            documentsPulled.push(...successfulDocuments);
+          }
+          const documents = documentsPulled;
+          data.documents.push({
+            collectionId: collectionId,
+            data: JSON.stringify(documents),
+          });
+          progress += documents.length;
+
+          console.log(
+            `Collection ${collectionName} backed up ${collectionDocumentCount} documents (so far)`
+          );
+
+          // Update the operation log with the current progress
+          await logOperation(
+            database,
+            databaseId,
+            {
+              operationType: "backup",
+              collectionId: collectionId,
+              data: `Still backing up, ${data.collections.length} collections so far`,
+              progress: progress,
+              total: total,
+              error: "",
+              status: "in_progress",
+            },
+            backupOperation.$id
+          );
+
+          // Check if there are more documents to fetch
+          moreDocuments = documentResponse.documents.length === 500;
+          if (moreDocuments) {
+            lastDocumentId =
+              documentResponse.documents[documentResponse.documents.length - 1]
+                .$id;
+          }
+        }
+        console.log(
+          `Collection ${collectionName} backed up with ${collectionDocumentCount} documents.`
+        );
+      } catch (error) {
+        console.log(
+          `Collection ${collectionName} must not exist, continuing...`
+        );
+        continue;
+      }
+    }
+
+    // Check if there are more collections to fetch
+    moreCollections = collectionResponse.collections.length === 500;
+    if (moreCollections) {
+      lastCollectionId =
+        collectionResponse.collections[
+          collectionResponse.collections.length - 1
+        ].$id;
+    }
+  }
+
+  // Update the backup operation with the current progress and total
+  await logOperation(
+    database,
+    databaseId,
+    {
+      operationType: "backup",
+      collectionId: "",
+      data: `Still backing up, ${data.collections.length} collections so far`,
+      progress: progress,
+      total: total,
+      error: "",
+      status: "in_progress",
+    },
+    backupOperation.$id
+  );
+
+  // Create the backup with the accumulated data
+  const bucket = await initOrGetBackupStorage(storage);
+  const inputFile = InputFile.fromPlainText(
+    JSON.stringify(data),
+    `${new Date().toISOString()}-${databaseId}.json`
+  );
+  const fileCreated = await storage.createFile(
+    bucket.$id,
+    ID.unique(),
+    inputFile
+  );
+
+  // Final update to the backup operation marking it as completed
+  await logOperation(
+    database,
+    databaseId,
+    {
+      operationType: "backup",
+      collectionId: "",
+      data: fileCreated.$id,
+      progress: 100,
+      total: total, // Ensure the total reflects the actual total processed
+      error: "",
+      status: "completed",
+    },
+    backupOperation.$id
+  );
+  console.log("---------------------------------");
+  console.log("Database Backup Complete");
+  console.log("---------------------------------");
+};

package/src/migrations/users.ts

@@ -0,0 +1,148 @@
+import type { AppwriteConfig, ConfigCollection } from "./schema.js";
+import { Databases, ID, Query, Users, type Models } from "node-appwrite";
+import {
+  AuthUserSchema,
+  type AuthUser,
+  type AuthUserCreate,
+} from "../schemas/authUser.js";
+import _ from "lodash";
+
+export class UsersController {
+  private config: AppwriteConfig;
+  private users: Users;
+  static userFields = [
+    "email",
+    "name",
+    "password",
+    "phone",
+    "labels",
+    "prefs",
+    "userId",
+    "$createdAt",
+    "$updatedAt",
+  ];
+
+  constructor(config: AppwriteConfig, db: Databases) {
+    this.config = config;
+    this.users = new Users(this.config.appwriteClient!);
+  }
+
+  async wipeUsers() {
+    const users = await this.users.list([Query.limit(25)]);
+    const allUsers = users.users;
+    let lastDocumentId: string | undefined;
+    if (users.total > 25) {
+      lastDocumentId = users.users[users.users.length - 1].$id;
+    }
+    while (lastDocumentId) {
+      const moreUsers = await this.users.list([
+        Query.limit(25),
+        Query.cursorAfter(lastDocumentId),
+      ]);
+      allUsers.push(...moreUsers.users);
+      lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
+    }
+    console.log("Deleting all users");
+    for (const user of allUsers) {
+      await this.users.delete(user.$id);
+    }
+  }
+
+  async createUserAndReturn(item: AuthUserCreate) {
+    console.log("Creating user with item", item);
+
+    // Attempt to find an existing user by email or phone.
+    let foundUsers: Models.User<Models.Preferences>[] = [];
+    if (item.email) {
+      const foundUsersByEmail = await this.users.list([
+        Query.equal("email", item.email),
+      ]);
+      foundUsers = foundUsersByEmail.users;
+    }
+    if (item.phone) {
+      const foundUsersByPhone = await this.users.list([
+        Query.equal("phone", item.phone),
+      ]);
+      foundUsers = foundUsers.length
+        ? foundUsers.concat(foundUsersByPhone.users)
+        : foundUsersByPhone.users;
+    }
+
+    let userToReturn = foundUsers[0] || undefined;
+
+    if (!userToReturn) {
+      console.log("Creating user cause not found");
+      userToReturn = await this.users.create(
+        item.userId || ID.unique(),
+        item.email || undefined,
+        item.phone && item.phone.length < 15 && item.phone.startsWith("+")
+          ? item.phone
+          : undefined,
+        item.password?.toLowerCase() || `changeMe${item.email}`.toLowerCase(),
+        item.name || undefined
+      );
+    } else {
+      console.log("Updating user cause found");
+      // Update user details as necessary, ensuring email uniqueness if attempting an update.
+      if (
+        item.email &&
+        item.email !== userToReturn.email &&
+        !_.isEmpty(item.email) &&
+        !_.isUndefined(item.email)
+      ) {
+        const emailExists = await this.users.list([
+          Query.equal("email", item.email),
+        ]);
+        if (emailExists.users.length === 0) {
+          userToReturn = await this.users.updateEmail(
+            userToReturn.$id,
+            item.email
+          );
+        } else {
+          console.log("Email update skipped: Email already exists.");
+        }
+      }
+      if (item.password) {
+        userToReturn = await this.users.updatePassword(
+          userToReturn.$id,
+          item.password.toLowerCase()
+        );
+      }
+      if (item.name && item.name !== userToReturn.name) {
+        userToReturn = await this.users.updateName(userToReturn.$id, item.name);
+      }
+      if (
+        item.phone &&
+        item.phone !== userToReturn.phone &&
+        item.phone.length < 15 &&
+        item.phone.startsWith("+") &&
+        (_.isUndefined(userToReturn.phone) || _.isEmpty(userToReturn.phone))
+      ) {
+        const userFoundWithPhone = await this.users.list([
+          Query.equal("phone", item.phone),
+        ]);
+        if (userFoundWithPhone.total === 0) {
+          userToReturn = await this.users.updatePhone(
+            userToReturn.$id,
+            item.phone
+          );
+        }
+      }
+    }
+    if (item.$createdAt && item.$updatedAt) {
+      console.log(
+        "$createdAt and $updatedAt are not yet supported, sorry about that!"
+      );
+    }
+    if (item.labels && item.labels.length) {
+      userToReturn = await this.users.updateLabels(
+        userToReturn.$id,
+        item.labels
+      );
+    }
+    if (item.prefs && Object.keys(item.prefs).length) {
+      userToReturn = await this.users.updatePrefs(userToReturn.$id, item.prefs);
+    }
+    return userToReturn;
+  }
+}

package/src/migrations/validationRules.ts

@@ -0,0 +1,63 @@
+import _ from "lodash";
+
+export interface ValidationRules {
+  [key: string]: (value: any, ...args: any[]) => boolean;
+}
+
+export const validationRules = {
+  isNumber: (value: any): boolean => _.isNumber(value),
+  isString: (value: any): boolean => _.isString(value),
+  isBoolean: (value: any): boolean => _.isBoolean(value),
+  isArray: (value: any): boolean => _.isArray(value),
+  isObject: (value: any): boolean =>
+    _.isObject(value) && !_.isArray(value) && !_.isFunction(value),
+  isNull: (value: any): boolean => _.isNull(value),
+  isValidEmail: (value: string): boolean =>
+    value.match(/^[\w\-\.]+@([\w-]+\.)+[\w-]{2,}$/) !== null,
+  isValidPhone: (value: string): boolean =>
+    value.match(/^[+]?[(]?[0-9]{3}[)]?[-\s.]?[0-9]{3}[-\s.]?[0-9]{4,6}$/im) !==
+    null,
+  isValidPassword: (value: string): boolean =>
+    value.match(
+      /^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*#?&])[A-Za-z\d@$!%*#?&]{8,}$/
+    ) !== null,
+  isValidUrl: (value: string): boolean =>
+    value.match(
+      /^(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?$/
+    ) !== null,
+  isValidHex: (value: string): boolean =>
+    value.match(/^#([a-f0-9]{6}|[a-f0-9]{3})$/i) !== null,
+  isValidHexColor: (value: string): boolean =>
+    value.match(/^#([a-f0-9]{6}|[a-f0-9]{3})$/i) !== null,
+  isValidHexAlpha: (value: string): boolean =>
+    value.match(/^#([a-f0-9]{8}|[a-f0-9]{4})$/i) !== null,
+  isValidDate: (value: string): boolean =>
+    value.match(/^\d{4}-\d{2}-\d{2}$/) !== null,
+  isValidTime: (value: string): boolean =>
+    value.match(/^\d{2}:\d{2}(:\d{2})?$/) !== null,
+  isNullish: (value: any): boolean => _.isNull(value) || _.isUndefined(value),
+  isUndefined: (value: any): boolean => _.isUndefined(value),
+  isDefined: (value: any): boolean =>
+    !_.isUndefined(value) && !_.isNull(value) && !_.isEmpty(value),
+  isDate: (value: any): boolean => _.isDate(value),
+  isEmpty: (value: any): boolean => _.isEmpty(value),
+  isInteger: (value: any): boolean => _.isInteger(value),
+  isFloat: (value: any): boolean => _.isNumber(value) && !_.isInteger(value),
+  isArrayLike: (value: any): boolean => _.isArrayLike(value),
+  isArrayLikeObject: (value: any): boolean => _.isArrayLikeObject(value),
+  isFunction: (value: any): boolean => _.isFunction(value),
+  isLength: (value: any): boolean => _.isLength(value),
+  isMap: (value: any): boolean => _.isMap(value),
+  isSet: (value: any): boolean => _.isSet(value),
+  isRegExp: (value: any): boolean => _.isRegExp(value),
+  isSymbol: (value: any): boolean => _.isSymbol(value),
+  isObjectLike: (value: any): boolean => _.isObjectLike(value),
+  isPlainObject: (value: any): boolean => _.isPlainObject(value),
+  isSafeInteger: (value: any): boolean => _.isSafeInteger(value),
+  isTypedArray: (value: any): boolean => _.isTypedArray(value),
+  isEqual: (value: any, other: any): boolean => _.isEqual(value, other),
+  isMatch: (object: any, source: any): boolean => _.isMatch(object, source),
+  has: (object: any, path: string): boolean => _.has(object, path),
+  get: (object: any, path: string, defaultValue: any): any =>
+    _.get(object, path, defaultValue),
+};

package/src/schemas/authUser.ts

@@ -0,0 +1,23 @@
+import { z } from "zod";
+
+export const AuthUserSchema = z.object({
+  $id: z.string(),
+  $createdAt: z.string().optional(),
+  $updatedAt: z.string().optional(),
+  name: z.string().nullish(),
+  email: z.string().nullish(),
+  phone: z.string().nullish(),
+  prefs: z.record(z.string()).optional().default({}),
+  labels: z.array(z.string()).optional().default([]),
+});
+
+export type AuthUser = z.infer<typeof AuthUserSchema>;
+
+export const AuthUserCreateSchema = AuthUserSchema.omit({
+  $id: true,
+}).extend({
+  userId: z.string().optional(),
+  password: z.string().optional(),
+});
+
+export type AuthUserCreate = z.infer<typeof AuthUserCreateSchema>;

package/src/setup.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import { setupDirsFiles } from "./utils/setupFiles.js";
+
+const args = process.argv.slice(2);
+
+const genExample = args.includes("--example");
+
+setupDirsFiles(genExample);

package/src/types.ts

@@ -0,0 +1,14 @@
+export type { AppwriteConfig } from "./migrations/schema.js";
+export type { ConverterFunctions } from "./migrations/converters.js";
+export type { ValidationRules } from "./migrations/validationRules.js";
+export type { AfterImportActions } from "./migrations/afterImportActions.js";
+export {
+  type AuthUserCreate,
+  AuthUserCreateSchema,
+  type AuthUser,
+  AuthUserSchema,
+} from "./schemas/authUser.js";
+export { getFileViewUrl, getFileDownloadUrl } from "./utils/helperFunctions.js";
+export { converterFunctions } from "./migrations/converters.js";
+export { validationRules } from "./migrations/validationRules.js";
+export { afterImportActions } from "./migrations/afterImportActions.js";