appwrite-utils-cli 0.0.1

package/src/migrations/backup.ts

@@ -0,0 +1,189 @@
+import { z } from "zod";
+import {
+  attributeSchema,
+  type Attribute,
+  parseAttribute,
+  CollectionCreateSchema,
+} from "./schema.js";
+
+export const BackupSchema = z.object({
+  $id: z.string(),
+  $createdAt: z.string(),
+  $updatedAt: z.string(),
+  database: z.string(),
+  collections: z.array(z.string()),
+  documents: z
+    .array(
+      z.object({
+        collectionId: z.string(),
+        data: z.string(),
+      })
+    )
+    .default([]),
+});
+
+export type Backup = z.infer<typeof BackupSchema>;
+
+export const BackupCreateSchema = BackupSchema.omit({
+  $id: true,
+  $createdAt: true,
+  $updatedAt: true,
+});
+
+export type BackupCreate = z.infer<typeof BackupCreateSchema>;
+
+export const BatchSchema = z.object({
+  $id: z.string(),
+  $createdAt: z.string(),
+  $updatedAt: z.string(),
+  data: z.string().describe("The serialized data for this batch"),
+  processed: z
+    .boolean()
+    .default(false)
+    .describe("Whether the batch has been processed"),
+});
+
+export type Batch = z.infer<typeof BatchSchema>;
+
+export const BatchCreateSchema = BatchSchema.omit({
+  $id: true,
+  $createdAt: true,
+  $updatedAt: true,
+});
+
+export type BatchCreate = z.infer<typeof BatchCreateSchema>;
+
+export const OperationSchema = z.object({
+  $id: z.string(),
+  $createdAt: z.string(),
+  $updatedAt: z.string(),
+  operationType: z.string(),
+  collectionId: z.string(),
+  data: z.any(),
+  batches: z.array(z.string()).default([]).optional(),
+  progress: z.number(),
+  total: z.number(),
+  error: z.string(),
+  status: z.enum(["pending", "in_progress", "completed", "error"]),
+});
+
+export type Operation = z.infer<typeof OperationSchema>;
+
+export const OperationCreateSchema = OperationSchema.omit({
+  $id: true,
+  $createdAt: true,
+  $updatedAt: true,
+});
+
+export type OperationCreate = z.infer<typeof OperationCreateSchema>;
+
+export const getMigrationCollectionSchemas = () => {
+  const currentOperationsAttributes: Attribute[] = [
+    parseAttribute({
+      key: "operationType",
+      type: "string",
+      error: "Invalid Operation Type",
+      size: 50,
+      required: true,
+      array: false,
+      xdefault: null,
+    }),
+    attributeSchema.parse({
+      key: "collectionId",
+      type: "string",
+      error: "Invalid Collection Id",
+      size: 50,
+      array: false,
+      xdefault: null,
+    }),
+    attributeSchema.parse({
+      key: "batches",
+      type: "string",
+      error: "Invalid Batches",
+      size: 1073741824,
+      array: true,
+    }),
+    attributeSchema.parse({
+      key: "data",
+      type: "string",
+      error: "Invalid Data",
+      size: 1073741824,
+    }),
+    attributeSchema.parse({
+      key: "progress",
+      type: "integer",
+      error: "Invalid Progress",
+      required: true,
+      array: false,
+    }),
+    attributeSchema.parse({
+      key: "total",
+      type: "integer",
+      error: "Invalid Total",
+      required: true,
+      array: false,
+    }),
+    attributeSchema.parse({
+      key: "error",
+      type: "string",
+      error: "Operation Error",
+      required: false,
+      array: false,
+    }),
+    attributeSchema.parse({
+      key: "status",
+      type: "enum",
+      elements: ["pending", "in_progress", "completed", "error"],
+      error: "Invalid Status",
+      array: false,
+      xdefault: "pending",
+    }),
+  ];
+
+  const currentOperationsConfig = CollectionCreateSchema.parse({
+    name: "CurrentOperations",
+    enabled: true,
+    documentSecurity: false,
+    attributes: [],
+    indexes: [],
+  });
+
+  const batchesAttributes: Attribute[] = [
+    attributeSchema.parse({
+      key: "data",
+      type: "string",
+      size: 1073741824,
+      error: "Invalid Data",
+      required: true,
+      array: false,
+    }),
+    attributeSchema.parse({
+      key: "processed",
+      type: "boolean",
+      error: "Invalid Processed",
+      required: true,
+      array: false,
+      xdefault: false,
+    }),
+  ];
+
+  const batchesConfig = CollectionCreateSchema.parse({
+    name: "Batches",
+    enabled: true,
+    documentSecurity: false,
+    attributes: [],
+    indexes: [],
+  });
+
+  const toReturn = {
+    CurrentOperations: {
+      collection: currentOperationsConfig,
+      attributes: currentOperationsAttributes,
+    },
+    Batches: {
+      collection: batchesConfig,
+      attributes: batchesAttributes,
+    },
+  };
+  return toReturn;
+};

package/src/migrations/collections.ts

@@ -0,0 +1,303 @@
+import { Databases, ID, Permission, Query, type Models } from "node-appwrite";
+import type { AppwriteConfig, CollectionCreate } from "./schema.js";
+import { nameToIdMapping, processQueue } from "./queue.js";
+import { createUpdateCollectionAttributes } from "./attributes.js";
+import { createOrUpdateIndexes } from "./indexes.js";
+import {
+  ensureDirectoryExistence,
+  toCamelCase,
+  toPascalCase,
+  writeFileSync,
+} from "../utils/index.js";
+import _ from "lodash";
+import { SchemaGenerator } from "./schemaStrings.js";
+import path from "path";
+
+const { join } = _;
+
+export const documentExists = async (
+  db: Databases,
+  dbId: string,
+  targetCollectionId: string,
+  toCreateObject: any
+): Promise<Models.Document | null> => {
+  // Had to do this because kept running into issues with type checking arrays so, sorry 40ms
+  const collection = await db.getCollection(dbId, targetCollectionId);
+  const attributes = collection.attributes as any[];
+  let arrayTypeAttributes = attributes
+    .filter((attribute: any) => attribute.array === true)
+    .map((attribute: any) => attribute.key);
+  // Function to check if a string is JSON
+  const isJsonString = (str: string) => {
+    try {
+      const json = JSON.parse(str);
+      return typeof json === "object" && json !== null; // Check if parsed JSON is an object or array
+    } catch (e) {
+      return false;
+    }
+  };
+
+  // Validate and prepare query parameters
+  const validQueryParams = _.chain(toCreateObject)
+    .pickBy(
+      (value, key) =>
+        !arrayTypeAttributes.includes(key) &&
+        !key.startsWith("$") &&
+        !_.isNull(value) &&
+        !_.isUndefined(value) &&
+        !_.isEmpty(value) &&
+        !_.isObject(value) && // Keeps excluding objects
+        !_.isArray(value) && // Explicitly exclude arrays
+        !(_.isString(value) && isJsonString(value)) && // Exclude JSON strings
+        (_.isString(value) ? value.length < 4096 && value.length > 0 : true) // String length check
+    )
+    .mapValues((value, key) =>
+      _.isString(value) || _.isNumber(value) || _.isBoolean(value)
+        ? value
+        : null
+    )
+    .omitBy(_.isNull) // Remove any null values that might have been added in mapValues
+    .toPairs()
+    .slice(0, 25) // Limit to 25 to adhere to query limit
+    .map(([key, value]) => Query.equal(key, value as any))
+    .value();
+
+  // Execute the query with the validated and prepared parameters
+  const result = await db.listDocuments(
+    dbId,
+    targetCollectionId,
+    validQueryParams
+  );
+  return result.documents[0] || null;
+};
+
+export const checkForCollection = async (
+  db: Databases,
+  dbId: string,
+  collection: Partial<CollectionCreate>
+): Promise<Models.Collection | null> => {
+  try {
+    console.log(`Checking for collection with name: ${collection.name}`);
+    const response = await db.listCollections(dbId, [
+      Query.equal("name", collection.name!),
+    ]);
+    if (response.collections.length > 0) {
+      console.log(`Collection found: ${response.collections[0].$id}`);
+      return { ...collection, ...response.collections[0] };
+    } else {
+      console.log(`No collection found with name: ${collection.name}`);
+      return null;
+    }
+  } catch (error) {
+    console.error(`Error checking for collection: ${error}`);
+    return null;
+  }
+};
+
+// Helper function to fetch and cache collection by name
+export const fetchAndCacheCollectionByName = async (
+  db: Databases,
+  dbId: string,
+  collectionName: string
+): Promise<Models.Collection | undefined> => {
+  if (nameToIdMapping.has(collectionName)) {
+    const collectionId = nameToIdMapping.get(collectionName);
+    console.log(`\tCollection found in cache: ${collectionId}`);
+    return await db.getCollection(dbId, collectionId!);
+  } else {
+    console.log(`\tFetching collection by name: ${collectionName}`);
+    const collectionsPulled = await db.listCollections(dbId, [
+      Query.equal("name", collectionName),
+    ]);
+    if (collectionsPulled.total > 0) {
+      const collection = collectionsPulled.collections[0];
+      console.log(`\tCollection found: ${collection.$id}`);
+      nameToIdMapping.set(collectionName, collection.$id);
+      return collection;
+    } else {
+      console.log(`\tCollection not found by name: ${collectionName}`);
+      return undefined;
+    }
+  }
+};
+
+export const wipeDatabase = async (
+  database: Databases,
+  databaseId: string
+): Promise<{ collectionId: string; collectionName: string }[]> => {
+  console.log(`Wiping database: ${databaseId}`);
+  const { collections: existingCollections } = await database.listCollections(
+    databaseId
+  );
+  let collectionsDeleted: { collectionId: string; collectionName: string }[] =
+    [];
+  for (const { $id: collectionId, name: name } of existingCollections) {
+    console.log(`Deleting collection: ${collectionId}`);
+    collectionsDeleted.push({
+      collectionId: collectionId,
+      collectionName: name,
+    });
+    await database.deleteCollection(databaseId, collectionId);
+  }
+  return collectionsDeleted;
+};
+
+export const generateSchemas = async (
+  config: AppwriteConfig,
+  appwriteFolderPath: string
+): Promise<void> => {
+  const schemaGenerator = new SchemaGenerator(config, appwriteFolderPath);
+  schemaGenerator.generateSchemas();
+};
+
+export const createOrUpdateCollections = async (
+  database: Databases,
+  databaseId: string,
+  config: AppwriteConfig,
+  deletedCollections?: { collectionId: string; collectionName: string }[]
+): Promise<void> => {
+  const configCollections = config.collections;
+  for (const { attributes, indexes, ...collection } of configCollections) {
+    let collectionsFound = await database.listCollections(databaseId, [
+      Query.equal("name", collection.name),
+    ]);
+
+    const permissions = [];
+    if (collection.$permissions.length > 0) {
+      for (const permission of collection.$permissions) {
+        switch (permission.permission) {
+          case "read":
+            permissions.push(Permission.read(permission.target));
+            break;
+          case "create":
+            permissions.push(Permission.create(permission.target));
+            break;
+          case "update":
+            permissions.push(Permission.update(permission.target));
+            break;
+          case "delete":
+            permissions.push(Permission.delete(permission.target));
+            break;
+          case "write":
+            permissions.push(Permission.write(permission.target));
+            break;
+          default:
+            console.log(`Unknown permission: ${permission.permission}`);
+            break;
+        }
+      }
+    }
+    let collectionToUse =
+      collectionsFound.total > 0 ? collectionsFound.collections[0] : null;
+    if (!collectionToUse) {
+      console.log(`Creating collection: ${collection.name}`);
+      if (deletedCollections && deletedCollections.length > 0) {
+        const foundColl = deletedCollections.find(
+          (coll) =>
+            coll.collectionName.toLowerCase().trim().replace(" ", "") ===
+            collection.name.toLowerCase().trim().replace(" ", "")
+        );
+        if (foundColl) {
+          const collectionId = foundColl.collectionId || ID.unique();
+          console.log(
+            `Processing collection: ${collection.name} with ID: ${collectionId}`
+          );
+          collectionToUse = await database.createCollection(
+            databaseId,
+            collectionId,
+            collection.name,
+            permissions,
+            collection.documentSecurity,
+            collection.enabled
+          );
+          nameToIdMapping.set(collection.name, collectionToUse.$id);
+        } else {
+          collectionToUse = await database.createCollection(
+            databaseId,
+            ID.unique(),
+            collection.name,
+            permissions,
+            collection.documentSecurity,
+            collection.enabled
+          );
+          nameToIdMapping.set(collection.name, collectionToUse.$id);
+        }
+      } else {
+        collectionToUse = await database.createCollection(
+          databaseId,
+          ID.unique(),
+          collection.name,
+          permissions,
+          collection.documentSecurity,
+          collection.enabled
+        );
+        nameToIdMapping.set(collection.name, collectionToUse.$id);
+      }
+    } else {
+      console.log(`Collection ${collection.name} already exists.`);
+    }
+    console.log("Creating Attributes");
+    await createUpdateCollectionAttributes(
+      database,
+      databaseId,
+      collectionToUse,
+      attributes
+    );
+    console.log("Creating Indexes");
+    await createOrUpdateIndexes(
+      databaseId,
+      database,
+      collectionToUse.$id,
+      indexes
+    );
+  }
+  await processQueue(database, databaseId);
+};
+
+export const generateMockData = async (
+  database: Databases,
+  databaseId: string,
+  configCollections: any[]
+): Promise<void> => {
+  for (const { collection, mockFunction } of configCollections) {
+    if (mockFunction) {
+      console.log(`Generating mock data for collection: ${collection.name}`);
+      const mockData = mockFunction();
+      for (const data of mockData) {
+        await database.createDocument(
+          databaseId,
+          collection.$id,
+          ID.unique(),
+          data
+        );
+      }
+    }
+  }
+};
+
+export const fetchAllCollections = async (
+  dbId: string,
+  database: Databases
+): Promise<Models.Collection[]> => {
+  console.log(`Fetching all collections for database ID: ${dbId}`);
+  let collections: Models.Collection[] = [];
+  let moreCollections = true;
+  let lastCollectionId: string | undefined;
+
+  while (moreCollections) {
+    const queries = [Query.limit(500)];
+    if (lastCollectionId) {
+      queries.push(Query.cursorAfter(lastCollectionId));
+    }
+    const response = await database.listCollections(dbId, queries);
+    collections = collections.concat(response.collections);
+    moreCollections = response.collections.length === 500;
+    if (moreCollections) {
+      lastCollectionId =
+        response.collections[response.collections.length - 1].$id;
+    }
+  }
+
+  console.log(`Fetched a total of ${collections.length} collections.`);
+  return collections;
+};