@njdamstra/appwrite-utils-cli 1.8.9

package/src/shared/indexManager.ts

@@ -0,0 +1,254 @@
+import { type Index, type CollectionCreate } from "@njdamstra/appwrite-utils";
+import { Databases, IndexType, Query, type Models } from "node-appwrite";
+import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import chalk from "chalk";
+import pLimit from "p-limit";
+import { MessageFormatter } from "./messageFormatter.js";
+
+// Concurrency limits for different operations
+const indexLimit = pLimit(3);     // Low limit for index operations
+const queryLimit = pLimit(25);    // Higher limit for read operations
+
+export const indexesSame = (
+  databaseIndex: Models.Index,
+  configIndex: Index
+): boolean => {
+  return (
+    databaseIndex.key === configIndex.key &&
+    databaseIndex.type === configIndex.type &&
+    JSON.stringify(databaseIndex.attributes) === JSON.stringify(configIndex.attributes) &&
+    JSON.stringify(databaseIndex.orders) === JSON.stringify(configIndex.orders)
+  );
+};
+
+export const createOrUpdateIndex = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  index: Index,
+  options: {
+    verbose?: boolean;
+    forceRecreate?: boolean;
+  } = {}
+): Promise<Models.Index | null> => {
+  const { verbose = false, forceRecreate = false } = options;
+
+  return await indexLimit(async () => {
+    // Check for existing index
+    const existingIndexes = await queryLimit(() =>
+      tryAwaitWithRetry(async () => 
+        await db.listIndexes(dbId, collectionId, [Query.equal("key", index.key)])
+      )
+    );
+
+    let shouldCreate = false;
+    let existingIndex: Models.Index | undefined;
+
+    if (existingIndexes.total > 0) {
+      existingIndex = existingIndexes.indexes[0];
+      
+      if (forceRecreate || !indexesSame(existingIndex, index)) {
+        if (verbose) {
+          MessageFormatter.warning(`Updating index ${index.key} in collection ${collectionId}`, { prefix: "Index Manager" });
+        }
+        
+        // Delete existing index
+        await tryAwaitWithRetry(async () => {
+          await db.deleteIndex(dbId, collectionId, existingIndex!.key);
+        });
+        
+        await delay(500); // Wait for deletion to complete
+        shouldCreate = true;
+      } else {
+        if (verbose) {
+          MessageFormatter.success(`Index ${index.key} is up to date`, { prefix: "Index Manager" });
+        }
+        return existingIndex;
+      }
+    } else {
+      shouldCreate = true;
+      if (verbose) {
+        MessageFormatter.info(`Creating index ${index.key} in collection ${collectionId}`, { prefix: "Index Manager" });
+      }
+    }
+
+    if (shouldCreate) {
+      const newIndex = await tryAwaitWithRetry(async () => {
+        return await db.createIndex(
+          dbId,
+          collectionId,
+          index.key,
+          index.type as IndexType,
+          index.attributes,
+          index.orders
+        );
+      });
+
+      if (verbose) {
+        MessageFormatter.success(`Created index ${index.key}`, { prefix: "Index Manager" });
+      }
+
+      return newIndex;
+    }
+
+    return null;
+  });
+};
+
+export const createOrUpdateIndexes = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  indexes: Index[],
+  options: {
+    verbose?: boolean;
+    forceRecreate?: boolean;
+  } = {}
+): Promise<void> => {
+  const { verbose = false } = options;
+  
+  if (!indexes || indexes.length === 0) {
+    return;
+  }
+
+  if (verbose) {
+    MessageFormatter.info(`Processing ${indexes.length} indexes for collection ${collectionId}`, { prefix: "Index Manager" });
+  }
+
+  // Process indexes sequentially to avoid conflicts
+  for (const index of indexes) {
+    try {
+      await createOrUpdateIndex(dbId, db, collectionId, index, options);
+      
+      // Add delay between index operations to prevent rate limiting
+      await delay(250);
+    } catch (error) {
+      MessageFormatter.error(`Failed to process index ${index.key}`, error as Error, { prefix: "Index Manager" });
+      throw error;
+    }
+  }
+
+  if (verbose) {
+    MessageFormatter.success(`Completed processing indexes for collection ${collectionId}`, { prefix: "Index Manager" });
+  }
+};
+
+export const createUpdateCollectionIndexes = async (
+  db: Databases,
+  dbId: string,
+  collection: Models.Collection,
+  collectionConfig: CollectionCreate,
+  options: {
+    verbose?: boolean;
+    forceRecreate?: boolean;
+  } = {}
+): Promise<void> => {
+  if (!collectionConfig.indexes) return;
+
+  await createOrUpdateIndexes(
+    dbId,
+    db,
+    collection.$id,
+    collectionConfig.indexes,
+    options
+  );
+};
+
+export const deleteObsoleteIndexes = async (
+  db: Databases,
+  dbId: string,
+  collection: Models.Collection,
+  collectionConfig: CollectionCreate,
+  options: {
+    verbose?: boolean;
+  } = {}
+): Promise<void> => {
+  const { verbose = false } = options;
+
+  const configIndexes = collectionConfig.indexes || [];
+  const configIndexKeys = new Set(configIndexes.map(index => index.key));
+
+  // Get all existing indexes
+  const existingIndexes = await queryLimit(() =>
+    tryAwaitWithRetry(async () => 
+      await db.listIndexes(dbId, collection.$id)
+    )
+  );
+
+  // Find indexes that exist in the database but not in the config
+  const obsoleteIndexes = existingIndexes.indexes.filter(
+    (index) => !configIndexKeys.has(index.key)
+  );
+
+  if (obsoleteIndexes.length === 0) {
+    return;
+  }
+
+  if (verbose) {
+    MessageFormatter.warning(`Removing ${obsoleteIndexes.length} obsolete indexes from collection ${collection.name}`, { prefix: "Index Manager" });
+  }
+
+  // Process deletions with rate limiting
+  for (const index of obsoleteIndexes) {
+    await indexLimit(async () => {
+      await tryAwaitWithRetry(async () => {
+        await db.deleteIndex(dbId, collection.$id, index.key);
+      });
+    });
+
+    if (verbose) {
+      MessageFormatter.info(`Deleted obsolete index ${index.key}`, { prefix: "Index Manager" });
+    }
+
+    await delay(250);
+  }
+};
+
+export const validateIndexConfiguration = (
+  indexes: Index[],
+  options: {
+    verbose?: boolean;
+  } = {}
+): { valid: boolean; errors: string[] } => {
+  const { verbose = false } = options;
+  const errors: string[] = [];
+
+  for (const index of indexes) {
+    // Validate required fields
+    if (!index.key) {
+      errors.push(`Index missing required 'key' field`);
+    }
+
+    if (!index.type) {
+      errors.push(`Index '${index.key}' missing required 'type' field`);
+    }
+
+    if (!index.attributes || index.attributes.length === 0) {
+      errors.push(`Index '${index.key}' missing required 'attributes' field`);
+    }
+
+    // Validate index type
+    const validTypes = Object.values(IndexType);
+    if (index.type && !validTypes.includes(index.type as IndexType)) {
+      errors.push(`Index '${index.key}' has invalid type '${index.type}'. Valid types: ${validTypes.join(', ')}`);
+    }
+
+    // Validate orders array matches attributes length (if provided)
+    if (index.orders && index.attributes && index.orders.length !== index.attributes.length) {
+      errors.push(`Index '${index.key}' orders array length (${index.orders.length}) does not match attributes array length (${index.attributes.length})`);
+    }
+
+    // Check for duplicate keys within the same collection
+    const duplicateKeys = indexes.filter(i => i.key === index.key);
+    if (duplicateKeys.length > 1) {
+      errors.push(`Duplicate index key '${index.key}' found`);
+    }
+  }
+
+  if (verbose && errors.length > 0) {
+    MessageFormatter.error("Index validation errors", undefined, { prefix: "Index Manager" });
+    errors.forEach(error => MessageFormatter.error(`  - ${error}`, undefined, { prefix: "Index Manager" }));
+  }
+
+  return { valid: errors.length === 0, errors };
+};

package/src/shared/jsonSchemaGenerator.ts

@@ -0,0 +1,383 @@
+import fs from "fs";
+import path from "path";
+import type { AppwriteConfig, Attribute, CollectionCreate } from "@njdamstra/appwrite-utils";
+import { toCamelCase, toPascalCase } from "../utils/index.js";
+import chalk from "chalk";
+import {
+  extractSimpleRelationships,
+  resolveCollectionName,
+  type SimpleRelationship
+} from "./relationshipExtractor.js";
+import { MessageFormatter } from "./messageFormatter.js";
+
+export interface JsonSchemaProperty {
+  type: string | string[];
+  format?: string;
+  minimum?: number;
+  maximum?: number;
+  minLength?: number;
+  maxLength?: number;
+  pattern?: string;
+  enum?: any[];
+  items?: JsonSchemaProperty;
+  $ref?: string;
+  properties?: Record<string, JsonSchemaProperty>;
+  additionalProperties?: boolean;
+  required?: string[];
+  default?: any;
+  oneOf?: JsonSchemaProperty[];
+}
+
+export interface JsonSchema {
+  $schema: string;
+  $id: string;
+  title: string;
+  type: "object";
+  properties: Record<string, JsonSchemaProperty>;
+  required: string[];
+  additionalProperties: boolean;
+  definitions?: Record<string, JsonSchemaProperty>;
+}
+
+export class JsonSchemaGenerator {
+  private config: AppwriteConfig;
+  private appwriteFolderPath: string;
+  private relationshipMap = new Map<string, SimpleRelationship[]>();
+
+  constructor(config: AppwriteConfig, appwriteFolderPath: string) {
+    this.config = config;
+    this.appwriteFolderPath = appwriteFolderPath;
+    this.extractRelationships();
+  }
+
+  private resolveCollectionName = (idOrName: string): string => {
+    return resolveCollectionName(this.config, idOrName);
+  };
+
+  private extractRelationships(): void {
+    this.relationshipMap = extractSimpleRelationships(this.config);
+  }
+
+  private attributeToJsonSchemaProperty(attribute: Attribute): JsonSchemaProperty {
+    const property: JsonSchemaProperty = {
+      type: "string" // Default type
+    };
+
+
+    // Handle array attributes
+    if (attribute.array) {
+      property.type = "array";
+      property.items = this.getBaseTypeSchema(attribute);
+    } else {
+      Object.assign(property, this.getBaseTypeSchema(attribute));
+    }
+
+    // Set default value (only for attributes that support it)
+    if (attribute.type !== "relationship" && "xdefault" in attribute && 
+        attribute.xdefault !== undefined && attribute.xdefault !== null) {
+      property.default = attribute.xdefault;
+    }
+
+    return property;
+  }
+
+  private getBaseTypeSchema(attribute: Attribute): JsonSchemaProperty {
+    const schema: JsonSchemaProperty = {
+      type: "string" // Default type
+    };
+
+    switch (attribute.type) {
+      case "string":
+        schema.type = "string";
+        if (attribute.size) {
+          schema.maxLength = attribute.size;
+        }
+        break;
+
+      case "integer":
+        schema.type = "integer";
+        if (attribute.min !== undefined) {
+          schema.minimum = Number(attribute.min);
+        }
+        if (attribute.max !== undefined) {
+          schema.maximum = Number(attribute.max);
+        }
+        break;
+
+      case "double":
+      case "float": // Backward compatibility
+        schema.type = "number";
+        if (attribute.min !== undefined) {
+          schema.minimum = Number(attribute.min);
+        }
+        if (attribute.max !== undefined) {
+          schema.maximum = Number(attribute.max);
+        }
+        break;
+
+      case "boolean":
+        schema.type = "boolean";
+        break;
+
+      case "datetime":
+        schema.type = "string";
+        schema.format = "date-time";
+        break;
+
+      case "email":
+        schema.type = "string";
+        schema.format = "email";
+        break;
+
+      case "ip":
+        schema.type = "string";
+        schema.format = "ipv4";
+        break;
+
+      case "url":
+        schema.type = "string";
+        schema.format = "uri";
+        break;
+
+      case "enum":
+        schema.type = "string";
+        if (attribute.elements) {
+          schema.enum = attribute.elements;
+        }
+        break;
+
+      case "relationship":
+        if (attribute.relatedCollection) {
+          // For relationships, reference the related collection schema
+          schema.$ref = `#/definitions/${toPascalCase(this.resolveCollectionName(attribute.relatedCollection))}`;
+        } else {
+          schema.type = "string";
+        }
+        break;
+
+      default:
+        schema.type = "string";
+    }
+
+    return schema;
+  }
+
+  private createJsonSchema(collection: CollectionCreate): JsonSchema {
+    const pascalName = toPascalCase(collection.name);
+    const schema: JsonSchema = {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      $id: `https://example.com/schemas/${toCamelCase(collection.name)}.json`,
+      title: pascalName,
+      type: "object",
+      properties: {
+        // Standard Appwrite document fields
+        $id: {
+          type: "string",
+          pattern: "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,35}$"
+        },
+        $createdAt: {
+          type: "string",
+          format: "date-time"
+        },
+        $updatedAt: {
+          type: "string", 
+          format: "date-time"
+        },
+        $permissions: {
+          type: "array",
+          items: {
+            type: "string"
+          }
+        }
+      },
+      required: ["$id", "$createdAt", "$updatedAt"],
+      additionalProperties: false
+    };
+
+    // Add custom attributes
+    const requiredFields: string[] = [...schema.required];
+    
+    if (collection.attributes) {
+      collection.attributes.forEach((attribute) => {
+        schema.properties[attribute.key] = this.attributeToJsonSchemaProperty(attribute);
+        
+        if (attribute.required) {
+          requiredFields.push(attribute.key);
+        }
+      });
+    }
+
+    schema.required = requiredFields;
+
+    // Add relationship definitions if any exist
+    const relationships = this.relationshipMap.get(collection.name);
+    if (relationships && relationships.length > 0) {
+      schema.definitions = {};
+      
+      relationships.forEach((rel) => {
+        const relatedPascalName = toPascalCase(rel.relatedCollection);
+        schema.definitions![relatedPascalName] = {
+          type: "object",
+          properties: {
+            $id: { type: "string" },
+            $createdAt: { type: "string", format: "date-time" },
+            $updatedAt: { type: "string", format: "date-time" }
+          },
+          additionalProperties: true
+        };
+      });
+    }
+
+    return schema;
+  }
+
+  public generateJsonSchemas(options: {
+    outputFormat?: "json" | "typescript" | "both";
+    outputDirectory?: string;
+    verbose?: boolean;
+  } = {}): void {
+    const { outputFormat = "both", outputDirectory = "schemas", verbose = false } = options;
+
+    if (!this.config.collections) {
+      if (verbose) {
+        MessageFormatter.warning("No collections found in config", { prefix: "Schema" });
+      }
+      return;
+    }
+
+    // Create JSON schemas directory using provided outputDirectory
+    const jsonSchemasPath = path.isAbsolute(outputDirectory)
+      ? outputDirectory
+      : path.join(this.appwriteFolderPath, outputDirectory);
+    if (!fs.existsSync(jsonSchemasPath)) {
+      fs.mkdirSync(jsonSchemasPath, { recursive: true });
+    }
+
+    if (verbose) {
+      MessageFormatter.processing(`Generating JSON schemas for ${this.config.collections.length} collections...`, { prefix: "Schema" });
+    }
+
+    this.config.collections.forEach((collection) => {
+      const schema = this.createJsonSchema(collection);
+      const camelCaseName = toCamelCase(collection.name);
+
+      // Generate JSON file
+      if (outputFormat === "json" || outputFormat === "both") {
+        const jsonPath = path.join(jsonSchemasPath, `${camelCaseName}.json`);
+        fs.writeFileSync(jsonPath, JSON.stringify(schema, null, 2), { encoding: "utf-8" });
+
+        if (verbose) {
+          MessageFormatter.success(`JSON schema written to ${jsonPath}`, { prefix: "Schema" });
+        }
+      }
+
+      // Generate TypeScript file
+      if (outputFormat === "typescript" || outputFormat === "both") {
+        const tsContent = this.generateTypeScriptSchema(schema, collection.name);
+        const tsPath = path.join(jsonSchemasPath, `${camelCaseName}.schema.ts`);
+        fs.writeFileSync(tsPath, tsContent, { encoding: "utf-8" });
+
+        if (verbose) {
+          MessageFormatter.success(`TypeScript schema written to ${tsPath}`, { prefix: "Schema" });
+        }
+      }
+    });
+
+    // Generate index file only for TypeScript output
+    if (outputFormat === "typescript" || outputFormat === "both") {
+      this.generateIndexFile(outputFormat, jsonSchemasPath, verbose);
+    }
+
+    if (verbose) {
+      MessageFormatter.success("JSON schema generation completed", { prefix: "Schema" });
+    }
+  }
+
+  private generateTypeScriptSchema(schema: JsonSchema, collectionName: string): string {
+    const camelName = toCamelCase(collectionName);
+
+    return `// Auto-generated JSON schema for ${collectionName}
+import type { JSONSchema7 } from "json-schema";
+
+export const ${camelName}JsonSchema: JSONSchema7 = ${JSON.stringify(schema, null, 2)} as const;
+
+export type ${toPascalCase(collectionName)}JsonSchema = typeof ${camelName}JsonSchema;
+
+export default ${camelName}JsonSchema;
+`;
+  }
+
+  private generateIndexFile(outputFormat: string, jsonSchemasPath: string, verbose: boolean): void {
+    if (!this.config.collections) return;
+
+    const indexPath = path.join(jsonSchemasPath, "index.ts");
+
+    const imports: string[] = [];
+    const exports: string[] = [];
+
+    this.config.collections.forEach((collection) => {
+      const camelName = toCamelCase(collection.name);
+      const pascalName = toPascalCase(collection.name);
+
+      if (outputFormat === "typescript" || outputFormat === "both") {
+        imports.push(`import { ${camelName}JsonSchema } from "./${camelName}.schema.js";`);
+        exports.push(`  ${camelName}: ${camelName}JsonSchema,`);
+      }
+    });
+
+    const indexContent = `// Auto-generated index for JSON schemas
+${imports.join('\n')}
+
+export const jsonSchemas = {
+${exports.join('\n')}
+};
+
+export type JsonSchemas = typeof jsonSchemas;
+
+// Individual schema exports
+${this.config.collections.map(collection => {
+  const camelName = toCamelCase(collection.name);
+  return `export { ${camelName}JsonSchema } from "./${camelName}.schema.js";`;
+}).join('\n')}
+
+export default jsonSchemas;
+`;
+
+    fs.writeFileSync(indexPath, indexContent, { encoding: "utf-8" });
+
+    if (verbose) {
+      MessageFormatter.success(`Index file written to ${indexPath}`, { prefix: "Schema" });
+    }
+  }
+
+  public validateSchema(schema: JsonSchema): { valid: boolean; errors: string[] } {
+    const errors: string[] = [];
+
+    // Basic validation
+    if (!schema.$schema) {
+      errors.push("Missing $schema property");
+    }
+
+    if (!schema.title) {
+      errors.push("Missing title property");
+    }
+
+    if (!schema.properties) {
+      errors.push("Missing properties");
+    }
+
+    if (!schema.required || !Array.isArray(schema.required)) {
+      errors.push("Missing or invalid required array");
+    }
+
+    // Validate required Appwrite fields
+    const requiredAppwriteFields = ["$id", "$createdAt", "$updatedAt"];
+    requiredAppwriteFields.forEach((field) => {
+      if (!schema.properties[field]) {
+        errors.push(`Missing required Appwrite field: ${field}`);
+      }
+    });
+
+    return { valid: errors.length === 0, errors };
+  }
+}