appwrite-utils-cli 1.8.2 → 1.8.5

package/dist/shared/pydanticModelGenerator.js

@@ -0,0 +1,615 @@
+import fs from 'fs';
+import path from 'path';
+import { MessageFormatter } from './messageFormatter.js';
+// Embedded template for base Pydantic model (always written as base.py)
+const BASE_PYDANTIC_TEMPLATE = `"""
+Appwrite-compatible Pydantic base models for SmartScraper.
+
+Provides clean base classes for all Appwrite document models without SQLAlchemy dependencies.
+"""
+
+import json
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class BaseAppwriteModel(BaseModel):
+    """
+    Base Appwrite-compatible Pydantic model with field aliases for Appwrite's $ prefixed fields.
+
+    Handles the mapping between Python-compatible field names and Appwrite's $ prefixed fields:
+    - rid -> $id
+    - created_at -> $createdAt
+    - updated_at -> $updatedAt
+    - permissions -> $permissions
+    - database_id -> $databaseId
+    - collection_id -> $collectionId
+    - sequence -> $sequence
+    """
+
+    # Optional class-level defaults for database/collection identifiers
+    databaseId: ClassVar[str | None] = None
+    collectionId: ClassVar[str | None] = None
+
+    rid: str = Field(..., alias="$id", description="Appwrite document ID")
+    created_at: datetime = Field(..., alias="$createdAt", description="Document creation timestamp")
+    updated_at: datetime = Field(
+        ..., alias="$updatedAt", description="Document last update timestamp"
+    )
+    permissions: list[str] = Field(
+        default_factory=list, alias="$permissions", description="Document permissions"
+    )
+    database_id: str = Field(..., alias="$databaseId", description="Appwrite database ID")
+    collection_id: str = Field(..., alias="$collectionId", description="Appwrite collection ID")
+    sequence: int | None = Field(None, alias="$sequence", description="Document sequence number")
+
+    class Config:
+        """Pydantic configuration for Appwrite compatibility"""
+
+        from_attributes = True
+        populate_by_name = True  # Allow both field name and alias
+        extra = "allow"  # Allow additional fields from Appwrite
+        json_encoders = {datetime: lambda v: v.isoformat() if v else None}
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def parse_datetime(cls, v: str | datetime) -> datetime:
+        """Parse datetime from string or return datetime object"""
+        if isinstance(v, str):
+            # Handle ISO format with or without microseconds
+            try:
+                return datetime.fromisoformat(v.replace("Z", "+00:00"))
+            except ValueError:
+                # Fallback for other formats
+                return datetime.fromisoformat(v)
+        return v
+
+    def to_appwrite_dict(self) -> dict[str, Any]:
+        """Convert model to dictionary with Appwrite field names ($ prefixed)"""
+        return self.model_dump(by_alias=True, exclude_unset=True)
+
+    def to_python_dict(self) -> dict[str, Any]:
+        """Convert model to dictionary with Python field names (no $ prefix)"""
+        return self.model_dump(by_alias=False, exclude_unset=True)
+
+    @classmethod
+    def from_appwrite_document(cls, document: dict[str, Any]):
+        """Create model instance from Appwrite document with $ prefixed fields"""
+        return cls.model_validate(document)
+
+    def to_update_payload(self, exclude_unset: bool = True) -> dict[str, Any]:
+        """Convert model to update payload excluding system fields and None values"""
+        data = self.model_dump(by_alias=False, exclude_unset=exclude_unset)
+        return strip_appwrite_keys(data)
+
+
+class CreateBase(BaseModel):
+    """
+    Base model for creating documents in Appwrite.
+    Makes all Appwrite system fields optional since they're auto-generated.
+    """
+
+    rid: str | None = Field(None, alias="$id", description="Optional custom document ID")
+    created_at: datetime | None = Field(
+        None, alias="$createdAt", description="Auto-generated creation timestamp"
+    )
+    updated_at: datetime | None = Field(
+        None, alias="$updatedAt", description="Auto-generated update timestamp"
+    )
+    permissions: list[str] | None = Field(
+        None, alias="$permissions", description="Optional document permissions"
+    )
+    database_id: str | None = Field(
+        None, alias="$databaseId", description="Auto-set database ID"
+    )
+    collection_id: str | None = Field(
+        None, alias="$collectionId", description="Auto-set collection ID"
+    )
+    sequence: int | None = Field(
+        None, alias="$sequence", description="Auto-generated sequence number"
+    )
+
+    class Config:
+        """Pydantic configuration for creation payloads"""
+
+        from_attributes = True
+        populate_by_name = True
+        extra = "allow"
+        json_encoders = {datetime: lambda v: v.isoformat() if v else None}
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def parse_datetime(cls, v: str | datetime | None) -> datetime | None:
+        """Parse datetime from string or return datetime object"""
+        if v is None:
+            return None
+        if isinstance(v, str):
+            try:
+                return datetime.fromisoformat(v.replace("Z", "+00:00"))
+            except ValueError:
+                return datetime.fromisoformat(v)
+        return v
+
+    def strip_appwrite_fields(self) -> dict[str, Any]:
+        """
+        Remove Appwrite system fields and return clean data for creation.
+        Useful when preparing data for Appwrite document creation.
+        """
+        excluded_fields = {
+            "rid",
+            "$id",
+            "created_at",
+            "$createdAt",
+            "updated_at",
+            "$updatedAt",
+            "permissions",
+            "$permissions",
+            "database_id",
+            "$databaseId",
+            "collection_id",
+            "$collectionId",
+            "sequence",
+            "$sequence",
+        }
+
+        data = self.model_dump(by_alias=False, exclude_unset=True)
+        return {k: v for k, v in data.items() if k not in excluded_fields}
+
+
+class UpdateBase(BaseModel):
+    """
+    Generic base model for partial updates.
+    Makes all fields optional for PATCH operations.
+    """
+
+    class Config:
+        """Pydantic configuration for update payloads"""
+
+        from_attributes = True
+        extra = "allow"
+        json_encoders = {datetime: lambda v: v.isoformat() if v else None}
+
+    def get_update_data(self, exclude_unset: bool = True) -> dict[str, Any]:
+        """
+        Get update data excluding None values and optionally unset fields.
+        Perfect for PATCH operations where only changed fields should be sent.
+        """
+        data = self.model_dump(exclude_unset=exclude_unset)
+        return {k: v for k, v in data.items() if v is not None}
+
+    def get_creation_data(self) -> dict[str, Any]:
+        """Get clean data for Appwrite document creation"""
+        return convert_to_create_payload(self)
+
+
+# ============================================================================
+# UTILITY FUNCTIONS
+# ============================================================================
+
+
+def strip_appwrite_keys(data: dict[str, Any]) -> dict[str, Any]:
+    """
+    Remove Appwrite system fields ($ prefixed) from a dictionary.
+
+    Args:
+        data: Dictionary that may contain Appwrite system fields
+
+    Returns:
+        Dictionary with Appwrite system fields removed
+
+    Example:
+        >>> data = {"name": "John", "$id": "123", "$createdAt": "2023-01-01"}
+        >>> strip_appwrite_keys(data)
+        {"name": "John"}
+    """
+    excluded_keys = {
+        "$id",
+        "$createdAt",
+        "$updatedAt",
+        "$permissions",
+        "$databaseId",
+        "$collectionId",
+        "$sequence",
+    }
+    return {k: v for k, v in data.items() if k not in excluded_keys}
+
+
+def convert_to_create_payload(model_instance: BaseModel) -> dict[str, Any]:
+    """
+    Convert any Pydantic model instance to a clean creation payload.
+    Removes Appwrite system fields and None values.
+
+    Args:
+        model_instance: Pydantic model instance
+
+    Returns:
+        Dictionary suitable for Appwrite document creation
+
+    Example:
+        >>> user = UserModel(name="John", rid="123", created_at=datetime.now())
+        >>> convert_to_create_payload(user)
+        {"name": "John"}
+    """
+    data = model_instance.model_dump(exclude_unset=True)
+    # Remove Appwrite system fields and None values
+    clean_data = strip_appwrite_keys(data)
+    return {k: v for k, v in clean_data.items() if v is not None}
+
+
+def convert_to_update_payload(
+    model_instance: BaseModel, exclude_unset: bool = True
+) -> dict[str, Any]:
+    """
+    Convert any Pydantic model instance to a clean update payload.
+    Removes None values and optionally unset fields.
+
+    Args:
+        model_instance: Pydantic model instance
+        exclude_unset: Whether to exclude fields that weren't explicitly set
+
+    Returns:
+        Dictionary suitable for Appwrite document updates
+
+    Example:
+        >>> user_update = UserUpdateModel(name="Jane")
+        >>> convert_to_update_payload(user_update)
+        {"name": "Jane"}
+    """
+    data = model_instance.model_dump(exclude_unset=exclude_unset)
+    return {k: v for k, v in data.items() if v is not None}
+
+
+# ============================================================================
+# JSON FIELD HELPER MIXINS
+# ============================================================================
+
+
+class JSONFieldMixin:
+    """
+    Mixin providing standardized JSON field helper methods.
+    Use this to add consistent JSON encode/decode patterns to models.
+    """
+
+    def _encode_json_field(self, data: Any) -> str | None:
+        """Safely encode data to JSON string"""
+        if data is None:
+            return None
+        try:
+            return json.dumps(data)
+        except (TypeError, ValueError):
+            return None
+
+    def _decode_json_field(self, json_str: str | None, default: Any = None) -> Any:
+        """Safely decode JSON string to data"""
+        if not json_str:
+            return default
+        try:
+            return json.loads(json_str)
+        except (json.JSONDecodeError, TypeError):
+            return default
+
+    def _decode_json_list(self, json_str: str | None) -> list[Any]:
+        """Safely decode JSON string to list"""
+        return self._decode_json_field(json_str, [])
+
+    def _decode_json_dict(self, json_str: str | None) -> dict[str, Any]:
+        """Safely decode JSON string to dictionary"""
+        return self._decode_json_field(json_str, {})
+
+
+class TimestampMixin:
+    """
+    Mixin providing standardized timestamp handling for business timestamps.
+    Use this for models that need to handle ISO timestamp strings.
+    """
+
+    def _set_timestamp(self, date: datetime | None) -> str | None:
+        """Convert datetime to ISO string"""
+        return date.isoformat() if date else None
+
+    def _get_timestamp(self, timestamp_str: str | None) -> datetime | None:
+        """Convert ISO string to datetime"""
+        if not timestamp_str:
+            return None
+        try:
+            return datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
+        except ValueError:
+            return None
+
+
+class StringArrayMixin:
+    """
+    Mixin providing standardized string array handling for many-to-many relationships.
+    Use this for models that manage arrays of IDs for relationships.
+    """
+
+    def _add_to_array(self, array: list[str], item: str) -> None:
+        """Add item to array if not already present"""
+        if item not in array:
+            array.append(item)
+
+    def _remove_from_array(self, array: list[str], item: str) -> None:
+        """Remove item from array if present"""
+        if item in array:
+            array.remove(item)
+
+    def _ensure_array_field(self, field_value: list[str] | None) -> list[str]:
+        """Ensure field is a list, return empty list if None"""
+        return field_value or []
+
+
+# ============================================================================
+# ENHANCED UTILITY FUNCTIONS
+# ============================================================================
+
+
+def safe_json_encode(data: Any) -> str | None:
+    """
+    Safely encode any data to JSON string.
+
+    Args:
+        data: Data to encode
+
+    Returns:
+        JSON string or None if encoding fails
+
+    Example:
+        >>> safe_json_encode({"key": "value"})
+        '{"key": "value"}'
+        >>> safe_json_encode(None)
+        None
+    """
+    if data is None:
+        return None
+    try:
+        return json.dumps(data)
+    except (TypeError, ValueError):
+        return None
+
+
+def safe_json_decode(json_str: str | None, default: Any = None) -> Any:
+    """
+    Safely decode JSON string to data.
+
+    Args:
+        json_str: JSON string to decode
+        default: Default value if decoding fails
+
+    Returns:
+        Decoded data or default value
+
+    Example:
+        >>> safe_json_decode('{"key": "value"}')
+        {'key': 'value'}
+        >>> safe_json_decode('invalid', {})
+        {}
+    """
+    if not json_str:
+        return default
+    try:
+        return json.loads(json_str)
+    except (json.JSONDecodeError, TypeError):
+        return default
+
+
+def create_json_field_helpers(field_name: str, default_type: type[Any] = dict):
+    """
+    Create getter/setter methods for JSON fields.
+    Useful for dynamically adding JSON field helpers to models.
+
+    Args:
+        field_name: Name of the JSON field
+        default_type: Default type for the field (dict or list)
+
+    Returns:
+        Tuple of (getter, setter) functions
+
+    Example:
+        >>> get_metadata, set_metadata = create_json_field_helpers('metadata')
+        >>> # Then add to model class
+    """
+    def getter(self) -> Any:
+        json_str = getattr(self, field_name, None)
+        default = default_type() if callable(default_type) else default_type
+        return safe_json_decode(json_str, default)
+
+    def setter(self, value: Any) -> None:
+        setattr(self, field_name, safe_json_encode(value))
+
+    return getter, setter
+
+
+def validate_appwrite_document(document: dict[str, Any]) -> bool:
+    """
+    Validate that a dictionary contains required Appwrite document fields.
+
+    Args:
+        document: Dictionary to validate
+
+    Returns:
+        True if valid Appwrite document format
+
+    Example:
+        >>> doc = {"$id": "123", "$createdAt": "2023-01-01T00:00:00Z", "name": "test"}
+        >>> validate_appwrite_document(doc)
+        True
+    """
+    required_fields = {"$id", "$createdAt", "$updatedAt"}
+    return all(field in document for field in required_fields)
+
+
+def batch_prepare_documents(
+    models: list[BaseModel], batch_size: int = 100
+) -> list[list[dict[str, Any]]]:
+    """
+    Prepare model instances for batch creation in Appwrite.
+    Splits into batches and removes system fields.
+
+    Args:
+        models: List of Pydantic model instances
+        batch_size: Maximum documents per batch
+
+    Returns:
+        List of batches, each containing clean document data
+
+    Example:
+        >>> users = [CreateUser(name="John"), CreateUser(name="Jane")]
+        >>> batches = batch_prepare_documents(users, batch_size=1)
+        >>> len(batches)
+        2
+    """
+    clean_docs = [convert_to_create_payload(model) for model in models]
+
+    batches = []
+    for i in range(0, len(clean_docs), batch_size):
+        batch = clean_docs[i:i + batch_size]
+        batches.append(batch)
+
+    return batches
+`;
+export class PydanticModelGenerator {
+    config;
+    appwriteFolderPath;
+    constructor(config, appwriteFolderPath) {
+        this.config = config;
+        this.appwriteFolderPath = appwriteFolderPath;
+    }
+    generatePydanticModels(options) {
+        const { baseOutputDirectory, verbose = false } = options;
+        const pyDir = baseOutputDirectory;
+        if (!fs.existsSync(pyDir))
+            fs.mkdirSync(pyDir, { recursive: true });
+        this.writeBase(pyDir, verbose);
+        const collections = this.config.collections || [];
+        for (const coll of collections) {
+            const fileName = `${this.toSnake(coll.name)}.py`;
+            const filePath = path.join(pyDir, fileName);
+            const code = this.generateModel(coll.name, coll.attributes || []);
+            fs.writeFileSync(filePath, code, { encoding: 'utf-8' });
+            if (verbose)
+                MessageFormatter.success(`Pydantic model written to ${filePath}`, { prefix: 'Schema' });
+        }
+        // __init__.py to ease imports
+        const initPath = path.join(pyDir, '__init__.py');
+        try {
+            const exports = (this.config.collections || []).map(c => `from .${this.toSnake(c.name)} import ${this.toPascal(c.name)}`).join('\n');
+            fs.writeFileSync(initPath, `${exports}\n`, { encoding: 'utf-8' });
+        }
+        catch { }
+    }
+    writeBase(pyDir, verbose) {
+        const basePath = path.join(pyDir, 'base.py');
+        // Always write embedded template content
+        fs.writeFileSync(basePath, BASE_PYDANTIC_TEMPLATE, { encoding: 'utf-8' });
+        if (verbose)
+            MessageFormatter.success(`Base Pydantic model written to ${basePath}`, { prefix: 'Schema' });
+    }
+    generateModel(name, attributes) {
+        const pascal = this.toPascal(name);
+        const imports = new Set();
+        imports.add("from .base import BaseAppwriteModel");
+        const typeImports = new Set();
+        typeImports.add('from pydantic import Field');
+        const typingImports = new Set();
+        const fields = [];
+        for (const attr of attributes) {
+            if (!attr || !attr.key)
+                continue;
+            const ann = this.mapAttributeToPythonType(attr, typingImports);
+            const required = !!attr.required;
+            const isArray = !!attr.array;
+            const defaultInitializer = this.defaultInitializer(attr, required, isArray);
+            fields.push(`    ${attr.key}: ${ann}${defaultInitializer}`);
+        }
+        const header = this.composeHeader(imports, typeImports, typingImports);
+        return `${header}\n\nclass ${pascal}(BaseAppwriteModel):\n${fields.join('\n')}\n`;
+    }
+    composeHeader(imports, typeImports, typingImports) {
+        const lines = ["from __future__ import annotations"];
+        lines.push(...Array.from(typeImports));
+        if (typingImports.size > 0) {
+            lines.push(`from typing import ${Array.from(typingImports).sort().join(', ')}`);
+        }
+        // datetime import if referenced; include by default as safe
+        lines.push('from datetime import datetime');
+        lines.push(...Array.from(imports));
+        return lines.join('\n');
+    }
+    defaultInitializer(attr, required, isArray) {
+        if (required)
+            return '';
+        // Optional fields default to None; arrays can be None to distinguish missing vs empty
+        return ' = None';
+    }
+    mapAttributeToPythonType(attr, typingImports) {
+        const t = String(attr.type || '').toLowerCase();
+        const isArray = !!attr.array;
+        let base;
+        switch (t) {
+            case 'string':
+            case 'email':
+            case 'ip':
+            case 'url':
+                base = 'str';
+                break;
+            case 'integer':
+                base = 'int';
+                break;
+            case 'double':
+            case 'float':
+                base = 'float';
+                break;
+            case 'boolean':
+                base = 'bool';
+                break;
+            case 'datetime':
+                base = 'datetime';
+                break;
+            case 'enum': {
+                const els = Array.isArray(attr.elements) ? attr.elements : [];
+                if (els.length > 0) {
+                    typingImports.add('Literal');
+                    base = `Literal[${els.map((e) => `'${e.replace(/'/g, "\\'")}'`).join(', ')}]`;
+                }
+                else {
+                    base = 'str';
+                }
+                break;
+            }
+            case 'relationship': {
+                const relType = attr.relationType || '';
+                base = (relType === 'oneToMany' || relType === 'manyToMany') ? 'list[str]' : 'str';
+                break;
+            }
+            default:
+                base = 'str';
+                break;
+        }
+        if (isArray && t !== 'relationship') {
+            base = `list[${base}]`;
+        }
+        const required = !!attr.required;
+        if (!required) {
+            base = `${base} | None`;
+        }
+        return base;
+    }
+    toSnake(s) {
+        return s
+            .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
+            .replace(/[^a-zA-Z0-9]+/g, '_')
+            .replace(/_+/g, '_')
+            .replace(/^_|_$/g, '')
+            .toLowerCase();
+    }
+    toPascal(s) {
+        return s
+            .replace(/[^a-zA-Z0-9]+/g, ' ')
+            .split(' ')
+            .filter(Boolean)
+            .map(w => w.charAt(0).toUpperCase() + w.slice(1))
+            .join('');
+    }
+}

package/dist/shared/schemaGenerator.d.ts

@@ -31,9 +31,10 @@ export declare class SchemaGenerator {
     private updateTypeScriptConfig;
     private extractRelationships;
     generateSchemas(options?: {
-        format?: "zod" | "json" | "both";
+        format?: "zod" | "json" | "pydantic" | "both" | "all";
         verbose?: boolean;
-    }): void;
+        outputDir?: string;
+    }): Promise<void>;
     createSchemaStringV4: (name: string, attributes: Attribute[]) => string;
     typeToZod: (attribute: Attribute) => string;
 }

package/dist/shared/schemaGenerator.js

@@ -289,21 +289,28 @@ export default appwriteConfig;
     extractRelationships() {
         this.relationshipMap = extractTwoWayRelationships(this.config);
     }
-    generateSchemas(options = {}) {
-        const { format = "both", verbose = false } = options;
+    async generateSchemas(options = {}) {
+        const { format = "both", verbose = false, outputDir } = options;
         if (!this.config.collections) {
             return;
         }
         // Create schemas directory using config setting
-        const outputDir = this.config.schemaConfig?.outputDirectory || "schemas";
-        const schemasPath = outputDir === "schemas"
-            ? resolveSchemaDir(this.appwriteFolderPath)
-            : path.join(this.appwriteFolderPath, outputDir);
+        const configuredDir = outputDir || this.config.schemaConfig?.outputDirectory || "schemas";
+        let schemasPath;
+        if (path.isAbsolute(configuredDir)) {
+            schemasPath = configuredDir;
+        }
+        else if (configuredDir === "schemas") {
+            schemasPath = resolveSchemaDir(this.appwriteFolderPath);
+        }
+        else {
+            schemasPath = path.join(this.appwriteFolderPath, configuredDir);
+        }
         if (!fs.existsSync(schemasPath)) {
             fs.mkdirSync(schemasPath, { recursive: true });
         }
         // Generate Zod schemas (TypeScript)
-        if (format === "zod" || format === "both") {
+        if (format === "zod" || format === "both" || format === "all") {
             this.config.collections.forEach((collection) => {
                 const schemaString = this.createSchemaStringV4(collection.name, collection.attributes || []);
                 const camelCaseName = toCamelCase(collection.name);
@@ -315,14 +322,20 @@ export default appwriteConfig;
             });
         }
         // Generate JSON schemas (all at once)
-        if (format === "json" || format === "both") {
+        if (format === "json" || format === "both" || format === "all") {
             const jsonSchemaGenerator = new JsonSchemaGenerator(this.config, this.appwriteFolderPath);
             jsonSchemaGenerator.generateJsonSchemas({
                 outputFormat: format === "json" ? "json" : "both",
-                outputDirectory: outputDir,
+                outputDirectory: configuredDir,
                 verbose: verbose
             });
         }
+        // Generate Python Pydantic models
+        if (format === "pydantic" || format === "all") {
+            const mod = await import("./pydanticModelGenerator.js");
+            const pgen = new mod.PydanticModelGenerator(this.config, this.appwriteFolderPath);
+            pgen.generatePydanticModels({ baseOutputDirectory: schemasPath, verbose });
+        }
         if (verbose) {
             MessageFormatter.success(`Schema generation completed (format: ${format})`, { prefix: "Schema" });
         }