appwrite-utils-cli 1.6.5 → 1.6.7

package/dist/setupCommands.js

@@ -1 +1,484 @@
-export {};
+import { mkdirSync, writeFileSync, existsSync } from "node:fs";
+import path from "node:path";
+import { ulid } from "ulidx";
+import { MessageFormatter } from "./shared/messageFormatter.js";
+import { detectAppwriteVersionCached, fetchServerVersion, isVersionAtLeast } from "./utils/versionDetection.js";
+import { loadAppwriteProjectConfig, findAppwriteProjectConfig, isTablesDBProject } from "./utils/projectConfig.js";
+import { findYamlConfig, generateYamlConfigTemplate } from "./config/yamlConfig.js";
+import { loadYamlConfig } from "./config/yamlConfig.js";
+import { hasSessionAuth } from "./utils/sessionAuth.js";
+/**
+ * Get terminology configuration based on API mode
+ */
+export function getTerminologyConfig(useTables) {
+    return useTables
+        ? {
+            container: "table",
+            containerName: "Table",
+            fields: "columns",
+            fieldName: "Column",
+            security: "rowSecurity",
+            schemaRef: "table.schema.json",
+            items: "rows"
+        }
+        : {
+            container: "collection",
+            containerName: "Collection",
+            fields: "attributes",
+            fieldName: "Attribute",
+            security: "documentSecurity",
+            schemaRef: "collection.schema.json",
+            items: "documents"
+        };
+}
+/**
+ * Detect API mode using multiple detection sources
+ * Priority: appwrite.json > server version > default (collections)
+ */
+export async function detectApiMode(basePath) {
+    let useTables = false;
+    let detectionSource = "default";
+    let serverVersion;
+    try {
+        // Priority 1: Check for existing appwrite.json project config
+        const projectConfigPath = findAppwriteProjectConfig(basePath);
+        if (projectConfigPath) {
+            const projectConfig = loadAppwriteProjectConfig(projectConfigPath);
+            if (projectConfig) {
+                useTables = isTablesDBProject(projectConfig);
+                detectionSource = "appwrite.json";
+                MessageFormatter.info(`Detected ${useTables ? 'TablesDB' : 'Collections'} project from ${projectConfigPath}`, { prefix: "Setup" });
+                return {
+                    apiMode: useTables ? 'tablesdb' : 'legacy',
+                    useTables,
+                    detectionSource
+                };
+            }
+        }
+        // Priority 2: Try reading existing YAML config for version detection
+        const yamlPath = findYamlConfig(basePath);
+        if (yamlPath) {
+            const cfg = await loadYamlConfig(yamlPath);
+            if (cfg) {
+                const endpoint = cfg.appwriteEndpoint;
+                const projectId = cfg.appwriteProject;
+                if (hasSessionAuth(endpoint, projectId)) {
+                    MessageFormatter.info("Using session authentication for version detection", { prefix: "Setup" });
+                }
+                const ver = await fetchServerVersion(endpoint);
+                serverVersion = ver || undefined;
+                if (isVersionAtLeast(ver || undefined, '1.8.0')) {
+                    useTables = true;
+                    detectionSource = "server-version";
+                    MessageFormatter.info(`Detected TablesDB support (Appwrite ${ver})`, { prefix: "Setup" });
+                }
+                else {
+                    MessageFormatter.info(`Using Collections API (Appwrite ${ver || 'unknown'})`, { prefix: "Setup" });
+                }
+                return {
+                    apiMode: useTables ? 'tablesdb' : 'legacy',
+                    useTables,
+                    detectionSource,
+                    serverVersion
+                };
+            }
+        }
+    }
+    catch (error) {
+        MessageFormatter.warning(`Version detection failed, defaulting to Collections API: ${error instanceof Error ? error.message : String(error)}`, { prefix: "Setup" });
+    }
+    // Default to Collections API
+    return {
+        apiMode: 'legacy',
+        useTables: false,
+        detectionSource: 'default'
+    };
+}
+/**
+ * Create directory structure for Appwrite project
+ */
+export function createProjectDirectories(basePath, useTables) {
+    const appwriteFolder = path.join(basePath, ".appwrite");
+    const containerName = useTables ? "tables" : "collections";
+    const containerFolder = path.join(appwriteFolder, containerName);
+    const schemaFolder = path.join(appwriteFolder, "schemas");
+    const yamlSchemaFolder = path.join(appwriteFolder, ".yaml_schemas");
+    const dataFolder = path.join(appwriteFolder, "importData");
+    // Create all directories
+    for (const dir of [appwriteFolder, containerFolder, schemaFolder, yamlSchemaFolder, dataFolder]) {
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+    }
+    return {
+        appwriteFolder,
+        containerFolder,
+        schemaFolder,
+        yamlSchemaFolder,
+        dataFolder
+    };
+}
+/**
+ * Create example YAML schema file with correct terminology
+ */
+export function createExampleSchema(containerFolder, terminology) {
+    const yamlExample = `# yaml-language-server: $schema=../.yaml_schemas/${terminology.schemaRef}
+# Example ${terminology.containerName} Definition
+name: Example${terminology.containerName}
+id: example_${terminology.container}_${Date.now()}
+${terminology.security}: false
+enabled: true
+permissions:
+  - permission: read
+    target: any
+  - permission: create
+    target: users
+  - permission: update
+    target: users
+  - permission: delete
+    target: users
+${terminology.fields}:
+  - key: title
+    type: string
+    size: 255
+    required: true
+    description: "The title of the item"
+  - key: description
+    type: string
+    size: 1000
+    required: false
+    description: "A longer description"
+  - key: isActive
+    type: boolean
+    required: false
+    default: true${terminology.container === 'table' ? `
+  - key: uniqueCode
+    type: string
+    size: 50
+    required: false
+    unique: true
+    description: "Unique identifier code (TablesDB feature)"` : ''}
+indexes:
+  - key: title_search
+    type: fulltext
+    attributes:
+      - title
+importDefs: []
+`;
+    const examplePath = path.join(containerFolder, `Example${terminology.containerName}.yaml`);
+    writeFileSync(examplePath, yamlExample);
+    MessageFormatter.info(`Created example ${terminology.container} definition with ${terminology.fields} terminology`, { prefix: "Setup" });
+    return examplePath;
+}
+/**
+ * Create JSON schema for YAML validation
+ */
+export function createYamlValidationSchema(yamlSchemaFolder, useTables) {
+    const schemaFileName = useTables ? "table.schema.json" : "collection.schema.json";
+    const containerType = useTables ? "Table" : "Collection";
+    const fieldsName = useTables ? "columns" : "attributes";
+    const fieldsDescription = useTables ? "Table columns (fields)" : "Collection attributes (fields)";
+    const securityField = useTables ? "rowSecurity" : "documentSecurity";
+    const securityDescription = useTables ? "Enable row-level permissions" : "Enable document-level permissions";
+    const itemType = useTables ? "row" : "document";
+    const schema = {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "$id": `https://appwrite-utils.dev/schemas/${schemaFileName}`,
+        "title": `Appwrite ${containerType} Definition`,
+        "description": `Schema for defining Appwrite ${useTables ? 'tables' : 'collections'} in YAML${useTables ? ' (TablesDB API)' : ''}`,
+        "type": "object",
+        "properties": {
+            "name": {
+                "type": "string",
+                "description": `The name of the ${useTables ? 'table' : 'collection'}`
+            },
+            "id": {
+                "type": "string",
+                "description": `The ID of the ${useTables ? 'table' : 'collection'} (optional, auto-generated if not provided)`,
+                "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,35}$"
+            },
+            [securityField]: {
+                "type": "boolean",
+                "default": false,
+                "description": securityDescription
+            },
+            "enabled": {
+                "type": "boolean",
+                "default": true,
+                "description": `Whether the ${useTables ? 'table' : 'collection'} is enabled`
+            },
+            "permissions": {
+                "type": "array",
+                "description": `${containerType}-level permissions`,
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "permission": {
+                            "type": "string",
+                            "enum": ["read", "create", "update", "delete"],
+                            "description": "The permission type"
+                        },
+                        "target": {
+                            "type": "string",
+                            "description": "Permission target (e.g., 'any', 'users', 'users/verified', 'label:admin')"
+                        }
+                    },
+                    "required": ["permission", "target"],
+                    "additionalProperties": false
+                }
+            },
+            [fieldsName]: {
+                "type": "array",
+                "description": fieldsDescription,
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "key": {
+                            "type": "string",
+                            "description": `${useTables ? 'Column' : 'Attribute'} name`,
+                            "pattern": "^[a-zA-Z][a-zA-Z0-9]*$"
+                        },
+                        "type": {
+                            "type": "string",
+                            "enum": ["string", "integer", "double", "boolean", "datetime", "email", "ip", "url", "enum", "relationship"],
+                            "description": `${useTables ? 'Column' : 'Attribute'} data type`
+                        },
+                        "size": {
+                            "type": "number",
+                            "description": `Maximum size for string ${useTables ? 'columns' : 'attributes'}`,
+                            "minimum": 1,
+                            "maximum": 1073741824
+                        },
+                        "required": {
+                            "type": "boolean",
+                            "default": false,
+                            "description": `Whether the ${useTables ? 'column' : 'attribute'} is required`
+                        },
+                        "array": {
+                            "type": "boolean",
+                            "default": false,
+                            "description": `Whether the ${useTables ? 'column' : 'attribute'} is an array`
+                        },
+                        ...(useTables ? {
+                            "unique": {
+                                "type": "boolean",
+                                "default": false,
+                                "description": "Whether the column values must be unique (TablesDB feature)"
+                            }
+                        } : {}),
+                        "default": {
+                            "description": `Default value for the ${useTables ? 'column' : 'attribute'}`
+                        },
+                        "description": {
+                            "type": "string",
+                            "description": `${useTables ? 'Column' : 'Attribute'} description`
+                        },
+                        "min": {
+                            "type": "number",
+                            "description": `Minimum value for numeric ${useTables ? 'columns' : 'attributes'}`
+                        },
+                        "max": {
+                            "type": "number",
+                            "description": `Maximum value for numeric ${useTables ? 'columns' : 'attributes'}`
+                        },
+                        "elements": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "description": `Allowed values for enum ${useTables ? 'columns' : 'attributes'}`
+                        },
+                        "relatedCollection": {
+                            "type": "string",
+                            "description": `Related ${useTables ? 'table' : 'collection'} name for relationship ${useTables ? 'columns' : 'attributes'}`
+                        },
+                        "relationType": {
+                            "type": "string",
+                            "enum": ["oneToOne", "oneToMany", "manyToOne", "manyToMany"],
+                            "description": "Type of relationship"
+                        },
+                        "twoWay": {
+                            "type": "boolean",
+                            "description": "Whether the relationship is bidirectional"
+                        },
+                        "twoWayKey": {
+                            "type": "string",
+                            "description": "Key name for the reverse relationship"
+                        },
+                        "onDelete": {
+                            "type": "string",
+                            "enum": ["cascade", "restrict", "setNull"],
+                            "description": `Action to take when related ${itemType} is deleted`
+                        },
+                        "side": {
+                            "type": "string",
+                            "enum": ["parent", "child"],
+                            "description": "Side of the relationship"
+                        }
+                    },
+                    "required": ["key", "type"],
+                    "additionalProperties": false,
+                    "allOf": [
+                        {
+                            "if": {
+                                "properties": { "type": { "const": "enum" } }
+                            },
+                            "then": {
+                                "required": ["elements"]
+                            }
+                        },
+                        {
+                            "if": {
+                                "properties": { "type": { "const": "relationship" } }
+                            },
+                            "then": {
+                                "required": ["relatedCollection", "relationType"]
+                            }
+                        }
+                    ]
+                }
+            },
+            "indexes": {
+                "type": "array",
+                "description": `Database indexes for the ${useTables ? 'table' : 'collection'}`,
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "key": {
+                            "type": "string",
+                            "description": "Index name"
+                        },
+                        "type": {
+                            "type": "string",
+                            "enum": ["key", "fulltext", "unique"],
+                            "description": "Index type"
+                        },
+                        "attributes": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "description": `${useTables ? 'Columns' : 'Attributes'} to index`,
+                            "minItems": 1
+                        },
+                        "orders": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                                "enum": ["ASC", "DESC"]
+                            },
+                            "description": `Sort order for each ${useTables ? 'column' : 'attribute'}`
+                        }
+                    },
+                    "required": ["key", "type", "attributes"],
+                    "additionalProperties": false
+                }
+            },
+            "importDefs": {
+                "type": "array",
+                "description": "Import definitions for data migration",
+                "default": []
+            }
+        },
+        "required": ["name"],
+        "additionalProperties": false
+    };
+    const schemaPath = path.join(yamlSchemaFolder, schemaFileName);
+    writeFileSync(schemaPath, JSON.stringify(schema, null, 2));
+    return schemaPath;
+}
+/**
+ * Initialize a new Appwrite project with correct directory structure and terminology
+ */
+export async function initProject(basePath, forceApiMode) {
+    const projectPath = basePath || process.cwd();
+    // Detect API mode
+    const detection = forceApiMode
+        ? {
+            apiMode: forceApiMode,
+            useTables: forceApiMode === 'tablesdb',
+            detectionSource: 'forced',
+        }
+        : await detectApiMode(projectPath);
+    const { useTables, detectionSource } = detection;
+    const terminology = getTerminologyConfig(useTables);
+    // Create directory structure
+    const dirs = createProjectDirectories(projectPath, useTables);
+    // Generate YAML config
+    const configPath = path.join(dirs.appwriteFolder, "config.yaml");
+    generateYamlConfigTemplate(configPath);
+    // Create example schema file
+    createExampleSchema(dirs.containerFolder, terminology);
+    // Create JSON validation schema
+    createYamlValidationSchema(dirs.yamlSchemaFolder, useTables);
+    // Success messages
+    const containerType = useTables ? "TablesDB" : "Collections";
+    MessageFormatter.success(`Created YAML config and setup files/directories in .appwrite/ folder.`, { prefix: "Setup" });
+    MessageFormatter.info(`Project configured for ${containerType} API (${detectionSource} detection)`, { prefix: "Setup" });
+    MessageFormatter.info("You can now configure your project in .appwrite/config.yaml", { prefix: "Setup" });
+    MessageFormatter.info(`${terminology.containerName}s can be defined in .appwrite/${terminology.container}s/ as .yaml files`, { prefix: "Setup" });
+    MessageFormatter.info("Schemas will be generated in .appwrite/schemas/", { prefix: "Setup" });
+    MessageFormatter.info("Import data can be placed in .appwrite/importData/", { prefix: "Setup" });
+    if (useTables) {
+        MessageFormatter.info("TablesDB features: unique constraints, enhanced performance, row-level security", { prefix: "Setup" });
+    }
+}
+/**
+ * Create a new collection or table schema file
+ */
+export async function createSchema(name, basePath, forceApiMode) {
+    const projectPath = basePath || process.cwd();
+    // Detect API mode
+    const detection = forceApiMode
+        ? {
+            apiMode: forceApiMode,
+            useTables: forceApiMode === 'tablesdb',
+            detectionSource: 'forced',
+        }
+        : await detectApiMode(projectPath);
+    const { useTables } = detection;
+    const terminology = getTerminologyConfig(useTables);
+    // Find or create container directory
+    const appwriteFolder = path.join(projectPath, ".appwrite");
+    const containerFolder = path.join(appwriteFolder, useTables ? "tables" : "collections");
+    if (!existsSync(containerFolder)) {
+        mkdirSync(containerFolder, { recursive: true });
+    }
+    // Create YAML schema file
+    const yamlSchema = `# yaml-language-server: $schema=../.yaml_schemas/${terminology.schemaRef}
+# ${terminology.containerName} Definition: ${name}
+name: ${name}
+id: ${ulid()}
+${terminology.security}: false
+enabled: true
+permissions:
+  - permission: read
+    target: any
+  - permission: create
+    target: users
+  - permission: update
+    target: users
+  - permission: delete
+    target: users
+${terminology.fields}:
+  # Add your ${terminology.fields} here
+  # Example:
+  # - key: title
+  #   type: string
+  #   size: 255
+  #   required: true
+  #   description: "The title of the item"
+indexes:
+  # Add your indexes here
+  # Example:
+  # - key: title_search
+  #   type: fulltext
+  #   attributes:
+  #     - title
+importDefs: []
+`;
+    const schemaPath = path.join(containerFolder, `${name}.yaml`);
+    writeFileSync(schemaPath, yamlSchema);
+    MessageFormatter.success(`Created ${terminology.container} schema: ${schemaPath}`, { prefix: "Setup" });
+    MessageFormatter.info(`Add your ${terminology.fields} to define the ${terminology.container} structure`, { prefix: "Setup" });
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.6.5",
+  "version": "1.6.7",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/collections/indexes.ts

@@ -5,6 +5,9 @@ import { delay, tryAwaitWithRetry, calculateExponentialBackoff } from "../utils/
 import { isLegacyDatabases } from "../utils/typeGuards.js";
 import { MessageFormatter } from "../shared/messageFormatter.js";
 
+// System attributes that are always available for indexing in Appwrite
+const SYSTEM_ATTRIBUTES = ['$id', '$createdAt', '$updatedAt', '$permissions'];
+
 // Interface for index with status
 interface IndexWithStatus {
   key: string;
@@ -116,12 +119,15 @@ export const createOrUpdateIndexWithStatusCheck = async (
     // First, validate that all required attributes exist
     const freshCollection = await db.getCollection(dbId, collectionId);
     const existingAttributeKeys = freshCollection.attributes.map((attr: any) => attr.key);
-    
-    const missingAttributes = index.attributes.filter(attr => !existingAttributeKeys.includes(attr));
+
+    // Include system attributes that are always available
+    const allAvailableAttributes = [...existingAttributeKeys, ...SYSTEM_ATTRIBUTES];
+
+    const missingAttributes = index.attributes.filter(attr => !allAvailableAttributes.includes(attr));
 
     if (missingAttributes.length > 0) {
       MessageFormatter.error(`Index '${index.key}' cannot be created: missing attributes [${missingAttributes.join(', ')}] (type: ${index.type})`);
-      MessageFormatter.error(`Available attributes: [${existingAttributeKeys.join(', ')}]`);
+      MessageFormatter.error(`Available attributes: [${existingAttributeKeys.join(', ')}, ${SYSTEM_ATTRIBUTES.join(', ')}]`);
       return false; // Don't retry if attributes are missing
     }
     
@@ -279,17 +285,35 @@ export const createOrUpdateIndex = async (
   if (existingIndex.total === 0) {
     // No existing index, create it
     createIndex = true;
-  } else if (
-    !existingIndex.indexes.some(
-      (existingIndex) =>
-        (existingIndex.key === index.key &&
-          existingIndex.type === index.type &&
-          existingIndex.attributes === index.attributes)
-    )
-  ) {
-    // Existing index doesn't match, delete and recreate
-    await db.deleteIndex(dbId, collectionId, existingIndex.indexes[0].key);
-    createIndex = true;
+  } else {
+    const existing = existingIndex.indexes[0];
+
+    // Check key and type
+    const keyMatches = existing.key === index.key;
+    const typeMatches = existing.type === index.type;
+
+    // Compare attributes as SETS (order doesn't matter, only content)
+    const existingAttrsSet = new Set(existing.attributes);
+    const newAttrsSet = new Set(index.attributes);
+    const attributesMatch =
+      existingAttrsSet.size === newAttrsSet.size &&
+      [...existingAttrsSet].every(attr => newAttrsSet.has(attr));
+
+    // Compare orders as SETS if both exist (order doesn't matter)
+    let ordersMatch = true;
+    if (index.orders && existing.orders) {
+      const existingOrdersSet = new Set(existing.orders);
+      const newOrdersSet = new Set(index.orders);
+      ordersMatch =
+        existingOrdersSet.size === newOrdersSet.size &&
+        [...existingOrdersSet].every(ord => newOrdersSet.has(ord));
+    }
+
+    // Only recreate if something genuinely changed
+    if (!keyMatches || !typeMatches || !attributesMatch || !ordersMatch) {
+      await db.deleteIndex(dbId, collectionId, existing.key);
+      createIndex = true;
+    }
   }
   
   if (createIndex) {

package/src/collections/methods.ts

@@ -339,8 +339,11 @@ export const createOrUpdateCollections = async (
     // Add delay after creating attributes
     await delay(250);
 
-    // For PUSH operations, only use indexes from local config (not remote)
-    const indexesToUse = indexes || [];
+    // Prefer local config indexes, but fall back to collection's own indexes if no local config exists
+    const localCollectionConfig = config.collections?.find(
+      c => c.name === collectionData.name || c.$id === collectionData.$id
+    );
+    const indexesToUse = localCollectionConfig?.indexes ?? indexes ?? [];
 
     MessageFormatter.progress("Creating Indexes", { prefix: "Collections" });
     await createOrUpdateIndexesWithStatusCheck(
@@ -351,6 +354,16 @@ export const createOrUpdateCollections = async (
       indexesToUse as Indexes
     );
 
+    // Delete indexes that exist on server but not in local config
+    const { deleteObsoleteIndexes } = await import('../shared/indexManager.js');
+    await deleteObsoleteIndexes(
+      database,
+      databaseId,
+      collectionToUse!,
+      { indexes: indexesToUse } as any,
+      { verbose: true }
+    );
+
     // Mark this collection as fully processed to prevent re-processing
     markCollectionProcessed(collectionToUse!.$id, collectionData.name);
 
@@ -562,8 +575,11 @@ export const createOrUpdateCollectionsViaAdapter = async (
       }
     }
 
-    // Indexes
-    const idxs = (indexes || []) as any[];
+    // Prefer local config indexes, but fall back to collection's own indexes if no local config exists (TablesDB path)
+    const localTableConfig = config.collections?.find(
+      c => c.name === collectionData.name || c.$id === collectionData.$id
+    );
+    const idxs = (localTableConfig?.indexes ?? indexes ?? []) as any[];
     for (const idx of idxs) {
       try {
         await adapter.createIndex({