appwrite-utils-cli 1.9.7 → 1.11.0

package/dist/shared/operationQueue.js

@@ -1,311 +0,0 @@
-import { Query } from "node-appwrite";
-import { createOrUpdateAttributeWithStatusCheck } from "../collections/attributes.js";
-import { fetchAndCacheCollectionByName } from "../collections/methods.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-import { logger } from "../shared/logging.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-// Global state management
-export const queuedOperations = [];
-export const nameToIdMapping = new Map();
-// Keys are scoped per database to avoid cross-database collisions
-// Collections key format: `${databaseId}::${collectionId}`
-// Attributes key format: `${databaseId}::${collectionId}::${attributeKey}`
-export const processedCollections = new Set();
-export const processedAttributes = new Set();
-// Helpers to build scoped keys
-const collectionKey = (databaseId, collectionId) => `${databaseId}::${collectionId}`;
-const attributeKeyScoped = (databaseId, collectionId, key) => `${databaseId}::${collectionId}::${key}`;
-export const enqueueOperation = (operation) => {
-    // Avoid duplicate queue entries for same attribute
-    const attributeKey = operation.attribute?.key;
-    const collectionId = operation.collectionId;
-    logger.info('Enqueueing operation', {
-        type: operation.type,
-        attributeKey,
-        collectionId,
-        dependencies: operation.dependencies,
-        queueSizeBefore: queuedOperations.length,
-        operation: 'enqueueOperation'
-    });
-    if (attributeKey && collectionId) {
-        const duplicateIndex = queuedOperations.findIndex((op) => op.collectionId === collectionId && op.attribute?.key === attributeKey);
-        if (duplicateIndex !== -1) {
-            MessageFormatter.info(`Replacing existing queue entry for attribute: ${attributeKey}`);
-            logger.info('Replacing duplicate queue entry', {
-                attributeKey,
-                collectionId,
-                duplicateIndex,
-                operation: 'enqueueOperation'
-            });
-            queuedOperations[duplicateIndex] = operation;
-            return;
-        }
-    }
-    queuedOperations.push(operation);
-    logger.debug('Operation enqueued successfully', {
-        attributeKey,
-        collectionId,
-        queueSizeAfter: queuedOperations.length,
-        operation: 'enqueueOperation'
-    });
-};
-/**
- * Clear all caches and processing state - use between operations
- */
-export const clearProcessingState = () => {
-    const sizeBefore = {
-        collections: processedCollections.size,
-        attributes: processedAttributes.size,
-        nameMapping: nameToIdMapping.size
-    };
-    processedCollections.clear();
-    processedAttributes.clear();
-    nameToIdMapping.clear();
-    logger.debug("Cleared processing state caches", { operation: "clearProcessingState", sizeBefore });
-    logger.info('Processing state cleared', {
-        sizeBefore,
-        operation: 'clearProcessingState'
-    });
-};
-/**
- * Check if a collection has already been fully processed
- */
-export const isCollectionProcessed = (collectionId, databaseId) => {
-    return processedCollections.has(collectionKey(databaseId, collectionId));
-};
-/**
- * Mark a collection as fully processed
- */
-export const markCollectionProcessed = (collectionId, collectionName, databaseId) => {
-    processedCollections.add(collectionKey(databaseId, collectionId));
-    const logData = {
-        databaseId,
-        collectionId,
-        collectionName,
-        totalProcessedCollections: processedCollections.size,
-        operation: 'markCollectionProcessed'
-    };
-    if (collectionName) {
-        MessageFormatter.success(`Marked collection '${collectionName}' (${collectionId}) as processed`, { prefix: 'Tables' });
-    }
-    logger.info('Collection marked as processed', logData);
-};
-/**
- * Check if a specific attribute has been processed
- */
-export const isAttributeProcessed = (databaseId, collectionId, attributeKey) => {
-    return processedAttributes.has(attributeKeyScoped(databaseId, collectionId, attributeKey));
-};
-/**
- * Mark a specific attribute as processed
- */
-export const markAttributeProcessed = (databaseId, collectionId, attributeKey) => {
-    const identifier = attributeKeyScoped(databaseId, collectionId, attributeKey);
-    processedAttributes.add(identifier);
-    logger.debug('Attribute marked as processed', {
-        databaseId,
-        collectionId,
-        attributeKey,
-        identifier,
-        totalProcessedAttributes: processedAttributes.size,
-        operation: 'markAttributeProcessed'
-    });
-};
-/**
- * Process only specific attributes in the queue, not entire collections
- * This prevents triggering full collection re-processing cycles
- */
-export const processQueue = async (db, dbId) => {
-    const startTime = Date.now();
-    if (queuedOperations.length === 0) {
-        MessageFormatter.info("No queued operations to process");
-        logger.info('Queue processing skipped - no operations', {
-            dbId,
-            operation: 'processQueue'
-        });
-        return;
-    }
-    MessageFormatter.section(`Starting surgical queue processing of ${queuedOperations.length} operations for ${dbId}`);
-    logger.info('Starting queue processing', {
-        dbId,
-        queueSize: queuedOperations.length,
-        operations: queuedOperations.map(op => ({
-            type: op.type,
-            attributeKey: op.attribute?.key,
-            collectionId: op.collectionId,
-            dependencies: op.dependencies
-        })),
-        operation: 'processQueue'
-    });
-    let progress = true;
-    let attempts = 0;
-    const maxAttempts = 3; // Prevent infinite loops
-    while (progress && attempts < maxAttempts) {
-        progress = false;
-        attempts++;
-        MessageFormatter.info(`Queue processing attempt ${attempts}/${maxAttempts}`);
-        logger.info('Queue processing attempt started', {
-            attempt: attempts,
-            maxAttempts,
-            remainingOperations: queuedOperations.length,
-            dbId,
-            operation: 'processQueue'
-        });
-        for (let i = queuedOperations.length - 1; i >= 0; i--) {
-            const operation = queuedOperations[i];
-            if (!operation.attribute || !operation.collectionId) {
-                MessageFormatter.warning("Invalid operation, removing from queue");
-                queuedOperations.splice(i, 1);
-                continue;
-            }
-            const attributeKey = operation.attribute.key;
-            const collectionId = operation.collectionId;
-            // Skip if this specific attribute was already processed (per database)
-            if (isAttributeProcessed(dbId, collectionId, attributeKey)) {
-                MessageFormatter.debug(`Attribute '${attributeKey}' already processed, removing from queue`);
-                logger.debug('Removing already processed attribute from queue', {
-                    attributeKey,
-                    collectionId,
-                    queueIndex: i,
-                    operation: 'processQueue'
-                });
-                queuedOperations.splice(i, 1);
-                continue;
-            }
-            let targetCollection;
-            // Resolve the target collection (where the attribute will be created)
-            try {
-                targetCollection = await tryAwaitWithRetry(async () => {
-                    if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                        // DatabaseAdapter
-                        return (await db.getTable({ databaseId: dbId, tableId: collectionId })).data;
-                    }
-                    else {
-                        // Legacy Databases
-                        return await db.getCollection(dbId, collectionId);
-                    }
-                });
-            }
-            catch (e) {
-                const errorMessage = e instanceof Error ? e.message : String(e);
-                MessageFormatter.error(`Target collection ${collectionId} not found, removing from queue`);
-                logger.error('Target collection not found during queue processing', {
-                    collectionId,
-                    attributeKey,
-                    error: errorMessage,
-                    operation: 'processQueue'
-                });
-                queuedOperations.splice(i, 1);
-                continue;
-            }
-            // For relationship attributes, ensure the related collection exists
-            let canProcess = true;
-            if (operation.attribute.type === "relationship") {
-                const relatedCollection = operation.attribute.relatedCollection;
-                if (relatedCollection) {
-                    // Try to resolve related collection by ID first, then by name
-                    let relatedFound = false;
-                    try {
-                        await tryAwaitWithRetry(async () => {
-                            if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                                // DatabaseAdapter
-                                return (await db.getTable({ databaseId: dbId, tableId: relatedCollection })).data;
-                            }
-                            else {
-                                // Legacy Databases
-                                return await db.getCollection(dbId, relatedCollection);
-                            }
-                        });
-                        relatedFound = true;
-                        nameToIdMapping.set(relatedCollection, relatedCollection); // Cache the ID mapping
-                    }
-                    catch (_) {
-                        // Try by name lookup
-                        const cachedId = nameToIdMapping.get(relatedCollection);
-                        if (cachedId) {
-                            try {
-                                await tryAwaitWithRetry(async () => {
-                                    if ('getMetadata' in db && typeof db.getMetadata === 'function') {
-                                        // DatabaseAdapter
-                                        return (await db.getTable({ databaseId: dbId, tableId: cachedId })).data;
-                                    }
-                                    else {
-                                        // Legacy Databases
-                                        return await db.getCollection(dbId, cachedId);
-                                    }
-                                });
-                                relatedFound = true;
-                            }
-                            catch (_) {
-                                nameToIdMapping.delete(relatedCollection); // Remove stale cache
-                            }
-                        }
-                        if (!relatedFound) {
-                            // Final attempt: search by name
-                            try {
-                                const collections = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                                    ? await db.listTables({ databaseId: dbId, queries: [Query.equal("name", relatedCollection)] })
-                                    : await db.listCollections(dbId, [Query.equal("name", relatedCollection)]);
-                                if (collections.total && collections.total > 0) {
-                                    const firstCollection = 'getMetadata' in db && typeof db.getMetadata === 'function'
-                                        ? collections.tables?.[0]
-                                        : collections.collections?.[0];
-                                    nameToIdMapping.set(relatedCollection, firstCollection.$id);
-                                    relatedFound = true;
-                                }
-                            }
-                            catch (_) {
-                                // Related collection truly doesn't exist yet
-                            }
-                        }
-                    }
-                    if (!relatedFound) {
-                        MessageFormatter.warning(`Related collection '${relatedCollection}' not ready for attribute '${attributeKey}', keeping in queue`);
-                        canProcess = false;
-                    }
-                }
-            }
-            if (canProcess && targetCollection) {
-                MessageFormatter.progress(`Processing queued ${operation.attribute.type} attribute: '${attributeKey}' for collection: '${targetCollection.name}'`);
-                const success = await createOrUpdateAttributeWithStatusCheck(db, dbId, targetCollection, operation.attribute);
-                if (success) {
-                    MessageFormatter.success(`Successfully processed queued attribute: '${attributeKey}'`);
-                    logger.info('Queued attribute processed successfully', {
-                        attributeKey,
-                        collectionId,
-                        targetCollectionName: targetCollection.name,
-                        operation: 'processQueue'
-                    });
-                    markAttributeProcessed(dbId, collectionId, attributeKey);
-                    queuedOperations.splice(i, 1);
-                    progress = true;
-                }
-                else {
-                    MessageFormatter.error(`Failed to process queued attribute: '${attributeKey}', removing from queue`);
-                    logger.error('Failed to process queued attribute', {
-                        attributeKey,
-                        collectionId,
-                        targetCollectionName: targetCollection.name,
-                        operation: 'processQueue'
-                    });
-                    queuedOperations.splice(i, 1);
-                }
-            }
-        }
-        if (queuedOperations.length === 0) {
-            break;
-        }
-        MessageFormatter.info(`Remaining operations after attempt ${attempts}: ${queuedOperations.length}`);
-    }
-    if (queuedOperations.length > 0) {
-        MessageFormatter.warning(`${queuedOperations.length} operations remain unresolved after ${maxAttempts} attempts:`);
-        queuedOperations.forEach((op, index) => {
-            MessageFormatter.warning(`  ${index + 1}. ${op.attribute?.type} attribute '${op.attribute?.key}' for collection ${op.collectionId}`);
-        });
-        MessageFormatter.warning("These may have unmet dependencies or require manual intervention");
-    }
-    else {
-        MessageFormatter.success("All queued operations processed successfully");
-    }
-    MessageFormatter.section(`Surgical queue processing complete for ${dbId}`);
-};

package/dist/shared/operationsTable.d.ts

@@ -1,26 +0,0 @@
-import type { DatabaseAdapter } from "../adapters/DatabaseAdapter.js";
-import { type OperationRecord } from "./operationsTableSchema.js";
-/**
- * Creates the operations tracking table in the specified database
- * Table is created with underscore prefix to indicate it's a system table
- */
-export declare function createOperationsTable(db: DatabaseAdapter, databaseId: string): Promise<void>;
-/**
- * Finds an existing operation or creates a new one
- * Useful for resuming interrupted operations
- */
-export declare function findOrCreateOperation(db: DatabaseAdapter, databaseId: string, operationType: string, params?: Partial<OperationRecord>): Promise<OperationRecord>;
-/**
- * Updates an existing operation record
- */
-export declare function updateOperation(db: DatabaseAdapter, databaseId: string, operationId: string, updates: Partial<OperationRecord>): Promise<OperationRecord>;
-/**
- * Gets a single operation by ID
- */
-export declare function getOperation(db: DatabaseAdapter, databaseId: string, operationId: string): Promise<OperationRecord | null>;
-/**
- * Cleans up old completed operations
- * @param olderThan - Optional date, defaults to operations older than 7 days
- * @returns Number of operations deleted
- */
-export declare function cleanupOperations(db: DatabaseAdapter, databaseId: string, olderThan?: Date): Promise<number>;

package/dist/shared/operationsTable.js

@@ -1,286 +0,0 @@
-import { logger } from "./logging.js";
-import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
-import { Query, ID } from "node-appwrite";
-import { OPERATIONS_TABLE_ID, OPERATIONS_TABLE_NAME, OperationRecordSchema, } from "./operationsTableSchema.js";
-/**
- * Operations Table Manager
- *
- * Manages dynamic operations tracking tables in each database.
- * Creates _appwrite_operations table on-demand for tracking imports, exports, transfers, etc.
- */
-/**
- * Checks if operations table exists in database
- */
-async function tableExists(db, databaseId) {
-    try {
-        await db.getTable({ databaseId, tableId: OPERATIONS_TABLE_ID });
-        return true;
-    }
-    catch (error) {
-        return false;
-    }
-}
-/**
- * Creates the operations tracking table in the specified database
- * Table is created with underscore prefix to indicate it's a system table
- */
-export async function createOperationsTable(db, databaseId) {
-    // Check if table already exists
-    const exists = await tableExists(db, databaseId);
-    if (exists) {
-        logger.debug("Operations table already exists", {
-            databaseId,
-            tableId: OPERATIONS_TABLE_ID,
-        });
-        return;
-    }
-    logger.info("Creating operations tracking table", {
-        databaseId,
-        tableId: OPERATIONS_TABLE_ID,
-    });
-    // Create table
-    await tryAwaitWithRetry(async () => {
-        await db.createTable({
-            databaseId,
-            id: OPERATIONS_TABLE_ID,
-            name: OPERATIONS_TABLE_NAME,
-        });
-    });
-    // Create attributes with retry logic
-    const attributes = [
-        {
-            key: "operationType",
-            type: "string",
-            size: 50,
-            required: true,
-        },
-        {
-            key: "targetCollection",
-            type: "string",
-            size: 50,
-            required: false,
-        },
-        {
-            key: "status",
-            type: "enum",
-            elements: ["pending", "in_progress", "completed", "failed", "cancelled"],
-            required: true,
-            default: "pending",
-        },
-        {
-            key: "progress",
-            type: "integer",
-            required: true,
-            default: 0,
-        },
-        {
-            key: "total",
-            type: "integer",
-            required: true,
-            default: 0,
-        },
-        {
-            key: "data",
-            type: "string",
-            size: 1048576, // 1MB for serialized data
-            required: false,
-        },
-        {
-            key: "error",
-            type: "string",
-            size: 10000,
-            required: false,
-        },
-    ];
-    for (const attr of attributes) {
-        await tryAwaitWithRetry(async () => {
-            await db.createAttribute({
-                databaseId,
-                tableId: OPERATIONS_TABLE_ID,
-                ...attr,
-            });
-        });
-    }
-    logger.info("Operations tracking table created successfully", {
-        databaseId,
-        tableId: OPERATIONS_TABLE_ID,
-        attributes: attributes.length,
-    });
-}
-/**
- * Finds an existing operation or creates a new one
- * Useful for resuming interrupted operations
- */
-export async function findOrCreateOperation(db, databaseId, operationType, params) {
-    // Ensure operations table exists
-    await createOperationsTable(db, databaseId);
-    // Try to find existing pending operation
-    try {
-        const queries = [
-            Query.equal("operationType", operationType),
-            Query.equal("status", "pending"),
-        ];
-        if (params?.targetCollection) {
-            queries.push(Query.equal("targetCollection", params.targetCollection));
-        }
-        const response = await db.listRows({
-            databaseId,
-            tableId: OPERATIONS_TABLE_ID,
-            queries,
-        });
-        if (response.rows && response.rows.length > 0) {
-            logger.debug("Found existing operation", {
-                operationId: response.rows[0].$id,
-                operationType,
-            });
-            return response.rows[0];
-        }
-    }
-    catch (error) {
-        logger.debug("No existing operation found, creating new one", {
-            operationType,
-            error: error instanceof Error ? error.message : String(error),
-        });
-    }
-    // Create new operation
-    const newOperation = {
-        operationType,
-        targetCollection: params?.targetCollection,
-        status: "pending",
-        progress: params?.progress ?? 0,
-        total: params?.total ?? 0,
-        data: params?.data ? JSON.stringify(params.data) : undefined,
-        error: params?.error,
-    };
-    const result = await db.createRow({
-        databaseId,
-        tableId: OPERATIONS_TABLE_ID,
-        id: ID.unique(),
-        data: newOperation,
-    });
-    logger.info("Created new operation", {
-        operationId: result.data?.$id,
-        operationType,
-    });
-    return result.data;
-}
-/**
- * Updates an existing operation record
- */
-export async function updateOperation(db, databaseId, operationId, updates) {
-    // Prepare update data (exclude system fields like $id, $createdAt, $updatedAt)
-    const updateData = {};
-    if (updates.operationType !== undefined)
-        updateData.operationType = updates.operationType;
-    if (updates.targetCollection !== undefined)
-        updateData.targetCollection = updates.targetCollection;
-    if (updates.status !== undefined)
-        updateData.status = updates.status;
-    if (updates.progress !== undefined)
-        updateData.progress = updates.progress;
-    if (updates.total !== undefined)
-        updateData.total = updates.total;
-    if (updates.error !== undefined)
-        updateData.error = updates.error;
-    if (updates.data !== undefined) {
-        updateData.data =
-            typeof updates.data === "string"
-                ? updates.data
-                : JSON.stringify(updates.data);
-    }
-    try {
-        const result = await db.updateRow({
-            databaseId,
-            tableId: OPERATIONS_TABLE_ID,
-            id: operationId,
-            data: updateData,
-        });
-        logger.debug("Updated operation", {
-            operationId,
-            updates: Object.keys(updateData),
-        });
-        return result.data;
-    }
-    catch (error) {
-        logger.error("Failed to update operation", {
-            operationId,
-            error: error instanceof Error ? error.message : String(error),
-        });
-        throw error;
-    }
-}
-/**
- * Gets a single operation by ID
- */
-export async function getOperation(db, databaseId, operationId) {
-    try {
-        const result = await db.getRow({
-            databaseId,
-            tableId: OPERATIONS_TABLE_ID,
-            id: operationId,
-        });
-        return result.data;
-    }
-    catch (error) {
-        logger.debug("Operation not found", {
-            operationId,
-            error: error instanceof Error ? error.message : String(error),
-        });
-        return null;
-    }
-}
-/**
- * Cleans up old completed operations
- * @param olderThan - Optional date, defaults to operations older than 7 days
- * @returns Number of operations deleted
- */
-export async function cleanupOperations(db, databaseId, olderThan) {
-    const cutoffDate = olderThan || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
-    const cutoffIso = cutoffDate.toISOString();
-    try {
-        // Query for completed operations older than cutoff
-        const response = await db.listRows({
-            databaseId,
-            tableId: OPERATIONS_TABLE_ID,
-            queries: [
-                Query.equal("status", ["completed", "failed", "cancelled"]),
-                Query.lessThan("$createdAt", cutoffIso),
-            ],
-        });
-        if (!response.rows || response.rows.length === 0) {
-            logger.debug("No old operations to clean up", { databaseId });
-            return 0;
-        }
-        // Delete in batches
-        let deletedCount = 0;
-        for (const operation of response.rows) {
-            try {
-                await db.deleteRow({
-                    databaseId,
-                    tableId: OPERATIONS_TABLE_ID,
-                    id: operation.$id,
-                });
-                deletedCount++;
-            }
-            catch (error) {
-                logger.warn("Failed to delete operation", {
-                    operationId: operation.$id,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-            }
-        }
-        logger.info("Cleaned up old operations", {
-            databaseId,
-            deletedCount,
-            cutoffDate: cutoffIso,
-        });
-        return deletedCount;
-    }
-    catch (error) {
-        logger.error("Failed to cleanup operations", {
-            databaseId,
-            error: error instanceof Error ? error.message : String(error),
-        });
-        return 0;
-    }
-}

package/dist/shared/operationsTableSchema.d.ts

@@ -1,48 +0,0 @@
-/**
- * Schema for operation tracking table (_appwrite_operations)
- *
- * This table is created dynamically in each database to track long-running operations
- * like imports, exports, transfers, and backups.
- */
-import { z } from "zod";
-export interface OperationRecord {
-    $id: string;
-    $createdAt: string;
-    $updatedAt: string;
-    operationType: string;
-    targetCollection?: string;
-    status: OperationStatus;
-    progress: number;
-    total: number;
-    data?: any;
-    error?: string;
-}
-export type OperationStatus = 'pending' | 'in_progress' | 'completed' | 'failed' | 'cancelled';
-export declare const OPERATION_STATUSES: readonly ["pending", "in_progress", "completed", "failed", "cancelled"];
-export declare const OperationStatusSchema: z.ZodEnum<{
-    pending: "pending";
-    in_progress: "in_progress";
-    completed: "completed";
-    cancelled: "cancelled";
-    failed: "failed";
-}>;
-export declare const OperationRecordSchema: z.ZodObject<{
-    $id: z.ZodString;
-    $createdAt: z.ZodString;
-    $updatedAt: z.ZodString;
-    operationType: z.ZodString;
-    targetCollection: z.ZodOptional<z.ZodString>;
-    status: z.ZodEnum<{
-        pending: "pending";
-        in_progress: "in_progress";
-        completed: "completed";
-        cancelled: "cancelled";
-        failed: "failed";
-    }>;
-    progress: z.ZodNumber;
-    total: z.ZodNumber;
-    data: z.ZodOptional<z.ZodAny>;
-    error: z.ZodOptional<z.ZodString>;
-}, z.core.$strip>;
-export declare const OPERATIONS_TABLE_ID = "_appwrite_operations";
-export declare const OPERATIONS_TABLE_NAME = "Operations Tracking";