directus-template-cli 0.7.6 → 0.8.0-partials.0

package/dist/lib/extract/expand-schema-plan.d.ts

@@ -0,0 +1,2 @@
+import { type TemplatePlan } from '../template-plan/index.js';
+export declare function expandSchemaPlan(plan: TemplatePlan): Promise<TemplatePlan>;

package/dist/lib/extract/expand-schema-plan.js

@@ -0,0 +1,55 @@
+import { readCollections, readRelations } from '@directus/sdk';
+import { ux } from '@oclif/core';
+import { api } from '../sdk.js';
+import { includesCollection } from '../template-plan/index.js';
+export async function expandSchemaPlan(plan) {
+    if (!plan.partial || !plan.collections)
+        return plan;
+    const collections = (await api.client.request(readCollections()));
+    const availableCollections = collections
+        .filter((collection) => !collection.collection.startsWith('directus_', 0))
+        .map((collection) => collection.collection)
+        .filter((collection) => includesCollection(collection, { ...plan, collections: undefined }));
+    const collectionMap = new Map(collections.map((collection) => [collection.collection, collection]));
+    const available = new Set(availableCollections);
+    const selected = new Set(plan.collections.filter((collection) => available.has(collection)));
+    const relations = (await api.client.request(readRelations()));
+    let changed = true;
+    while (changed) {
+        changed = false;
+        const candidates = [];
+        for (const collection of selected) {
+            const group = collectionMap.get(collection)?.meta?.group;
+            if (group)
+                candidates.push(group);
+        }
+        for (const relation of relations) {
+            if (selected.has(relation.collection) && relation.related_collection) {
+                candidates.push(relation.related_collection);
+            }
+            if (relation.related_collection && selected.has(relation.related_collection)) {
+                candidates.push(relation.collection);
+            }
+            if (selected.has(relation.collection) && relation.meta?.one_allowed_collections) {
+                candidates.push(...relation.meta.one_allowed_collections);
+            }
+        }
+        for (const collection of candidates) {
+            if (!available.has(collection))
+                continue;
+            if (selected.has(collection))
+                continue;
+            selected.add(collection);
+            changed = true;
+        }
+    }
+    const schemaCollections = [...selected];
+    const addedCollections = schemaCollections.filter((collection) => !plan.collections?.includes(collection));
+    if (addedCollections.length > 0) {
+        ux.warn(`Schema scope expanded collections: ${addedCollections.join(', ')}`);
+    }
+    return {
+        ...plan,
+        schemaCollections,
+    };
+}

package/dist/lib/extract/extract-assets.js

@@ -5,10 +5,13 @@ import path from 'pathe';
 import { DIRECTUS_PINK } from '../constants.js';
 import { api } from '../sdk.js';
 import catchError from '../utils/catch-error.js';
-async function getAssetList() {
-    return api.client.request(readFiles({ limit: -1 }));
+// Keep asset pages conservative because each item may trigger a binary download.
+const PAGE_SIZE = 100;
+async function getAssetPage(page) {
+    return api.client.request(readFiles({ limit: PAGE_SIZE, page }));
 }
 async function downloadFile(file, dir) {
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
     const response = await api.client.request(() => ({
         method: 'GET',
         path: `/assets/${file.id}`,
@@ -29,10 +32,20 @@ export async function downloadAllFiles(dir) {
         if (path && !fs.existsSync(fullPath)) {
             fs.mkdirSync(fullPath, { recursive: true });
         }
-        const fileList = await getAssetList();
-        await Promise.all(fileList.map(file => downloadFile(file, dir).catch(error => {
-            catchError(`Error downloading ${file.filename_disk}: ${error.message}`);
-        })));
+        let page = 1;
+        while (true) {
+            ux.action.status = `Downloading assets page ${page}`;
+            // Page asset metadata sequentially and finish each page before fetching the next, to avoid queuing all downloads at once.
+            // eslint-disable-next-line no-await-in-loop
+            const fileList = await getAssetPage(page);
+            // eslint-disable-next-line no-await-in-loop
+            await Promise.all(fileList.map((file) => downloadFile(file, dir).catch((error) => {
+                catchError(`Error downloading ${file.filename_disk}: ${error.message}`);
+            })));
+            if (fileList.length < PAGE_SIZE)
+                break;
+            page++;
+        }
     }
     catch (error) {
         catchError(error);

package/dist/lib/extract/extract-collections.d.ts

@@ -1,4 +1,5 @@
+import { type TemplatePlan } from '../template-plan/index.js';
 /**
  * Extract collections from the Directus instance
  */
-export default function extractCollections(dir: string): Promise<void>;
+export default function extractCollections(dir: string, plan?: TemplatePlan): Promise<void>;

package/dist/lib/extract/extract-collections.js

@@ -2,16 +2,19 @@ import { readCollections } from '@directus/sdk';
 import { ux } from '@oclif/core';
 import { DIRECTUS_PINK } from '../constants.js';
 import { api } from '../sdk.js';
+import { includesSchemaCollection } from '../template-plan/index.js';
 import catchError from '../utils/catch-error.js';
 import writeToFile from '../utils/write-to-file.js';
 /**
  * Extract collections from the Directus instance
  */
-export default async function extractCollections(dir) {
+export default async function extractCollections(dir, plan) {
     ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting collections'));
     try {
         const response = await api.client.request(readCollections());
-        const collections = response.filter(collection => !collection.collection.startsWith('directus_'));
+        const collections = response
+            .filter((collection) => !collection.collection.startsWith('directus_'))
+            .filter((collection) => includesSchemaCollection(collection.collection, plan));
         await writeToFile('collections', collections, dir);
     }
     catch (error) {

package/dist/lib/extract/extract-content.d.ts

@@ -1 +1,2 @@
-export declare function extractContent(dir: string): Promise<void>;
+import { type TemplatePlan, type TemplateWarning } from '../template-plan/index.js';
+export declare function extractContent(dir: string, plan?: TemplatePlan): Promise<TemplateWarning[]>;

package/dist/lib/extract/extract-content.js

@@ -1,33 +1,126 @@
-import { readCollections, readItems } from '@directus/sdk';
+import { readCollections, readItems, readRelations } from '@directus/sdk';
 import { ux } from '@oclif/core';
 import { DIRECTUS_PINK } from '../constants.js';
 import { api } from '../sdk.js';
+import { getBrokenJunctionCollections, includesCollection, } from '../template-plan/index.js';
 import catchError from '../utils/catch-error.js';
 import writeToFile from '../utils/write-to-file.js';
-async function getCollections() {
+// Content items are JSON-only, so pages can be larger than asset download pages.
+const PAGE_SIZE = 500;
+async function getCollections(relations, plan) {
     const response = await api.client.request(readCollections());
+    const brokenJunctions = getBrokenJunctionCollections(relations, plan);
     return response
-        .filter(item => !item.collection.startsWith('directus_', 0))
-        .filter(item => item.schema != null)
-        .map(i => i.collection);
+        .filter((item) => !item.collection.startsWith('directus_', 0))
+        .filter((item) => item.schema !== null)
+        .map((i) => i.collection)
+        .filter((collection) => includesCollection(collection, plan))
+        .filter((collection) => !brokenJunctions.has(collection));
 }
-async function getDataFromCollection(collection, dir) {
+async function getCollectionItems(collection) {
+    const items = [];
+    let page = 1;
+    while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const response = (await api.client.request(readItems(collection, { limit: PAGE_SIZE, page })));
+        items.push(...response);
+        if (response.length < PAGE_SIZE)
+            break;
+        page++;
+    }
+    return items;
+}
+function getExcludedRelationFields(collection, relations, plan) {
+    if (!plan?.partial || plan.relationStrategy === 'deep')
+        return [];
+    const m2oFields = relations
+        .filter((relation) => relation.collection === collection)
+        .filter((relation) => Boolean(relation.related_collection))
+        .filter((relation) => !includesCollection(relation.related_collection, plan))
+        .map((relation) => ({
+        field: relation.field,
+        relatedCollection: relation.related_collection,
+        type: 'm2o',
+    }));
+    const aliasFields = relations
+        .filter((relation) => relation.related_collection === collection)
+        .filter((relation) => Boolean(relation.meta?.one_field))
+        .filter((relation) => !includesCollection(relation.collection, plan))
+        .map((relation) => ({
+        field: relation.meta.one_field,
+        relatedCollection: relation.collection,
+        type: 'alias',
+    }));
+    return [...m2oFields, ...aliasFields];
+}
+function hasValue(value) {
+    if (Array.isArray(value))
+        return value.length > 0;
+    return value !== null && value !== undefined;
+}
+function emptyExcludedRelations(items, relations) {
+    for (const item of items) {
+        for (const relation of relations) {
+            if (!(relation.field in item))
+                continue;
+            if (relation.type === 'alias') {
+                delete item[relation.field];
+            }
+            else {
+                item[relation.field] = null;
+            }
+        }
+    }
+}
+function getBrokenRelationWarnings(collection, items, relations) {
+    return relations
+        .map((relation) => ({
+        collection,
+        count: items.filter((item) => hasValue(item[relation.field])).length,
+        field: relation.field,
+        relatedCollection: relation.relatedCollection,
+        type: 'excluded_relation',
+    }))
+        .filter((warning) => warning.count > 0);
+}
+async function getDataFromCollection(collection, dir, relations, plan) {
     try {
-        const response = await api.client.request(readItems(collection, { limit: -1 }));
+        ux.action.status = `Extracting content: ${collection}`;
+        const response = await getCollectionItems(collection);
+        const excludedRelations = getExcludedRelationFields(collection, relations, plan);
+        if (plan?.relationStrategy === 'empty') {
+            emptyExcludedRelations(response, excludedRelations);
+        }
+        const warnings = plan?.relationStrategy === 'preserve' ? getBrokenRelationWarnings(collection, response, excludedRelations) : [];
         await writeToFile(`${collection}`, response, `${dir}/content/`);
+        return warnings;
     }
     catch (error) {
-        catchError(error);
+        catchError(error, {
+            context: { collection, function: 'getDataFromCollection' },
+            fatal: true,
+        });
+        throw error;
     }
 }
-export async function extractContent(dir) {
+export async function extractContent(dir, plan) {
     ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting content'));
+    const warnings = [];
     try {
-        const collections = await getCollections();
-        await Promise.all(collections.map(collection => getDataFromCollection(collection, dir)));
+        const relations = (await api.client.request(readRelations()));
+        const collections = await getCollections(relations, plan);
+        for (const collection of collections) {
+            // eslint-disable-next-line no-await-in-loop
+            const collectionWarnings = await getDataFromCollection(collection, dir, relations, plan);
+            warnings.push(...collectionWarnings);
+        }
     }
     catch (error) {
-        catchError(error);
+        catchError(error, {
+            context: { function: 'extractContent' },
+            fatal: true,
+        });
     }
     ux.action.stop();
+    return warnings;
 }

package/dist/lib/extract/extract-fields.d.ts

@@ -1,4 +1,5 @@
+import { type TemplatePlan } from '../template-plan/index.js';
 /**
  * Extract fields from the Directus instance
  */
-export default function extractFields(dir: string): Promise<void>;
+export default function extractFields(dir: string, plan?: TemplatePlan): Promise<void>;

package/dist/lib/extract/extract-fields.js

@@ -2,12 +2,13 @@ import { readFields } from '@directus/sdk';
 import { ux } from '@oclif/core';
 import { DIRECTUS_PINK } from '../constants.js';
 import { api } from '../sdk.js';
+import { includesSchemaCollection } from '../template-plan/index.js';
 import catchError from '../utils/catch-error.js';
 import writeToFile from '../utils/write-to-file.js';
 /**
  * Extract fields from the Directus instance
  */
-export default async function extractFields(dir) {
+export default async function extractFields(dir, plan) {
     ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting fields'));
     try {
         const response = await api.client.request(readFields());
@@ -15,10 +16,9 @@ export default async function extractFields(dir) {
             throw new TypeError('Unexpected response format');
         }
         const fields = response
-            .filter(
-        // @ts-ignore
-        (i) => i.meta && !i.meta.system)
-            .map(i => {
+            .filter((i) => i.meta && !i.meta.system)
+            .filter((i) => includesSchemaCollection(i.collection, plan))
+            .map((i) => {
             if (i.meta) {
                 delete i.meta.id;
             }

package/dist/lib/extract/extract-relations.d.ts

@@ -1,4 +1,5 @@
+import { type TemplatePlan } from '../template-plan/index.js';
 /**
  * Extract relations from the Directus instance
  */
-export default function extractRelations(dir: string): Promise<void>;
+export default function extractRelations(dir: string, plan?: TemplatePlan): Promise<void>;

package/dist/lib/extract/extract-relations.js

@@ -2,12 +2,13 @@ import { readFields, readRelations } from '@directus/sdk';
 import { ux } from '@oclif/core';
 import { DIRECTUS_PINK } from '../constants.js';
 import { api } from '../sdk.js';
+import { includesRelation } from '../template-plan/index.js';
 import catchError from '../utils/catch-error.js';
 import writeToFile from '../utils/write-to-file.js';
 /**
  * Extract relations from the Directus instance
  */
-export default async function extractRelations(dir) {
+export default async function extractRelations(dir, plan) {
     ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting relations'));
     try {
         const response = await api.client.request(readRelations());
@@ -16,9 +17,10 @@ export default async function extractRelations(dir) {
         const customFields = fields.filter((i) => !i.meta?.system);
         const relations = response
             // Filter out relations where the collection starts with 'directus_' && the field is not within the customFields array
-            .filter((i) => !i.collection.startsWith('directus_', 0)
-            || customFields.some((f) => f.collection === i.collection && f.field === i.field))
-            .map(i => {
+            .filter((i) => !i.collection.startsWith('directus_', 0) ||
+            customFields.some((f) => f.collection === i.collection && f.field === i.field))
+            .filter((i) => includesRelation(i.collection, i.related_collection, plan))
+            .map((i) => {
             delete i.meta.id;
             return i;
         });

package/dist/lib/extract/index.d.ts

@@ -1 +1,2 @@
-export default function extract(dir: string): Promise<{}>;
+import { type TemplatePlan } from '../template-plan/index.js';
+export default function extract(dir: string, plan?: TemplatePlan): Promise<{}>;

package/dist/lib/extract/index.js

@@ -1,5 +1,9 @@
 import { ux } from '@oclif/core';
-import fs from 'node:fs';
+import fs from 'node:fs/promises';
+import { buildTemplatePlan, writeTemplateMetadata, } from '../template-plan/index.js';
+import catchError from '../utils/catch-error.js';
+import { expandDeepPlan } from './expand-deep-plan.js';
+import { expandSchemaPlan } from './expand-schema-plan.js';
 import extractAccess from './extract-access.js';
 import { downloadAllFiles } from './extract-assets.js';
 import extractCollections from './extract-collections.js';
@@ -19,34 +23,68 @@ import extractSchema from './extract-schema.js';
 import extractSettings from './extract-settings.js';
 import extractTranslations from './extract-translations.js';
 import extractUsers from './extract-users.js';
-export default async function extract(dir) {
-    // Get the destination directory for the actual files
+export default async function extract(dir, plan = buildTemplatePlan()) {
     const destination = `${dir}/src`;
-    // Check if directory exists, if not, then create it.
-    if (!fs.existsSync(destination)) {
-        ux.stdout(`Attempting to create directory at: ${destination}`);
-        fs.mkdirSync(destination, { recursive: true });
-    }
-    await extractSchema(destination);
-    await extractCollections(destination);
-    await extractFields(destination);
-    await extractRelations(destination);
-    await extractFolders(destination);
-    await extractFiles(destination);
-    await extractUsers(destination);
-    await extractRoles(destination);
-    await extractPermissions(destination);
-    await extractPolicies(destination);
-    await extractAccess(destination);
-    await extractPresets(destination);
-    await extractTranslations(destination);
-    await extractFlows(destination);
-    await extractOperations(destination);
-    await extractDashboards(destination);
-    await extractPanels(destination);
-    await extractSettings(destination);
-    await extractExtensions(destination);
-    await extractContent(destination);
-    await downloadAllFiles(destination);
+    const schemaPlan = await expandSchemaPlan(plan);
+    const effectivePlan = await expandDeepPlan(schemaPlan);
+    try {
+        await fs.mkdir(destination, { recursive: true });
+    }
+    catch (error) {
+        catchError(error, { context: { destination }, fatal: true });
+    }
+    if (effectivePlan.components.schema) {
+        await extractSchema(destination);
+        await extractCollections(destination, effectivePlan);
+        await extractFields(destination, effectivePlan);
+        await extractRelations(destination, effectivePlan);
+    }
+    if (effectivePlan.components.files) {
+        await extractFolders(destination);
+        await extractFiles(destination);
+        await downloadAllFiles(destination);
+    }
+    if (effectivePlan.components.users || effectivePlan.components.permissions) {
+        await extractRoles(destination);
+        await extractPermissions(destination);
+        await extractPolicies(destination);
+        if (effectivePlan.components.users) {
+            await extractUsers(destination);
+        }
+        await extractAccess(destination);
+    }
+    if (effectivePlan.components.settings) {
+        await extractPresets(destination);
+        await extractTranslations(destination);
+        await extractSettings(destination);
+    }
+    if (effectivePlan.components.flows) {
+        await extractFlows(destination);
+        await extractOperations(destination);
+    }
+    if (effectivePlan.components.dashboards) {
+        await extractDashboards(destination);
+        await extractPanels(destination);
+    }
+    if (effectivePlan.components.extensions) {
+        await extractExtensions(destination);
+    }
+    const warnings = [];
+    if (effectivePlan.components.content) {
+        const contentWarnings = await extractContent(destination, effectivePlan);
+        warnings.push(...contentWarnings);
+    }
+    for (const warning of warnings) {
+        ux.warn(`Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`);
+    }
+    try {
+        await writeTemplateMetadata(destination, effectivePlan, warnings);
+    }
+    catch (error) {
+        catchError(error, {
+            context: { function: 'writeTemplateMetadata' },
+            fatal: true,
+        });
+    }
     return {};
 }

package/dist/lib/load/apply-flags.d.ts

@@ -1,16 +1,21 @@
 export interface ApplyFlags {
+    allowBrokenRelations?: boolean;
+    collections?: string;
     content: boolean;
     dashboards: boolean;
     directusToken: string;
     directusUrl: string;
     disableTelemetry?: boolean;
+    excludeCollections?: string;
     extensions: boolean;
     files: boolean;
     flows: boolean;
+    noAssets?: boolean;
     noExit?: boolean;
     partial: boolean;
     permissions: boolean;
     programmatic: boolean;
+    relationStrategy?: 'deep' | 'empty' | 'preserve';
     schema: boolean;
     settings: boolean;
     templateLocation: string;
@@ -19,6 +24,4 @@ export interface ApplyFlags {
     userPassword: string;
     users?: boolean;
 }
-export declare const loadFlags: readonly ["content", "dashboards", "extensions", "files", "flows", "permissions", "schema", "settings", "users"];
 export declare function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags;
-export declare function validateInteractiveFlags(flags: ApplyFlags): ApplyFlags;

package/dist/lib/load/apply-flags.js

@@ -1,16 +1,4 @@
 import { ux } from '@oclif/core';
-import catchError from '../utils/catch-error.js';
-export const loadFlags = [
-    'content',
-    'dashboards',
-    'extensions',
-    'files',
-    'flows',
-    'permissions',
-    'schema',
-    'settings',
-    'users',
-];
 export function validateProgrammaticFlags(flags) {
     const { directusToken, directusUrl, templateLocation, userEmail, userPassword } = flags;
     if (!directusUrl)
@@ -19,43 +7,5 @@ export function validateProgrammaticFlags(flags) {
         ux.error('Either Directus token or email and password are required for programmatic mode.');
     if (!templateLocation)
         ux.error('Template location is required for programmatic mode.');
-    return flags.partial ? handlePartialFlags(flags) : setAllFlagsTrue(flags);
-}
-export function validateInteractiveFlags(flags) {
-    return flags.partial ? handlePartialFlags(flags) : setAllFlagsTrue(flags);
-}
-function handlePartialFlags(flags) {
-    const enabledFlags = loadFlags.filter(flag => flags[flag] === true);
-    const disabledFlags = loadFlags.filter(flag => flags[flag] === false);
-    if (enabledFlags.length > 0) {
-        for (const flag of loadFlags)
-            flags[flag] = enabledFlags.includes(flag);
-    }
-    else if (disabledFlags.length > 0) {
-        for (const flag of loadFlags)
-            flags[flag] = !disabledFlags.includes(flag);
-    }
-    else {
-        setAllFlagsTrue(flags);
-    }
-    handleDependencies(flags);
-    if (!loadFlags.some(flag => flags[flag])) {
-        catchError(new Error('When using --partial, at least one component must be loaded.'), { fatal: true });
-    }
-    return flags;
-}
-function handleDependencies(flags) {
-    if (flags.content && (!flags.schema || !flags.files)) {
-        flags.schema = flags.files = true;
-        ux.warn('Content loading requires schema and files. Enabling schema and files flags.');
-    }
-    if (flags.users && !flags.permissions) {
-        flags.permissions = true;
-        ux.warn('User loading requires permissions. Enabling permissions flag.');
-    }
-}
-function setAllFlagsTrue(flags) {
-    for (const flag of loadFlags)
-        flags[flag] = true;
     return flags;
 }

package/dist/lib/load/finalize-collections.d.ts

@@ -0,0 +1,2 @@
+import { type TemplatePlan } from '../template-plan/index.js';
+export default function finalizeCollections(dir: string, plan?: TemplatePlan): Promise<void>;

package/dist/lib/load/finalize-collections.js

@@ -0,0 +1,28 @@
+import { updateCollection } from '@directus/sdk';
+import { ux } from '@oclif/core';
+import { DIRECTUS_PINK } from '../constants.js';
+import { api } from '../sdk.js';
+import { includesSchemaCollection } from '../template-plan/index.js';
+import catchError from '../utils/catch-error.js';
+import readFile from '../utils/read-file.js';
+export default async function finalizeCollections(dir, plan) {
+    const collections = readFile('collections', dir)
+        .filter((collection) => includesSchemaCollection(collection.collection, plan))
+        .filter((collection) => !collection.collection.startsWith('directus_'));
+    ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${collections.length} collections`));
+    const collectionNames = new Set(collections.map((collection) => collection.collection));
+    for await (const collection of collections) {
+        const meta = { ...collection.meta };
+        if (meta.group && !collectionNames.has(meta.group)) {
+            ux.warn(`Skipping missing group "${meta.group}" for collection "${collection.collection}"`);
+            delete meta.group;
+        }
+        try {
+            await api.client.request(updateCollection(collection.collection, { meta }));
+        }
+        catch (error) {
+            catchError(error, { context: { collection: collection.collection, group: meta.group } });
+        }
+    }
+    ux.action.stop();
+}

package/dist/lib/load/finalize-fields.d.ts

@@ -0,0 +1,2 @@
+import { type TemplatePlan } from '../template-plan/index.js';
+export default function finalizeFields(dir: string, plan?: TemplatePlan): Promise<void>;

package/dist/lib/load/finalize-fields.js

@@ -0,0 +1,25 @@
+import { updateField } from '@directus/sdk';
+import { ux } from '@oclif/core';
+import { DIRECTUS_PINK } from '../constants.js';
+import { api } from '../sdk.js';
+import { includesSchemaCollection } from '../template-plan/index.js';
+import catchError from '../utils/catch-error.js';
+import readFile from '../utils/read-file.js';
+export default async function finalizeFields(dir, plan) {
+    const fields = readFile('fields', dir)
+        .filter((field) => includesSchemaCollection(field.collection, plan))
+        .filter((field) => field.schema);
+    ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${fields.length} fields`));
+    for await (const field of fields) {
+        try {
+            await api.client.request(updateField(field.collection, field.field, {
+                meta: field.meta ? { ...field.meta } : undefined,
+                schema: field.schema ? { ...field.schema } : undefined,
+            }));
+        }
+        catch (error) {
+            catchError(error, { context: { collection: field.collection, field: field.field } });
+        }
+    }
+    ux.action.stop();
+}