great 1.0.2 → 1.2.0

package/dist/push/index.js

@@ -1,17 +1,20 @@
 import fs from 'node:fs';
-import path from 'node:path';
 import chalk from 'chalk';
+import parseJson from '../utils/parseJson.js';
 import { sentenceCase } from 'change-case';
-import { fetchIdFromFile, fetchIntegrationFromFile, } from '../utils/fetchLocal.js';
+import { fetchIntegrationFromFile } from '../utils/fetchLocal.js';
+import { readGreatFile, writeGreatFile, getElementRecord, setElementRecord, removeElementRecordById, sameProvenance, } from '../utils/greatFile.js';
+import { readElementFolder, folderToElementType, } from '../utils/elementFiles.js';
 import createIntegrationFromFile from '../utils/createIntegrationFromFile.js';
-import { createGraphQLClient, } from '../utils/graphqlClient.js';
-import { handleGraphQLError } from '../utils/graphqlError.js';
-import { getApiBaseUrl } from '../utils/config.js';
+import { getClient } from '../utils/graphqlClient.js';
+import { getGraphQLErrorMessage, handleGraphQLError, } from '../utils/graphqlError.js';
 import hashContent from '../utils/hashContent.js';
 import { VALID_VALUE_TYPES, isValidValueElementType, } from '../utils/valueTypes.js';
-import { setValueForElement } from '../value/set.js';
+import { setValueForElement } from '../values/set.js';
 import { promptForValueSetting } from '../utils/promptForValue.js';
-import { ElementType, } from '../generated/graphql.js';
+import { SHADOW_MARKER } from '../utils/writeShadowElement.js';
+import resolveIntegrationType from '../utils/integrationType.js';
+import { ElementType } from '../generated/schema.js';
 import { gql } from 'graphql-tag';
 const IntegrationForPushQueryDocument = gql `
   query IntegrationForPush($id: ID!) {
@@ -19,6 +22,14 @@ const IntegrationForPushQueryDocument = gql `
       id
       key
       name
+      type
+      templateImports {
+        id
+        timestamp
+        template {
+          id
+        }
+      }
       latest {
         timestamp
         elements {
@@ -30,13 +41,23 @@ const IntegrationForPushQueryDocument = gql `
           createdAt
           content {
             checksum
-            timestamp
+            createdAt
+            sourceTimestamp
+            provenance
           }
         }
       }
     }
   }
 `;
+const UpdateIntegrationTypeMutationDocument = gql `
+  mutation UpdateIntegrationType($id: ID!, $type: IntegrationType!) {
+    updateIntegrationType(id: $id, type: $type) {
+      id
+      type
+    }
+  }
+`;
 const CreateElementMutationDocument = gql `
   mutation CreateElement(
     $integrationId: ID!
@@ -57,6 +78,52 @@ const CreateElementMutationDocument = gql `
     }
   }
 `;
+const IncludeElementMutationDocument = gql `
+  mutation IncludeElement(
+    $integrationId: ID!
+    $elementId: ID!
+    $sourceTimestamp: DateTime
+    $provenance: [ID!]
+  ) {
+    includeElement(
+      integrationId: $integrationId
+      elementId: $elementId
+      sourceTimestamp: $sourceTimestamp
+      provenance: $provenance
+    ) {
+      id
+      name
+      description
+      content {
+        data
+        checksum
+        createdAt
+      }
+    }
+  }
+`;
+const RegisterTemplateImportMutationDocument = gql `
+  mutation RegisterTemplateImport(
+    $integrationId: ID!
+    $templateId: ID!
+    $timestamp: DateTime!
+  ) {
+    registerTemplateImport(
+      integrationId: $integrationId
+      templateId: $templateId
+      timestamp: $timestamp
+    ) {
+      id
+      templateImports {
+        id
+        timestamp
+        template {
+          id
+        }
+      }
+    }
+  }
+`;
 const UpdateElementMutationDocument = gql `
   mutation UpdateElement($id: ID!, $name: String, $description: String) {
     updateElement(id: $id, name: $name, description: $description) {
@@ -71,7 +138,7 @@ const SetElementContentMutationDocument = gql `
     setElementContent(id: $id, content: $content) {
       id
       content {
-        timestamp
+        createdAt
         checksum
       }
     }
@@ -85,38 +152,16 @@ const DeleteElementMutationDocument = gql `
     }
   }
 `;
-const folderToElementType = {
-    schemas: ElementType.Schema,
-    services: ElementType.Service,
-    auths: ElementType.Auth,
-    mutations: ElementType.Mutation,
-    dictionaries: ElementType.Dictionary,
-    jobs: ElementType.Job,
-    values: ElementType.Value,
-    flows: ElementType.Flow,
-    retryRules: ElementType.RetryRule,
-};
+const SetSecretRefMutationDocument = gql `
+  mutation SetSecretRef($id: ID!, $secretRef: String!) {
+    setSecretRef(id: $id, secretRef: $secretRef)
+  }
+`;
 function readLocalElements(fsImpl) {
     const elements = [];
     for (const folder of Object.keys(folderToElementType)) {
-        if (!fsImpl.existsSync(folder)) {
-            continue;
-        }
-        const files = fsImpl.readdirSync(folder);
-        for (const file of files) {
-            if (!file.endsWith('.json')) {
-                continue;
-            }
-            const key = file.replace('.json', '');
-            const filePath = path.join(folder, file);
-            try {
-                const content = fsImpl.readFileSync(filePath, 'utf8');
-                const data = JSON.parse(content);
-                elements.push({ folder, key, filePath, data });
-            }
-            catch (error) {
-                throw new Error(`Failed to read or parse ${filePath}: ${error}`);
-            }
+        for (const { key, filePath, data } of readElementFolder(folder, undefined, fsImpl)) {
+            elements.push({ folder, key, filePath, data });
         }
     }
     return elements;
@@ -125,15 +170,32 @@ function compareLocalAndRemote(local, remote) {
     if (!remote) {
         return {
             category: 'conflict',
-            reason: 'Element exists locally with __id but not found in Integreat',
+            reason: 'Element exists locally with an id but not found in Integreat',
         };
     }
-    const localChecksum = hashContent(local.data);
     const localName = local.data.name || '';
     const localDesc = local.data.description || '';
-    const localTimestamp = typeof local.data.__timestamp === 'string'
-        ? new Date(local.data.__timestamp)
-        : undefined;
+    if (local.source) {
+        const hasNameChanged = localName !== (remote.name || '');
+        const hasDescriptionChanged = localDesc !== (remote.description || '');
+        const hasPinChanged = (local.source.timestamp ?? null) !== (remote.sourceTimestamp ?? null);
+        const hasProvenanceChanged = !sameProvenance(local.provenance, remote.provenance);
+        if (!hasNameChanged &&
+            !hasDescriptionChanged &&
+            !hasPinChanged &&
+            !hasProvenanceChanged) {
+            return { category: 'unchanged' };
+        }
+        if (hasPinChanged && local.timestamp !== remote.contentCreatedAt) {
+            return {
+                category: 'conflict',
+                reason: '  Shadow pin changed both local and remote',
+            };
+        }
+        return { category: 'update' };
+    }
+    const localChecksum = hashContent(local.data);
+    const localTimestamp = local.timestamp ? new Date(local.timestamp) : undefined;
     const remoteChecksum = remote.checksum || '';
     const remoteName = remote.name || '';
     const remoteDesc = remote.description || '';
@@ -148,7 +210,7 @@ function compareLocalAndRemote(local, remote) {
     }
     const metaDateConflict = localTimestamp && remoteCreatedAt && localTimestamp < remoteCreatedAt;
     const conflicts = [];
-    if (hasContentChanged && local.data.__timestamp !== remote.timestamp) {
+    if (hasContentChanged && local.timestamp !== remote.contentCreatedAt) {
         conflicts.push('Content changed both local and remote');
     }
     if (hasNameChanged && metaDateConflict) {
@@ -169,12 +231,17 @@ function compareLocalAndRemote(local, remote) {
 }
 function categorizeElements(localElements, remoteElementsMap) {
     const toCreate = [];
+    const toInclude = [];
     const toUpdate = [];
     const toDelete = [];
     const conflicts = [];
     const localElementIds = new Set();
     for (const local of localElements) {
-        const localId = local.data.__id;
+        const localId = local.id;
+        if (local.source && !localId) {
+            toInclude.push(local);
+            continue;
+        }
         if (!localId) {
             toCreate.push(local);
             continue;
@@ -194,11 +261,11 @@ function categorizeElements(localElements, remoteElementsMap) {
             toDelete.push(remoteLement);
         }
     }
-    return { toCreate, toUpdate, toDelete, conflicts };
+    return { toCreate, toInclude, toUpdate, toDelete, conflicts };
 }
-const prepareContentForApi = ({ __id, __key, __timestamp, name, description, ...content }, elementType) => {
+const prepareContentForApi = ({ name, description, ...content }, elementType) => {
     if (elementType === ElementType.Value) {
-        const { value, type, ...rest } = content;
+        const { value, ref, type, ...rest } = content;
         if (type === undefined) {
             throw new Error('Value element must have a "type" field. ' +
                 'Valid types: ' +
@@ -233,6 +300,65 @@ async function createElementOnApi(client, integrationId, element) {
     }
     return { id, name };
 }
+function writeShadowFileAndRecord(fsImpl, element, great, source, included) {
+    const name = element.data.name || included.name;
+    const description = element.data.description ||
+        included.description ||
+        undefined;
+    const content = included.content;
+    const fileData = {
+        [SHADOW_MARKER]: true,
+        ...(name ? { name } : {}),
+        ...(description ? { description } : {}),
+        ...content?.data,
+    };
+    fsImpl.writeFileSync(element.filePath, JSON.stringify(fileData, null, 2));
+    setElementRecord(great, element.folder, element.key, {
+        id: included.id,
+        ...(content?.createdAt ? { timestamp: content.createdAt } : {}),
+        ...(content?.checksum ? { hash: content.checksum } : {}),
+        source,
+        ...(element.provenance && element.provenance.length > 0
+            ? { provenance: element.provenance }
+            : {}),
+    });
+}
+async function includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great) {
+    const source = element.source;
+    if (!source) {
+        return false;
+    }
+    const provenance = element.provenance && element.provenance.length > 0
+        ? element.provenance
+        : undefined;
+    const result = await graphqlClient.request(IncludeElementMutationDocument, {
+        integrationId,
+        elementId: source.id,
+        sourceTimestamp: source.timestamp,
+        provenance,
+    });
+    if (result.error || !result.data?.includeElement) {
+        log(chalk.red(`Failed to include shadow element ${element.key} in ${element.folder}`));
+        return false;
+    }
+    const included = result.data.includeElement;
+    const localName = element.data.name || '';
+    const localDesc = element.data.description || '';
+    const nameDiffers = localName !== (included.name || '');
+    const descDiffers = localDesc !== (included.description || '');
+    if (nameDiffers || descDiffers) {
+        const updateResult = await graphqlClient.request(UpdateElementMutationDocument, {
+            id: included.id,
+            name: nameDiffers ? localName : undefined,
+            description: descDiffers ? localDesc : undefined,
+        });
+        if (updateResult.error) {
+            log(chalk.yellow(`Warning: Failed to update metadata for shadow ${element.key} in ${element.folder}`));
+        }
+    }
+    writeShadowFileAndRecord(fsImpl, element, great, source, included);
+    return true;
+}
 async function updateElementContent(client, elementId, content) {
     const result = await client.request(SetElementContentMutationDocument, {
         id: elementId,
@@ -241,25 +367,36 @@ async function updateElementContent(client, elementId, content) {
     if (result.error) {
         return undefined;
     }
-    return result.data?.setElementContent?.content?.timestamp;
+    return result.data?.setElementContent?.content?.createdAt;
 }
-function updateLocalFile(fsImpl, filePath, updatedId, updatedTimestamp, generatedName, keepValue = false) {
-    const content = fsImpl.readFileSync(filePath, 'utf8');
-    const parsed = JSON.parse(content);
-    const { __id, __key, __timestamp, name, description, value, ...data } = parsed;
-    const id = updatedId || __id;
-    const timestamp = updatedTimestamp || __timestamp;
+function updateLocalFile(fsImpl, element, great, updatedId, updatedTimestamp, generatedName, keepValue = false) {
+    const content = fsImpl.readFileSync(element.filePath, 'utf8');
+    const parsed = parseJson(content);
+    const { name, description, value, ref, ...data } = parsed;
+    const existing = getElementRecord(great, element.folder, element.key);
+    const id = updatedId ?? existing?.id;
+    const timestamp = updatedTimestamp ?? existing?.timestamp;
     const finalName = generatedName || name;
     const orderedData = {
-        ...(id ? { __id: id } : {}),
-        __key,
-        ...(timestamp ? { __timestamp: timestamp } : {}),
         ...(finalName ? { name: finalName } : {}),
         ...(description ? { description } : {}),
         ...data,
         ...(keepValue && value !== undefined ? { value } : {}),
+        ...(keepValue && ref !== undefined ? { ref } : {}),
     };
-    fsImpl.writeFileSync(filePath, JSON.stringify(orderedData, null, 2));
+    fsImpl.writeFileSync(element.filePath, JSON.stringify(orderedData, null, 2));
+    setElementRecord(great, element.folder, element.key, {
+        ...(id ? { id } : {}),
+        ...(timestamp ? { timestamp } : {}),
+        hash: hashContent(orderedData),
+    });
+}
+function recordElementProgress(great, element, fields) {
+    const existing = getElementRecord(great, element.folder, element.key);
+    setElementRecord(great, element.folder, element.key, {
+        ...existing,
+        ...fields,
+    });
 }
 function buildRemoteElementsMap(elements) {
     const remoteElements = new Map();
@@ -267,10 +404,12 @@ function buildRemoteElementsMap(elements) {
         remoteElements.set(element.id, {
             id: element.id,
             checksum: element.content?.checksum,
-            timestamp: element.content?.timestamp,
+            contentCreatedAt: element.content?.createdAt,
             name: element.name,
             description: element.description,
             createdAt: element.createdAt,
+            sourceTimestamp: element.content?.sourceTimestamp,
+            provenance: element.content?.provenance,
         });
     }
     return remoteElements;
@@ -283,7 +422,7 @@ function handleConflicts(conflicts, log) {
     }
     log(`\nRun ${chalk.bold('great pull')} to get remote changes, then manually merge and push again.`);
 }
-async function createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue) {
+async function createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue, great) {
     let id = element.remoteId;
     let generatedName;
     if (!id) {
@@ -294,6 +433,7 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
         }
         id = result.id;
         generatedName = result.name;
+        recordElementProgress(great, element, { id });
     }
     else {
         const apiElement = apiElementsMap.get(id);
@@ -316,6 +456,15 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
             }
         }
     }
+    if (element.source) {
+        const remote = apiElementsMap.get(id);
+        const pinChanged = (element.source.timestamp ?? null) !== (remote?.sourceTimestamp ?? null);
+        const provenanceChanged = !sameProvenance(element.provenance, remote?.provenance);
+        if (pinChanged || provenanceChanged) {
+            return includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great);
+        }
+        return true;
+    }
     const elementType = folderToElementType[element.folder];
     let content;
     try {
@@ -330,17 +479,31 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
         log(chalk.red(`Failed to ${element.remoteId ? 'update' : 'set'} content for ${element.key} in ${element.folder}`));
         return false;
     }
+    recordElementProgress(great, element, { id, timestamp });
     const hasValue = element.data.value !== undefined;
-    if (elementType === ElementType.Value && hasValue && shouldSetValue) {
-        const valueStr = String(element.data.value);
-        const result = await setValueForElement(id, valueStr, content, graphqlClient);
-        if (!result.success) {
-            log(chalk.red(`Failed to set value for ${element.key}: ${result.error}`));
-            return false;
+    const hasRef = element.data.ref !== undefined;
+    const isSecret = content.type === 'secret';
+    if (elementType === ElementType.Value && shouldSetValue) {
+        if (isSecret && hasRef && !hasValue) {
+            const secretRef = String(element.data.ref);
+            const result = await graphqlClient.request(SetSecretRefMutationDocument, { id, secretRef });
+            if (result.error) {
+                log(chalk.red(`Failed to set secret ref for ${element.key}: ${getGraphQLErrorMessage(result.error)}`));
+                return false;
+            }
+            log(chalk.green(`✓ Secret ref set for element "${element.key}"`));
+        }
+        else if (hasValue) {
+            const valueStr = String(element.data.value);
+            const result = await setValueForElement(id, valueStr, content, graphqlClient);
+            if (!result.success) {
+                log(chalk.red(`Failed to set value for ${element.key}: ${result.error}`));
+                return false;
+            }
+            log(chalk.green(`✓ Value set for element "${element.key}" (${content.type})`));
         }
-        log(chalk.green(`✓ Value set for element "${element.key}" (${content.type})`));
     }
-    updateLocalFile(fsImpl, element.filePath, element.remoteId ? undefined : id, timestamp, generatedName, !shouldSetValue && hasValue);
+    updateLocalFile(fsImpl, element, great, element.remoteId ? undefined : id, timestamp, generatedName, !shouldSetValue && hasValue);
     return true;
 }
 async function deleteElement(element, graphqlClient, log) {
@@ -353,29 +516,96 @@ async function deleteElement(element, graphqlClient, log) {
     }
     return true;
 }
-function logSummary(createdCount, updatedCount, deletedCount, log) {
-    const total = createdCount + updatedCount + deletedCount;
+function buildRemoteImportsMap(templateImports) {
+    const map = new Map();
+    for (const templateImport of templateImports) {
+        if (templateImport.template) {
+            map.set(templateImport.template.id, templateImport);
+        }
+    }
+    return map;
+}
+function findImportsToRegister(great, remoteImports) {
+    return (great.imports ?? []).filter((record) => {
+        const remote = remoteImports.get(record.templateId);
+        return !remote || remote.timestamp !== record.timestamp;
+    });
+}
+function rebuildImportRecords(great, templateImports) {
+    const records = [];
+    for (const templateImport of templateImports) {
+        if (templateImport.template) {
+            records.push({
+                templateId: templateImport.template.id,
+                timestamp: templateImport.timestamp,
+                id: templateImport.id,
+            });
+        }
+    }
+    if (records.length > 0 || great.imports?.length) {
+        great.imports = records;
+    }
+}
+async function registerTemplateImports(pending, initialRemoteImports, graphqlClient, integrationId, great, log) {
+    let latestRemoteImports = initialRemoteImports;
+    let registeredCount = 0;
+    for (const record of pending) {
+        const result = await graphqlClient.request(RegisterTemplateImportMutationDocument, {
+            integrationId,
+            templateId: record.templateId,
+            timestamp: record.timestamp,
+        });
+        if (result.error || !result.data?.registerTemplateImport) {
+            log(chalk.red(`Failed to register template import ${record.templateId}`));
+            return undefined;
+        }
+        latestRemoteImports = result.data.registerTemplateImport.templateImports;
+        log(chalk.green(`✓ Registered template import ${record.templateId}`));
+        registeredCount++;
+    }
+    rebuildImportRecords(great, latestRemoteImports);
+    return registeredCount;
+}
+async function syncIntegrationType(graphqlClient, integrationId, remoteType, localType, log) {
+    if (localType === remoteType) {
+        return;
+    }
+    const result = await graphqlClient.request(UpdateIntegrationTypeMutationDocument, { id: integrationId, type: localType });
+    if (result.error) {
+        handleGraphQLError(result.error, log);
+        return;
+    }
+    log(`${chalk.green('✓')} Updated integration type to ${chalk.cyan(localType)}`);
+}
+function logSummary(createdCount, includedCount, updatedCount, deletedCount, registeredCount, log) {
+    const total = createdCount + includedCount + updatedCount + deletedCount + registeredCount;
     if (total > 0) {
         const parts = [];
         if (createdCount > 0)
             parts.push(`${createdCount} created`);
+        if (includedCount > 0)
+            parts.push(`${includedCount} included`);
         if (updatedCount > 0)
             parts.push(`${updatedCount} updated`);
         if (deletedCount > 0)
             parts.push(`${deletedCount} deleted`);
+        if (registeredCount > 0)
+            parts.push(`${registeredCount} template imports registered`);
         log(`\n${chalk.green('✓')} ${parts.join(', ')}`);
     }
 }
 export const action = (log, apiToken, client, fsImpl = fs, config, promptForValueSettingFn = promptForValueSetting) => async function pushIntegration(options) {
-    let integrationId = fetchIdFromFile(fsImpl);
+    const great = readGreatFile(undefined, fsImpl);
+    let integrationId = great.id;
     if (!integrationId) {
-        log(chalk.yellow('No integration ID found in integration.json'));
+        log(chalk.yellow('No integration ID found'));
         log('Creating integration on server...\n');
-        const apiBaseUrl = getApiBaseUrl(config);
-        const graphqlClient = client ?? createGraphQLClient(apiToken, apiBaseUrl);
+        const graphqlClient = getClient(apiToken, client, config);
         const result = await createIntegrationFromFile(graphqlClient, apiToken ?? '', options ?? {}, config, fsImpl);
         if (!result.success || !result.integration) {
-            log(chalk.red(result.error));
+            if (result.error) {
+                handleGraphQLError(result.error, log);
+            }
             return;
         }
         const { integration } = result;
@@ -390,27 +620,26 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
             return;
         }
         const updatedData = {
-            __id: integration.id,
-            __key: integration.key,
             name: integration.name,
             description: fileData.description,
             status: integration.status,
-            account: {
-                __id: integration.account.id,
-                __key: integration.account.key,
-                name: integration.account.name,
-            },
+            type: integration.type,
             flags: fileData.flags,
             identConfig: fileData.identConfig,
             nonvalues: fileData.nonvalues,
             queueService: fileData.queueService,
         };
         fsImpl.writeFileSync('integration.json', JSON.stringify(updatedData, null, 2));
+        great.id = integration.id;
+        great.account = {
+            id: integration.account.id,
+            key: integration.account.key,
+        };
+        writeGreatFile(great, undefined, fsImpl);
         integrationId = integration.id;
         log(chalk.green('✓ Updated integration.json with server details\n'));
     }
-    const apiBaseUrl = getApiBaseUrl(config);
-    const graphqlClient = client ?? createGraphQLClient(apiToken, apiBaseUrl);
+    const graphqlClient = getClient(apiToken, client, config);
     const result = await graphqlClient.request(IntegrationForPushQueryDocument, { id: integrationId });
     if (result.error) {
         handleGraphQLError(result.error, log);
@@ -421,6 +650,8 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
         log(chalk.red('Integration has no latest version'));
         return;
     }
+    const localData = fetchIntegrationFromFile(fsImpl);
+    await syncIntegrationType(graphqlClient, integrationId, resolveIntegrationType(integration.type), resolveIntegrationType(localData?.type, log), log);
     const apiElementsMap = buildRemoteElementsMap(integration.latest.elements);
     let localElements;
     try {
@@ -430,12 +661,26 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
         log(chalk.red(`Error reading local files: ${error}`));
         return;
     }
-    const { toCreate, toUpdate, toDelete, conflicts } = categorizeElements(localElements, apiElementsMap);
+    for (const local of localElements) {
+        const record = getElementRecord(great, local.folder, local.key);
+        local.id = record?.id;
+        local.timestamp = record?.timestamp;
+        local.source = record?.source;
+        local.provenance = record?.provenance;
+    }
+    const { toCreate, toInclude, toUpdate, toDelete, conflicts } = categorizeElements(localElements, apiElementsMap);
     if (conflicts.length > 0) {
         handleConflicts(conflicts, log);
         return;
     }
-    if (toCreate.length + toUpdate.length + toDelete.length === 0) {
+    const remoteImportsMap = buildRemoteImportsMap(integration.templateImports ?? []);
+    const importsToRegister = findImportsToRegister(great, remoteImportsMap);
+    if (toCreate.length +
+        toInclude.length +
+        toUpdate.length +
+        toDelete.length +
+        importsToRegister.length ===
+        0) {
         log(chalk.green('No changes to push'));
         return;
     }
@@ -447,31 +692,57 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
         log(chalk.yellow('\nPush cancelled'));
         return;
     }
+    const persistProgress = () => writeGreatFile(great, undefined, fsImpl);
     let createdCount = 0;
     for (const element of toCreate) {
-        const success = await createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue);
-        if (!success)
+        const success = await createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue, great);
+        if (!success) {
+            persistProgress();
             return;
+        }
         log(chalk.green(`✓ Created ${element.folder}/${element.key}`));
         createdCount++;
     }
+    let includedCount = 0;
+    for (const element of toInclude) {
+        const success = await includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great);
+        if (!success) {
+            persistProgress();
+            return;
+        }
+        log(chalk.green(`✓ Included ${element.folder}/${element.key}`));
+        includedCount++;
+    }
     let updatedCount = 0;
     for (const element of toUpdate) {
-        const success = await createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue);
-        if (!success)
+        const success = await createOrUpdateElement(element, graphqlClient, integrationId, fsImpl, log, apiElementsMap, shouldSetValue, great);
+        if (!success) {
+            persistProgress();
             return;
+        }
         log(chalk.green(`✓ Updated ${element.folder}/${element.key}`));
         updatedCount++;
     }
     let deletedCount = 0;
     for (const element of toDelete) {
         const success = await deleteElement(element, graphqlClient, log);
-        if (!success)
+        if (!success) {
+            persistProgress();
             return;
+        }
+        removeElementRecordById(great, element.id);
         log(chalk.green(`✓ Deleted ${element.id}`));
         deletedCount++;
     }
-    logSummary(createdCount, updatedCount, deletedCount, log);
+    const registeredCount = await registerTemplateImports(importsToRegister, integration.templateImports ?? [], graphqlClient, integrationId, great, log);
+    if (registeredCount === undefined) {
+        persistProgress();
+        return;
+    }
+    const localKeys = new Set(localElements.map((el) => `${el.folder}/${el.key}`));
+    great.elements = Object.fromEntries(Object.entries(great.elements).filter(([key]) => localKeys.has(key)));
+    writeGreatFile(great, undefined, fsImpl);
+    logSummary(createdCount, includedCount, updatedCount, deletedCount, registeredCount, log);
 };
 export default (program, apiToken) => program
     .command('push')