great 1.3.0 → 1.4.1

package/dist/push/index.js

@@ -2,7 +2,7 @@ import fs from 'node:fs';
 import chalk from 'chalk';
 import { sentenceCase } from 'change-case';
 import { fetchIntegrationFromFile } from '../utils/fetchLocal.js';
-import { readGreatFile, writeGreatFile, getElementRecord, setElementRecord, removeElementRecordById, sameProvenance, } from '../utils/greatFile.js';
+import { readGreatFile, writeGreatFile, getElementRecord, setElementRecord, removeElementRecord, removeElementRecordById, sameProvenance, } from '../utils/greatFile.js';
 import { readWorkspaceElements, writeElementFile, folderToElementType, } from '../utils/elementFiles.js';
 import { ENDPOINT_LISTS_FOLDER, validateEndpointReferences, } from '../utils/serviceFile.js';
 import { promptForDetach } from '../utils/promptForDetach.js';
@@ -14,8 +14,9 @@ import { getGraphQLErrorMessage, handleGraphQLError, isNotFoundError, } from '..
 import hashContent from '../utils/hashContent.js';
 import { VALID_VALUE_TYPES, isValidValueElementType, } from '../utils/valueTypes.js';
 import { setValueForElement } from '../values/set.js';
-import { promptForValueSetting } from '../utils/promptForValue.js';
+import { promptForValueSetting, hasValueToSet, } from '../utils/promptForValue.js';
 import resolveIntegrationType from '../utils/integrationType.js';
+import { CONFIG_KEY, CONFIG_RECORD_FOLDER, buildConfigRecord, findConfigElement, flagsFromIntegration, hashFlags, } from '../utils/integrationConfig.js';
 import { ElementType } from '../generated/schema.js';
 import { gql } from 'graphql-tag';
 const IntegrationForPushQueryDocument = gql `
@@ -44,6 +45,7 @@ const IntegrationForPushQueryDocument = gql `
           content {
             checksum
             createdAt
+            sourceId
             sourceTimestamp
             provenance
           }
@@ -136,8 +138,8 @@ const UpdateElementMutationDocument = gql `
   }
 `;
 const SetElementContentMutationDocument = gql `
-  mutation SetElementContent($id: ID!, $content: JSON) {
-    setElementContent(id: $id, content: $content) {
+  mutation SetElementContent($id: ID!, $content: JSON, $provenance: [ID!]) {
+    setElementContent(id: $id, content: $content, provenance: $provenance) {
       id
       content {
         createdAt
@@ -168,6 +170,7 @@ function compareShadow(local, remote, source) {
     const isList = isEndpointList(local);
     const hasNameChanged = !isList && localName !== (remote.name || '');
     const hasDescriptionChanged = !isList && localDesc !== (remote.description || '');
+    const hasSourceChanged = source.id !== remote.sourceId;
     const hasPinChanged = (source.timestamp ?? null) !== (remote.sourceTimestamp ?? null);
     const hasProvenanceChanged = !sameProvenance(local.provenance, remote.provenance);
     const hasRemoteMoved = local.timestamp !== remote.contentCreatedAt;
@@ -181,11 +184,18 @@ function compareShadow(local, remote, source) {
     }
     if (!hasNameChanged &&
         !hasDescriptionChanged &&
+        !hasSourceChanged &&
         !hasPinChanged &&
         !hasProvenanceChanged) {
         return { category: 'unchanged' };
     }
-    if (hasPinChanged && hasRemoteMoved) {
+    if (hasSourceChanged && hasRemoteMoved) {
+        return {
+            category: 'conflict',
+            reason: '  Shadow source changed both local and remote',
+        };
+    }
+    else if (hasPinChanged && hasRemoteMoved) {
         return {
             category: 'conflict',
             reason: '  Shadow pin changed both local and remote',
@@ -214,7 +224,9 @@ function compareLocalAndRemote(local, remote) {
     const remoteCreatedAt = remote.createdAt
         ? new Date(remote.createdAt)
         : undefined;
-    const hasContentChanged = localChecksum !== remoteChecksum;
+    const hasContentChanged = localChecksum !== remoteChecksum ||
+        Boolean(remote.sourceId) ||
+        !sameProvenance(local.provenance, remote.provenance);
     const hasNameChanged = !isList && localName !== remoteName;
     const hasDescriptionChanged = !isList && localDesc !== remoteDesc;
     if (!hasContentChanged && !hasNameChanged && !hasDescriptionChanged) {
@@ -246,6 +258,7 @@ function categorizeElements(localElements, remoteElementsMap) {
     const toInclude = [];
     const toUpdate = [];
     const toDetach = [];
+    const toSetValue = [];
     const toDelete = [];
     const conflicts = [];
     const localElementIds = new Set();
@@ -271,6 +284,9 @@ function categorizeElements(localElements, remoteElementsMap) {
         else if (result.category === 'detach') {
             toDetach.push({ ...local, remoteId: remote?.id });
         }
+        else if (hasValueToSet(local)) {
+            toSetValue.push({ ...local, remoteId: remote?.id });
+        }
     }
     for (const [id, remoteLement] of remoteElementsMap) {
         if (!localElementIds.has(id)) {
@@ -279,7 +295,15 @@ function categorizeElements(localElements, remoteElementsMap) {
     }
     toDelete.sort((a, b) => Number(b.folder === ENDPOINT_LISTS_FOLDER) -
         Number(a.folder === ENDPOINT_LISTS_FOLDER));
-    return { toCreate, toInclude, toUpdate, toDetach, toDelete, conflicts };
+    return {
+        toCreate,
+        toInclude,
+        toUpdate,
+        toDetach,
+        toSetValue,
+        toDelete,
+        conflicts,
+    };
 }
 const prepareContentForApi = ({ name, description, ...content }, elementType) => {
     if (elementType === ElementType.Value) {
@@ -332,8 +356,9 @@ function buildShadowData(element, name, description, data) {
         ...data,
     };
 }
-function writeShadowFileAndRecord(fsImpl, element, great, source, included) {
+function writeShadowFileAndRecord(fsImpl, element, great, source, included, keepValue = false) {
     const content = included.content;
+    const { value, ref } = element.data;
     setElementRecord(great, element.folder, element.key, {
         id: included.id,
         ...(content?.createdAt ? { timestamp: content.createdAt } : {}),
@@ -343,9 +368,13 @@ function writeShadowFileAndRecord(fsImpl, element, great, source, included) {
             ? { provenance: element.provenance }
             : {}),
     });
-    writeElementFile(undefined, element.folder, element.key, buildShadowData(element, included.name, included.description, content?.data), great, fsImpl, element.filePath);
+    writeElementFile(undefined, element.folder, element.key, {
+        ...buildShadowData(element, included.name, included.description, content?.data),
+        ...(keepValue && value !== undefined ? { value } : {}),
+        ...(keepValue && ref !== undefined ? { ref } : {}),
+    }, great, fsImpl, element.filePath);
 }
-async function includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great) {
+async function includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great, shouldSetValue) {
     const source = element.source;
     if (!source) {
         return false;
@@ -378,19 +407,67 @@ async function includeShadowElement(element, graphqlClient, integrationId, fsImp
             log(chalk.yellow(`Warning: Failed to update metadata for shadow ${element.key} in ${element.folder}`));
         }
     }
-    writeShadowFileAndRecord(fsImpl, element, great, source, included);
+    const setsValue = shouldSetValue && hasValueToSet(element);
+    if (setsValue) {
+        recordElementProgress(great, element, { id: included.id });
+        const success = await setValueOrRef(graphqlClient, included.id, element, { type: included.content?.data?.type }, log);
+        if (!success) {
+            return false;
+        }
+    }
+    writeShadowFileAndRecord(fsImpl, element, great, source, included, !setsValue);
     return true;
 }
-async function updateElementContent(client, elementId, content) {
+async function updateElementContent(client, elementId, content, provenance) {
     const result = await client.request(SetElementContentMutationDocument, {
         id: elementId,
         content,
+        ...(provenance && provenance.length > 0 ? { provenance } : {}),
     });
     if (result.error) {
         return undefined;
     }
     return result.data?.setElementContent?.content?.createdAt;
 }
+async function setSecretRefForElement(client, elementId, element, log) {
+    const result = await client.request(SetSecretRefMutationDocument, { id: elementId, secretRef: String(element.data.ref) });
+    if (result.error) {
+        log(chalk.red(`Failed to set secret ref for ${element.key}: ${getGraphQLErrorMessage(result.error)}`));
+        return false;
+    }
+    log(chalk.green(`✓ Secret ref set for element "${element.key}"`));
+    return true;
+}
+async function setValueOrRef(client, id, element, content, log) {
+    const hasValue = element.data.value !== undefined;
+    const hasRef = element.data.ref !== undefined;
+    const isSecret = content.type === 'secret';
+    if (isSecret && hasRef && !hasValue) {
+        const success = await setSecretRefForElement(client, id, element, log);
+        if (!success) {
+            return false;
+        }
+    }
+    else if (hasValue) {
+        const valueStr = String(element.data.value);
+        const result = await setValueForElement(id, valueStr, content, client);
+        if (!result.success) {
+            log(chalk.red(`Failed to set value for ${element.key}: ${result.error}`));
+            return false;
+        }
+        log(chalk.green(`✓ Value set for element "${element.key}" (${content.type})`));
+    }
+    return true;
+}
+async function setValueOnly(element, id, graphqlClient, fsImpl, great, log) {
+    const success = await setValueOrRef(graphqlClient, id, element, { type: element.data.type }, log);
+    if (!success) {
+        return false;
+    }
+    const { value, ref, ...data } = element.data;
+    writeElementFile(undefined, element.folder, element.key, data, great, fsImpl, element.filePath);
+    return true;
+}
 function updateLocalFile(fsImpl, element, great, updatedId, updatedTimestamp, generatedName, keepValue = false) {
     const { name, description, value, ref, ...data } = element.data;
     const existing = getElementRecord(great, element.folder, element.key);
@@ -408,6 +485,9 @@ function updateLocalFile(fsImpl, element, great, updatedId, updatedTimestamp, ge
         ...(id ? { id } : {}),
         ...(timestamp ? { timestamp } : {}),
         hash: hashContent(orderedData),
+        ...(element.provenance && element.provenance.length > 0
+            ? { provenance: element.provenance }
+            : {}),
     });
     writeElementFile(undefined, element.folder, element.key, orderedData, great, fsImpl, element.filePath);
 }
@@ -446,6 +526,7 @@ function buildRemoteElementsMap(elements) {
             name: element.name,
             description: element.description,
             createdAt: element.createdAt,
+            sourceId: element.content?.sourceId,
             sourceTimestamp: element.content?.sourceTimestamp,
             provenance: element.content?.provenance,
         });
@@ -496,10 +577,14 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
     }
     if (element.source && !element.detach) {
         const remote = apiElementsMap.get(id);
+        const sourceChanged = element.source.id !== remote?.sourceId;
         const pinChanged = (element.source.timestamp ?? null) !== (remote?.sourceTimestamp ?? null);
         const provenanceChanged = !sameProvenance(element.provenance, remote?.provenance);
-        if (pinChanged || provenanceChanged) {
-            return includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great);
+        if (sourceChanged || pinChanged || provenanceChanged) {
+            return includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great, shouldSetValue);
+        }
+        if (shouldSetValue && hasValueToSet(element)) {
+            return setValueOnly(element, id, graphqlClient, fsImpl, great, log);
         }
         return true;
     }
@@ -512,7 +597,7 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
         log(chalk.red(`Failed to prepare content for ${element.key} in ${element.folder}: ${error instanceof Error ? error.message : String(error)}`));
         return false;
     }
-    const timestamp = await updateElementContent(graphqlClient, id, content);
+    const timestamp = await updateElementContent(graphqlClient, id, content, element.provenance);
     if (!timestamp) {
         log(chalk.red(`Failed to ${element.remoteId ? 'update' : 'set'} content for ${element.key} in ${element.folder}`));
         return false;
@@ -520,28 +605,13 @@ async function createOrUpdateElement(element, graphqlClient, integrationId, fsIm
     recordElementProgress(great, element, { id, timestamp });
     const hasValue = element.data.value !== undefined;
     const hasRef = element.data.ref !== undefined;
-    const isSecret = content.type === 'secret';
     if (elementType === ElementType.Value && shouldSetValue) {
-        if (isSecret && hasRef && !hasValue) {
-            const secretRef = String(element.data.ref);
-            const result = await graphqlClient.request(SetSecretRefMutationDocument, { id, secretRef });
-            if (result.error) {
-                log(chalk.red(`Failed to set secret ref for ${element.key}: ${getGraphQLErrorMessage(result.error)}`));
-                return false;
-            }
-            log(chalk.green(`✓ Secret ref set for element "${element.key}"`));
-        }
-        else if (hasValue) {
-            const valueStr = String(element.data.value);
-            const result = await setValueForElement(id, valueStr, content, graphqlClient);
-            if (!result.success) {
-                log(chalk.red(`Failed to set value for ${element.key}: ${result.error}`));
-                return false;
-            }
-            log(chalk.green(`✓ Value set for element "${element.key}" (${content.type})`));
+        const success = await setValueOrRef(graphqlClient, id, element, content, log);
+        if (!success) {
+            return false;
         }
     }
-    updateLocalFile(fsImpl, element, great, element.remoteId ? undefined : id, timestamp, generatedName, !shouldSetValue && hasValue);
+    updateLocalFile(fsImpl, element, great, element.remoteId ? undefined : id, timestamp, generatedName, !shouldSetValue && (hasValue || hasRef));
     return true;
 }
 async function deleteElement(element, graphqlClient, log) {
@@ -565,7 +635,7 @@ async function resolveDetaches(toDetach, promptForDetachFn, graphqlClient, fsImp
             return undefined;
         }
         if (shouldDetach) {
-            detached.push({ ...element, detach: true });
+            detached.push({ ...element, detach: true, provenance: undefined });
         }
         else {
             await restoreShadowFile(element, graphqlClient, fsImpl, great, log);
@@ -634,6 +704,110 @@ async function syncIntegrationType(graphqlClient, integrationId, remoteType, loc
     }
     log(`${chalk.green('✓')} Updated integration type to ${chalk.cyan(localType)}`);
 }
+function buildRemoteConfig(elements) {
+    const config = findConfigElement(elements);
+    return config
+        ? {
+            id: config.id,
+            key: CONFIG_KEY,
+            folder: CONFIG_RECORD_FOLDER,
+            checksum: config.content?.checksum,
+            contentCreatedAt: config.content?.createdAt,
+        }
+        : undefined;
+}
+function categorizeConfig(flags, remote, record) {
+    if (!remote) {
+        if (!flags) {
+            return { action: 'unchanged' };
+        }
+        return record?.id
+            ? {
+                action: 'conflict',
+                reason: 'Flags exist locally with an id but not found in Integreat',
+            }
+            : { action: 'create' };
+    }
+    const remoteChanged = (record?.timestamp ?? null) !== (remote.contentCreatedAt ?? null);
+    if (!flags) {
+        if (!record) {
+            return { action: 'unchanged' };
+        }
+        return remoteChanged
+            ? {
+                action: 'conflict',
+                reason: 'Flags removed locally but changed in Integreat',
+            }
+            : { action: 'delete' };
+    }
+    if (hashFlags(flags) === remote.checksum) {
+        return { action: 'unchanged' };
+    }
+    if (!remoteChanged) {
+        return { action: 'update' };
+    }
+    return {
+        action: 'conflict',
+        reason: record
+            ? 'Flags changed both local and remote'
+            : 'Flags differ from Integreat and have not been synced before',
+    };
+}
+const isConfigChange = ({ action }) => action === 'create' || action === 'update' || action === 'delete';
+function refreshConfigRecord(great, flags, remote) {
+    const existing = getElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY);
+    if (flags && remote) {
+        const record = buildConfigRecord(remote.id, remote.contentCreatedAt, flags);
+        setElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY, record);
+        return JSON.stringify(record) !== JSON.stringify(existing);
+    }
+    if (!flags && !remote && existing) {
+        removeElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY);
+        return true;
+    }
+    return false;
+}
+async function createConfigElement(graphqlClient, integrationId, great) {
+    const result = await graphqlClient.request(CreateElementMutationDocument, {
+        integrationId,
+        key: CONFIG_KEY,
+        type: ElementType.Config,
+        name: sentenceCase(CONFIG_KEY),
+    });
+    const id = result.error ? undefined : result.data?.createElement?.id;
+    if (id) {
+        setElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY, { id });
+    }
+    return id;
+}
+async function setConfigContent(graphqlClient, id, flags, great) {
+    const timestamp = await updateElementContent(graphqlClient, id, flags, undefined);
+    if (!timestamp) {
+        return false;
+    }
+    setElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY, buildConfigRecord(id, timestamp, flags));
+    return true;
+}
+async function pushConfig(change, flags, remote, graphqlClient, integrationId, great, log) {
+    if (change.action === 'delete' && remote) {
+        const success = await deleteElement(remote, graphqlClient, log);
+        if (success) {
+            removeElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY);
+            log(chalk.green('✓ Deleted flags'));
+        }
+        return success;
+    }
+    const id = change.action === 'create'
+        ? await createConfigElement(graphqlClient, integrationId, great)
+        : remote?.id;
+    const success = !!id && !!flags && (await setConfigContent(graphqlClient, id, flags, great));
+    if (!success) {
+        log(chalk.red(`Failed to ${change.action} flags`));
+        return false;
+    }
+    log(chalk.green(`✓ ${change.action === 'create' ? 'Created' : 'Updated'} flags`));
+    return true;
+}
 function logSummary(createdCount, includedCount, updatedCount, deletedCount, registeredCount, log) {
     const total = createdCount + includedCount + updatedCount + deletedCount + registeredCount;
     if (total > 0) {
@@ -737,9 +911,17 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
     for (const warning of validateEndpointReferences(localElements, great)) {
         log(chalk.yellow(`Warning: ${warning}`));
     }
-    const { toCreate, toInclude, toUpdate, toDetach, toDelete, conflicts } = categorizeElements(localElements, apiElementsMap);
-    if (conflicts.length > 0) {
-        handleConflicts(conflicts, log);
+    const { toCreate, toInclude, toUpdate, toDetach, toSetValue, toDelete, conflicts, } = categorizeElements(localElements, apiElementsMap);
+    const localFlags = flagsFromIntegration(localData);
+    const remoteConfig = buildRemoteConfig(integration.latest.elements);
+    const configChange = categorizeConfig(localFlags, remoteConfig, getElementRecord(great, CONFIG_RECORD_FOLDER, CONFIG_KEY));
+    if (conflicts.length > 0 || configChange.action === 'conflict') {
+        handleConflicts(configChange.action === 'conflict'
+            ? [
+                ...conflicts,
+                { filePath: 'integration.json', reason: configChange.reason },
+            ]
+            : conflicts, log);
         return;
     }
     const detached = await resolveDetaches(toDetach, promptForDetachFn, graphqlClient, fsImpl, great, log);
@@ -753,15 +935,20 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
     if (toCreate.length +
         toInclude.length +
         toUpdate.length +
+        toSetValue.length +
         toDelete.length +
-        importsToRegister.length ===
+        importsToRegister.length +
+        Number(isConfigChange(configChange)) ===
         0) {
+        if (refreshConfigRecord(great, localFlags, remoteConfig)) {
+            writeGreatFile(great, undefined, fsImpl);
+        }
         log(chalk.green('No changes to push'));
         return;
     }
     let shouldSetValue;
     try {
-        shouldSetValue = await promptForValueSettingFn([...toCreate, ...toUpdate], log);
+        shouldSetValue = await promptForValueSettingFn([...toCreate, ...toInclude, ...toUpdate, ...toSetValue], log);
     }
     catch {
         log(chalk.yellow('\nPush cancelled'));
@@ -780,7 +967,7 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
     }
     let includedCount = 0;
     for (const element of toInclude) {
-        const success = await includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great);
+        const success = await includeShadowElement(element, graphqlClient, integrationId, fsImpl, log, great, shouldSetValue);
         if (!success) {
             persistProgress();
             return;
@@ -798,6 +985,20 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
         log(chalk.green(`✓ Updated ${element.folder}/${element.key}`));
         updatedCount++;
     }
+    if (shouldSetValue) {
+        for (const element of toSetValue) {
+            if (!element.remoteId) {
+                continue;
+            }
+            const success = await setValueOnly(element, element.remoteId, graphqlClient, fsImpl, great, log);
+            if (!success) {
+                persistProgress();
+                return;
+            }
+            log(chalk.green(`✓ Updated ${element.folder}/${element.key}`));
+            updatedCount++;
+        }
+    }
     let deletedCount = 0;
     for (const element of toDelete) {
         const success = await deleteElement(element, graphqlClient, log);
@@ -812,12 +1013,31 @@ export const action = (log, apiToken, client, fsImpl = fs, config, promptForValu
         log(chalk.green(`✓ Deleted ${label}`));
         deletedCount++;
     }
+    if (isConfigChange(configChange)) {
+        const success = await pushConfig(configChange, localFlags, remoteConfig, graphqlClient, integrationId, great, log);
+        if (!success) {
+            persistProgress();
+            return;
+        }
+        if (configChange.action === 'create')
+            createdCount++;
+        if (configChange.action === 'update')
+            updatedCount++;
+        if (configChange.action === 'delete')
+            deletedCount++;
+    }
+    else {
+        refreshConfigRecord(great, localFlags, remoteConfig);
+    }
     const registeredCount = await registerTemplateImports(importsToRegister, integration.templateImports ?? [], graphqlClient, integrationId, great, log);
     if (registeredCount === undefined) {
         persistProgress();
         return;
     }
-    const localKeys = new Set(localElements.map((el) => `${el.folder}/${el.key}`));
+    const localKeys = new Set([
+        ...localElements.map((el) => `${el.folder}/${el.key}`),
+        `${CONFIG_RECORD_FOLDER}/${CONFIG_KEY}`,
+    ]);
     great.elements = Object.fromEntries(Object.entries(great.elements).filter(([key]) => localKeys.has(key)));
     writeGreatFile(great, undefined, fsImpl);
     logSummary(createdCount, includedCount, updatedCount, deletedCount, registeredCount, log);