@webiny/api-headless-cms-import-export 5.41.0-beta.0

package/tasks/domain/importFromUrlControllerSteps/getChildTasks.js

@@ -0,0 +1,92 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getChildTasks = void 0;
+var _tasks = require("@webiny/tasks");
+const mapServiceStatusToTaskStatus = (task, serviceInfo) => {
+  if (!serviceInfo) {
+    console.log(`Service info is missing for task ${task.id} (${task.definitionId}).`);
+    return null;
+  }
+  if (serviceInfo.status === "RUNNING") {
+    return _tasks.TaskDataStatus.RUNNING;
+  } else if (serviceInfo.status === "SUCCEEDED") {
+    return _tasks.TaskDataStatus.SUCCESS;
+  } else if (serviceInfo.status === "FAILED") {
+    return _tasks.TaskDataStatus.FAILED;
+  } else if (serviceInfo.status === "ABORTED") {
+    return _tasks.TaskDataStatus.ABORTED;
+  } else if (serviceInfo.status === "TIMED_OUT" || serviceInfo.status === "PENDING_REDRIVE") {
+    console.log(`Service status is ${serviceInfo.status} for task ${task.id} (${task.definitionId}).`);
+    return null;
+  }
+  return _tasks.TaskDataStatus.PENDING;
+};
+const getChildTasks = async ({
+  context,
+  task,
+  definition
+}) => {
+  const running = [];
+  const done = [];
+  const invalid = [];
+  const aborted = [];
+  const failed = [];
+  const collection = [];
+  const {
+    items
+  } = await context.tasks.listTasks({
+    where: {
+      parentId: task.id,
+      definitionId: definition
+    },
+    limit: 100000
+  });
+  for (const task of items) {
+    collection.push(task);
+    if (task.taskStatus === _tasks.TaskDataStatus.RUNNING || task.taskStatus === _tasks.TaskDataStatus.PENDING) {
+      /**
+       * We also need to check the actual status of the service.
+       * It can happen that the task is marked as running, but the service is not running.
+       */
+      const serviceInfo = await context.tasks.fetchServiceInfo(task);
+      const status = mapServiceStatusToTaskStatus(task, serviceInfo);
+      if (status === null || !serviceInfo) {
+        invalid.push(task.id);
+        continue;
+      } else if (status !== task.taskStatus) {
+        console.error(`Status of the task is not same as the status of the service (task: ${task.taskStatus}, service: ${status} / ${serviceInfo.status}).`);
+        invalid.push(task.id);
+        continue;
+      }
+      running.push(task.id);
+      continue;
+    } else if (task.taskStatus === _tasks.TaskDataStatus.SUCCESS) {
+      done.push(task.id);
+      continue;
+    } else if (task.taskStatus === _tasks.TaskDataStatus.FAILED) {
+      failed.push(task.id);
+      continue;
+    } else if (task.taskStatus === _tasks.TaskDataStatus.ABORTED) {
+      aborted.push(task.id);
+      continue;
+    }
+    /**
+     * Impossible to be in a state not listed above, but just in case.
+     */
+    invalid.push(task.id);
+  }
+  return {
+    running,
+    done,
+    invalid,
+    aborted,
+    failed,
+    collection
+  };
+};
+exports.getChildTasks = getChildTasks;
+
+//# sourceMappingURL=getChildTasks.js.map

package/tasks/domain/importFromUrlControllerSteps/getChildTasks.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_tasks","require","mapServiceStatusToTaskStatus","task","serviceInfo","console","log","id","definitionId","status","TaskDataStatus","RUNNING","SUCCESS","FAILED","ABORTED","PENDING","getChildTasks","context","definition","running","done","invalid","aborted","failed","collection","items","tasks","listTasks","where","parentId","limit","push","taskStatus","fetchServiceInfo","error","exports"],"sources":["getChildTasks.ts"],"sourcesContent":["import type { ITask, ITaskResponseDoneResultOutput } from \"@webiny/tasks\";\nimport { TaskDataStatus } from \"@webiny/tasks\";\nimport type { Context } from \"~/types\";\nimport { IStepFunctionServiceFetchResult } from \"@webiny/tasks/service/StepFunctionServicePlugin\";\n\nexport interface IGetChildTasksParams {\n    context: Context;\n    task: ITask;\n    definition: string;\n}\n\nconst mapServiceStatusToTaskStatus = (\n    task: ITask<any, any>,\n    serviceInfo: IStepFunctionServiceFetchResult | null\n) => {\n    if (!serviceInfo) {\n        console.log(`Service info is missing for task ${task.id} (${task.definitionId}).`);\n        return null;\n    }\n    if (serviceInfo.status === \"RUNNING\") {\n        return TaskDataStatus.RUNNING;\n    } else if (serviceInfo.status === \"SUCCEEDED\") {\n        return TaskDataStatus.SUCCESS;\n    } else if (serviceInfo.status === \"FAILED\") {\n        return TaskDataStatus.FAILED;\n    } else if (serviceInfo.status === \"ABORTED\") {\n        return TaskDataStatus.ABORTED;\n    } else if (serviceInfo.status === \"TIMED_OUT\" || serviceInfo.status === \"PENDING_REDRIVE\") {\n        console.log(\n            `Service status is ${serviceInfo.status} for task ${task.id} (${task.definitionId}).`\n        );\n        return null;\n    }\n    return TaskDataStatus.PENDING;\n};\n\nexport const getChildTasks = async <I, O extends ITaskResponseDoneResultOutput>({\n    context,\n    task,\n    definition\n}: IGetChildTasksParams) => {\n    const running: string[] = [];\n    const done: string[] = [];\n    const invalid: string[] = [];\n    const aborted: string[] = [];\n    const failed: string[] = [];\n    const collection: ITask<I, O>[] = [];\n\n    const { items } = await context.tasks.listTasks<I, O>({\n        where: {\n            parentId: task.id,\n            definitionId: definition\n        },\n        limit: 100000\n    });\n    for (const task of items) {\n        collection.push(task);\n\n        if (\n            task.taskStatus === TaskDataStatus.RUNNING ||\n            task.taskStatus === TaskDataStatus.PENDING\n        ) {\n            /**\n             * We also need to check the actual status of the service.\n             * It can happen that the task is marked as running, but the service is not running.\n             */\n            const serviceInfo = await context.tasks.fetchServiceInfo(task);\n            const status = mapServiceStatusToTaskStatus(task, serviceInfo);\n\n            if (status === null || !serviceInfo) {\n                invalid.push(task.id);\n                continue;\n            } else if (status !== task.taskStatus) {\n                console.error(\n                    `Status of the task is not same as the status of the service (task: ${task.taskStatus}, service: ${status} / ${serviceInfo.status}).`\n                );\n                invalid.push(task.id);\n                continue;\n            }\n            running.push(task.id);\n            continue;\n        } else if (task.taskStatus === TaskDataStatus.SUCCESS) {\n            done.push(task.id);\n            continue;\n        } else if (task.taskStatus === TaskDataStatus.FAILED) {\n            failed.push(task.id);\n            continue;\n        } else if (task.taskStatus === TaskDataStatus.ABORTED) {\n            aborted.push(task.id);\n            continue;\n        }\n        /**\n         * Impossible to be in a state not listed above, but just in case.\n         */\n        invalid.push(task.id);\n    }\n    return {\n        running,\n        done,\n        invalid,\n        aborted,\n        failed,\n        collection\n    };\n};\n"],"mappings":";;;;;;AACA,IAAAA,MAAA,GAAAC,OAAA;AAUA,MAAMC,4BAA4B,GAAGA,CACjCC,IAAqB,EACrBC,WAAmD,KAClD;EACD,IAAI,CAACA,WAAW,EAAE;IACdC,OAAO,CAACC,GAAG,CAAE,oCAAmCH,IAAI,CAACI,EAAG,KAAIJ,IAAI,CAACK,YAAa,IAAG,CAAC;IAClF,OAAO,IAAI;EACf;EACA,IAAIJ,WAAW,CAACK,MAAM,KAAK,SAAS,EAAE;IAClC,OAAOC,qBAAc,CAACC,OAAO;EACjC,CAAC,MAAM,IAAIP,WAAW,CAACK,MAAM,KAAK,WAAW,EAAE;IAC3C,OAAOC,qBAAc,CAACE,OAAO;EACjC,CAAC,MAAM,IAAIR,WAAW,CAACK,MAAM,KAAK,QAAQ,EAAE;IACxC,OAAOC,qBAAc,CAACG,MAAM;EAChC,CAAC,MAAM,IAAIT,WAAW,CAACK,MAAM,KAAK,SAAS,EAAE;IACzC,OAAOC,qBAAc,CAACI,OAAO;EACjC,CAAC,MAAM,IAAIV,WAAW,CAACK,MAAM,KAAK,WAAW,IAAIL,WAAW,CAACK,MAAM,KAAK,iBAAiB,EAAE;IACvFJ,OAAO,CAACC,GAAG,CACN,qBAAoBF,WAAW,CAACK,MAAO,aAAYN,IAAI,CAACI,EAAG,KAAIJ,IAAI,CAACK,YAAa,IACtF,CAAC;IACD,OAAO,IAAI;EACf;EACA,OAAOE,qBAAc,CAACK,OAAO;AACjC,CAAC;AAEM,MAAMC,aAAa,GAAG,MAAAA,CAAmD;EAC5EC,OAAO;EACPd,IAAI;EACJe;AACkB,CAAC,KAAK;EACxB,MAAMC,OAAiB,GAAG,EAAE;EAC5B,MAAMC,IAAc,GAAG,EAAE;EACzB,MAAMC,OAAiB,GAAG,EAAE;EAC5B,MAAMC,OAAiB,GAAG,EAAE;EAC5B,MAAMC,MAAgB,GAAG,EAAE;EAC3B,MAAMC,UAAyB,GAAG,EAAE;EAEpC,MAAM;IAAEC;EAAM,CAAC,GAAG,MAAMR,OAAO,CAACS,KAAK,CAACC,SAAS,CAAO;IAClDC,KAAK,EAAE;MACHC,QAAQ,EAAE1B,IAAI,CAACI,EAAE;MACjBC,YAAY,EAAEU;IAClB,CAAC;IACDY,KAAK,EAAE;EACX,CAAC,CAAC;EACF,KAAK,MAAM3B,IAAI,IAAIsB,KAAK,EAAE;IACtBD,UAAU,CAACO,IAAI,CAAC5B,IAAI,CAAC;IAErB,IACIA,IAAI,CAAC6B,UAAU,KAAKtB,qBAAc,CAACC,OAAO,IAC1CR,IAAI,CAAC6B,UAAU,KAAKtB,qBAAc,CAACK,OAAO,EAC5C;MACE;AACZ;AACA;AACA;MACY,MAAMX,WAAW,GAAG,MAAMa,OAAO,CAACS,KAAK,CAACO,gBAAgB,CAAC9B,IAAI,CAAC;MAC9D,MAAMM,MAAM,GAAGP,4BAA4B,CAACC,IAAI,EAAEC,WAAW,CAAC;MAE9D,IAAIK,MAAM,KAAK,IAAI,IAAI,CAACL,WAAW,EAAE;QACjCiB,OAAO,CAACU,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;QACrB;MACJ,CAAC,MAAM,IAAIE,MAAM,KAAKN,IAAI,CAAC6B,UAAU,EAAE;QACnC3B,OAAO,CAAC6B,KAAK,CACR,sEAAqE/B,IAAI,CAAC6B,UAAW,cAAavB,MAAO,MAAKL,WAAW,CAACK,MAAO,IACtI,CAAC;QACDY,OAAO,CAACU,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;QACrB;MACJ;MACAY,OAAO,CAACY,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;MACrB;IACJ,CAAC,MAAM,IAAIJ,IAAI,CAAC6B,UAAU,KAAKtB,qBAAc,CAACE,OAAO,EAAE;MACnDQ,IAAI,CAACW,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;MAClB;IACJ,CAAC,MAAM,IAAIJ,IAAI,CAAC6B,UAAU,KAAKtB,qBAAc,CAACG,MAAM,EAAE;MAClDU,MAAM,CAACQ,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;MACpB;IACJ,CAAC,MAAM,IAAIJ,IAAI,CAAC6B,UAAU,KAAKtB,qBAAc,CAACI,OAAO,EAAE;MACnDQ,OAAO,CAACS,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;MACrB;IACJ;IACA;AACR;AACA;IACQc,OAAO,CAACU,IAAI,CAAC5B,IAAI,CAACI,EAAE,CAAC;EACzB;EACA,OAAO;IACHY,OAAO;IACPC,IAAI;IACJC,OAAO;IACPC,OAAO;IACPC,MAAM;IACNC;EACJ,CAAC;AACL,CAAC;AAACW,OAAA,CAAAnB,aAAA,GAAAA,aAAA","ignoreList":[]}

package/tasks/domain/importFromUrlProcessAssets/ImportFromUrlProcessAssets.d.ts

@@ -0,0 +1,19 @@
+import type { ITaskResponseResult, ITaskRunParams } from "@webiny/tasks/types";
+import type { IImportFromUrlProcessAssets, IImportFromUrlProcessAssetsInput, IImportFromUrlProcessAssetsOutput } from "./abstractions/ImportFromUrlProcessAssets";
+import { Context } from "../../../types";
+import type { IFileFetcher } from "../../utils/fileFetcher";
+import type { ICompressedFileReader, IDecompressor } from "../../utils/decompressor";
+export interface IImportFromUrlProcessAssetsParams {
+    fileFetcher: IFileFetcher;
+    reader: ICompressedFileReader;
+    decompressor: IDecompressor;
+}
+export declare class ImportFromUrlProcessAssets<C extends Context = Context, I extends IImportFromUrlProcessAssetsInput = IImportFromUrlProcessAssetsInput, O extends IImportFromUrlProcessAssetsOutput = IImportFromUrlProcessAssetsOutput> implements IImportFromUrlProcessAssets<C, I, O> {
+    private readonly fileFetcher;
+    private readonly reader;
+    private readonly decompressor;
+    constructor(params: IImportFromUrlProcessAssetsParams);
+    run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>>;
+    private readManifest;
+    private takeNextAssetRecord;
+}

package/tasks/domain/importFromUrlProcessAssets/ImportFromUrlProcessAssets.js

@@ -0,0 +1,265 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ImportFromUrlProcessAssets = void 0;
+var _types = require("../../../types");
+var _constants = require("../../constants");
+var _getFilePath = require("../../utils/helpers/getFilePath");
+var _error = require("@webiny/error");
+class ImportFromUrlProcessAssets {
+  constructor(params) {
+    this.fileFetcher = params.fileFetcher;
+    this.reader = params.reader;
+    this.decompressor = params.decompressor;
+  }
+  async run(params) {
+    const {
+      context,
+      response,
+      input,
+      isCloseToTimeout,
+      isAborted
+    } = params;
+    const maxInsertErrors = input.maxInsertErrors || 100;
+    if (!input.modelId) {
+      return response.error({
+        message: `Missing "modelId" in the input.`,
+        code: "MISSING_MODEL_ID"
+      });
+    } else if (!input.file) {
+      return response.error({
+        message: `No file found in the provided data.`,
+        code: "NO_FILE_FOUND"
+      });
+    } else if (input.file.type !== _types.CmsImportExportFileType.ASSETS) {
+      return response.error({
+        message: `Invalid file type. Expected "${_types.CmsImportExportFileType.ASSETS}" but got "${input.file.type}".`,
+        code: "INVALID_FILE_TYPE"
+      });
+    }
+    const recordExists = async id => {
+      try {
+        const result = await context.fileManager.getFile(id);
+        return !!result;
+      } catch (ex) {
+        return false;
+      }
+    };
+    const result = structuredClone({
+      ...input,
+      errors: []
+    });
+    /**
+     * Read the compressed archive and get all the file information.
+     */
+    const sources = await this.reader.read(result.file.key);
+    if (sources.length === 0) {
+      return response.error({
+        message: `No files found in the compressed archive.`,
+        code: "NO_FILES_FOUND"
+      });
+    }
+
+    /**
+     * Read the manifest file.
+     * Should not be possible it does not exist, but let's handle it anyway.
+     */
+    let manifest;
+    try {
+      manifest = await this.readManifest(sources, result);
+    } catch (ex) {
+      console.error(ex);
+      return response.error(ex);
+    }
+    while (true) {
+      if (isCloseToTimeout()) {
+        return response.continue(result);
+      } else if (isAborted()) {
+        return response.aborted();
+      } else if (result.errors.length > maxInsertErrors) {
+        return response.error({
+          message: `Too many errors encountered while processing assets.`,
+          code: "TOO_MANY_ERRORS",
+          data: {
+            errors: result.errors
+          }
+        });
+      }
+      const record = this.takeNextAssetRecord(manifest, result.lastAsset);
+      if (!record) {
+        return response.done("No more assets to process");
+      }
+      result.lastAsset = record.id;
+      const source = sources.find(file => file.path === record.key);
+      if (!source) {
+        result.errors.push({
+          file: record.key,
+          message: `File not found in the compressed archive.`
+        });
+        continue;
+      }
+      /**
+       * Check if the file exists physically.
+       */
+      const existsPhysically = await this.fileFetcher.exists(source.path);
+      if (existsPhysically && !input.override) {
+        console.log(`Asset "${record.id}" / ${source.path} file already exists. Skipping...`);
+        continue;
+      }
+      /**
+       * Check if the file record already exists.
+       */
+      const exists = await recordExists(record.id);
+      if (exists && !input.override) {
+        console.log(`Asset "${record.id}" / ${source.path} record already exists. Skipping...`);
+        continue;
+      }
+      /**
+       * Upload the file to the S3 bucket, if it does not exist already.
+       */
+      if (!existsPhysically) {
+        try {
+          const file = await this.decompressor.extract({
+            source,
+            target: record.key
+          });
+          if (!file?.Key) {
+            result.errors.push({
+              file: record.key,
+              message: `Could not upload the file "${source.path}" to "${record.key}".`
+            });
+            continue;
+          }
+        } catch (ex) {
+          result.errors.push({
+            file: record.key,
+            message: ex.message
+          });
+          continue;
+        }
+      }
+      /**
+       * Update an existing file record.
+       */
+      if (exists) {
+        try {
+          await context.fileManager.updateFile(record.id, {
+            ...record
+          });
+        } catch (ex) {
+          result.errors.push({
+            file: record.key,
+            message: ex.message
+          });
+        }
+        continue;
+      }
+      /**
+       * Create a new file record.
+       */
+      try {
+        await context.fileManager.createFile({
+          ...record,
+          id: record.id,
+          key: record.key,
+          size: record.size,
+          type: record.type,
+          name: record.name,
+          meta: record.meta,
+          aliases: record.aliases,
+          extensions: record.extensions,
+          location: record.location,
+          tags: record.tags
+        });
+      } catch (ex) {
+        result.errors.push({
+          file: record.key,
+          message: ex.message
+        });
+      }
+    }
+  }
+  async readManifest(sources, input) {
+    let manifest = input?.manifest;
+    if (!manifest) {
+      const extractPath = (0, _getFilePath.getFilePath)(input.file.key);
+      const source = sources.find(file => file.path === _constants.MANIFEST_JSON);
+      if (!source) {
+        throw new _error.WebinyError({
+          message: `No manifest file found in the compressed archive.`,
+          code: "NO_MANIFEST_FILE"
+        });
+      }
+      const target = `extracted/${extractPath.path}/${source.path}`;
+      try {
+        const file = await this.decompressor.extract({
+          source,
+          target
+        });
+        if (!file.Key) {
+          throw new Error(`Could not upload the file "${source.path}" to "${target}".`);
+        }
+        manifest = file.Key;
+      } catch (ex) {
+        throw new _error.WebinyError({
+          message: ex.message,
+          code: "MANIFEST_DECOMPRESS_FAILED"
+        });
+      }
+    }
+    let file;
+    try {
+      file = await this.fileFetcher.read(manifest);
+      if (!file) {
+        throw new _error.WebinyError({
+          message: `Could not fetch the manifest file "${manifest}".`,
+          code: "MANIFEST_NOT_FOUND"
+        });
+      }
+    } catch (ex) {
+      throw new _error.WebinyError({
+        message: ex.message,
+        code: "MANIFEST_FETCH_FAILED",
+        data: ex
+      });
+    }
+    let json;
+    try {
+      json = JSON.parse(file);
+    } catch (ex) {
+      throw new _error.WebinyError({
+        message: ex.message,
+        code: "MANIFEST_PARSE_FAILED",
+        data: ex
+      });
+    }
+    if (!json.assets?.length) {
+      throw new _error.WebinyError({
+        message: `Invalid manifest file "${manifest}". Missing "assets" property.`,
+        code: "INVALID_MANIFEST_FILE"
+      });
+    } else if (!json.size) {
+      throw new _error.WebinyError({
+        message: `Invalid manifest file "${manifest}". Missing "size" property.`,
+        code: "INVALID_MANIFEST_FILE"
+      });
+    }
+    return json;
+  }
+  takeNextAssetRecord(manifest, lastAsset) {
+    if (!lastAsset) {
+      return manifest.assets[0];
+    }
+    const lastIndex = manifest.assets.findIndex(asset => asset.id === lastAsset);
+    if (lastIndex === -1) {
+      return null;
+    }
+    const asset = manifest.assets[lastIndex + 1];
+    return asset || null;
+  }
+}
+exports.ImportFromUrlProcessAssets = ImportFromUrlProcessAssets;
+
+//# sourceMappingURL=ImportFromUrlProcessAssets.js.map

package/tasks/domain/importFromUrlProcessAssets/ImportFromUrlProcessAssets.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_types","require","_constants","_getFilePath","_error","ImportFromUrlProcessAssets","constructor","params","fileFetcher","reader","decompressor","run","context","response","input","isCloseToTimeout","isAborted","maxInsertErrors","modelId","error","message","code","file","type","CmsImportExportFileType","ASSETS","recordExists","id","result","fileManager","getFile","ex","structuredClone","errors","sources","read","key","length","manifest","readManifest","console","continue","aborted","data","record","takeNextAssetRecord","lastAsset","done","source","find","path","push","existsPhysically","exists","override","log","extract","target","Key","updateFile","createFile","size","name","meta","aliases","extensions","location","tags","extractPath","getFilePath","MANIFEST_JSON","WebinyError","Error","json","JSON","parse","assets","lastIndex","findIndex","asset","exports"],"sources":["ImportFromUrlProcessAssets.ts"],"sourcesContent":["import type { ITaskResponseResult, ITaskRunParams } from \"@webiny/tasks/types\";\nimport type {\n    IImportFromUrlProcessAssets,\n    IImportFromUrlProcessAssetsInput,\n    IImportFromUrlProcessAssetsOutput\n} from \"~/tasks/domain/importFromUrlProcessAssets/abstractions/ImportFromUrlProcessAssets\";\nimport { CmsImportExportFileType, Context } from \"~/types\";\nimport type { IFileFetcher } from \"~/tasks/utils/fileFetcher\";\nimport type {\n    ICompressedFileReader,\n    IDecompressor,\n    IUnzipperFile\n} from \"~/tasks/utils/decompressor\";\nimport { MANIFEST_JSON } from \"~/tasks/constants\";\nimport { getFilePath } from \"~/tasks/utils/helpers/getFilePath\";\nimport { WebinyError } from \"@webiny/error\";\nimport type { ICmsAssetsManifestJson } from \"~/tasks/utils/types\";\nimport type { IResolvedAsset } from \"~/tasks/utils/entryAssets\";\n\nexport interface IImportFromUrlProcessAssetsParams {\n    fileFetcher: IFileFetcher;\n    reader: ICompressedFileReader;\n    decompressor: IDecompressor;\n}\n\nexport class ImportFromUrlProcessAssets<\n    C extends Context = Context,\n    I extends IImportFromUrlProcessAssetsInput = IImportFromUrlProcessAssetsInput,\n    O extends IImportFromUrlProcessAssetsOutput = IImportFromUrlProcessAssetsOutput\n> implements IImportFromUrlProcessAssets<C, I, O>\n{\n    private readonly fileFetcher: IFileFetcher;\n    private readonly reader: ICompressedFileReader;\n    private readonly decompressor: IDecompressor;\n\n    public constructor(params: IImportFromUrlProcessAssetsParams) {\n        this.fileFetcher = params.fileFetcher;\n        this.reader = params.reader;\n        this.decompressor = params.decompressor;\n    }\n\n    public async run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>> {\n        const { context, response, input, isCloseToTimeout, isAborted } = params;\n\n        const maxInsertErrors = input.maxInsertErrors || 100;\n\n        if (!input.modelId) {\n            return response.error({\n                message: `Missing \"modelId\" in the input.`,\n                code: \"MISSING_MODEL_ID\"\n            });\n        } else if (!input.file) {\n            return response.error({\n                message: `No file found in the provided data.`,\n                code: \"NO_FILE_FOUND\"\n            });\n        } else if (input.file.type !== CmsImportExportFileType.ASSETS) {\n            return response.error({\n                message: `Invalid file type. Expected \"${CmsImportExportFileType.ASSETS}\" but got \"${input.file.type}\".`,\n                code: \"INVALID_FILE_TYPE\"\n            });\n        }\n\n        const recordExists = async (id: string): Promise<boolean> => {\n            try {\n                const result = await context.fileManager.getFile(id);\n                return !!result;\n            } catch (ex) {\n                return false;\n            }\n        };\n\n        const result = structuredClone({\n            ...input,\n            errors: []\n        });\n        /**\n         * Read the compressed archive and get all the file information.\n         */\n        const sources = await this.reader.read(result.file.key);\n        if (sources.length === 0) {\n            return response.error({\n                message: `No files found in the compressed archive.`,\n                code: \"NO_FILES_FOUND\"\n            });\n        }\n\n        /**\n         * Read the manifest file.\n         * Should not be possible it does not exist, but let's handle it anyway.\n         */\n        let manifest: ICmsAssetsManifestJson;\n        try {\n            manifest = await this.readManifest(sources, result);\n        } catch (ex) {\n            console.error(ex);\n            return response.error(ex);\n        }\n\n        while (true) {\n            if (isCloseToTimeout()) {\n                return response.continue(result);\n            } else if (isAborted()) {\n                return response.aborted();\n            } else if (result.errors.length > maxInsertErrors) {\n                return response.error({\n                    message: `Too many errors encountered while processing assets.`,\n                    code: \"TOO_MANY_ERRORS\",\n                    data: {\n                        errors: result.errors\n                    }\n                });\n            }\n            const record = this.takeNextAssetRecord(manifest, result.lastAsset);\n            if (!record) {\n                return response.done(\"No more assets to process\");\n            }\n            result.lastAsset = record.id;\n            const source = sources.find(file => file.path === record.key);\n            if (!source) {\n                result.errors.push({\n                    file: record.key,\n                    message: `File not found in the compressed archive.`\n                });\n                continue;\n            }\n            /**\n             * Check if the file exists physically.\n             */\n            const existsPhysically = await this.fileFetcher.exists(source.path);\n            if (existsPhysically && !input.override) {\n                console.log(\n                    `Asset \"${record.id}\" / ${source.path} file already exists. Skipping...`\n                );\n                continue;\n            }\n            /**\n             * Check if the file record already exists.\n             */\n            const exists = await recordExists(record.id);\n            if (exists && !input.override) {\n                console.log(\n                    `Asset \"${record.id}\" / ${source.path} record already exists. Skipping...`\n                );\n                continue;\n            }\n            /**\n             * Upload the file to the S3 bucket, if it does not exist already.\n             */\n            if (!existsPhysically) {\n                try {\n                    const file = await this.decompressor.extract({\n                        source,\n                        target: record.key\n                    });\n                    if (!file?.Key) {\n                        result.errors.push({\n                            file: record.key,\n                            message: `Could not upload the file \"${source.path}\" to \"${record.key}\".`\n                        });\n                        continue;\n                    }\n                } catch (ex) {\n                    result.errors.push({\n                        file: record.key,\n                        message: ex.message\n                    });\n                    continue;\n                }\n            }\n            /**\n             * Update an existing file record.\n             */\n            if (exists) {\n                try {\n                    await context.fileManager.updateFile(record.id, {\n                        ...record\n                    });\n                } catch (ex) {\n                    result.errors.push({\n                        file: record.key,\n                        message: ex.message\n                    });\n                }\n                continue;\n            }\n            /**\n             * Create a new file record.\n             */\n            try {\n                await context.fileManager.createFile({\n                    ...record,\n                    id: record.id,\n                    key: record.key,\n                    size: record.size,\n                    type: record.type,\n                    name: record.name,\n                    meta: record.meta,\n                    aliases: record.aliases,\n                    extensions: record.extensions,\n                    location: record.location,\n                    tags: record.tags\n                });\n            } catch (ex) {\n                result.errors.push({\n                    file: record.key,\n                    message: ex.message\n                });\n            }\n        }\n    }\n\n    private async readManifest(\n        sources: IUnzipperFile[],\n        input: I\n    ): Promise<ICmsAssetsManifestJson> {\n        let manifest: string | undefined = input?.manifest;\n\n        if (!manifest) {\n            const extractPath = getFilePath(input.file.key);\n            const source = sources.find(file => file.path === MANIFEST_JSON);\n            if (!source) {\n                throw new WebinyError({\n                    message: `No manifest file found in the compressed archive.`,\n                    code: \"NO_MANIFEST_FILE\"\n                });\n            }\n            const target = `extracted/${extractPath.path}/${source.path}`;\n            try {\n                const file = await this.decompressor.extract({\n                    source,\n                    target\n                });\n                if (!file.Key) {\n                    throw new Error(`Could not upload the file \"${source.path}\" to \"${target}\".`);\n                }\n                manifest = file.Key;\n            } catch (ex) {\n                throw new WebinyError({\n                    message: ex.message,\n                    code: \"MANIFEST_DECOMPRESS_FAILED\"\n                });\n            }\n        }\n\n        let file: string | null;\n        try {\n            file = await this.fileFetcher.read(manifest);\n            if (!file) {\n                throw new WebinyError({\n                    message: `Could not fetch the manifest file \"${manifest}\".`,\n                    code: \"MANIFEST_NOT_FOUND\"\n                });\n            }\n        } catch (ex) {\n            throw new WebinyError({\n                message: ex.message,\n                code: \"MANIFEST_FETCH_FAILED\",\n                data: ex\n            });\n        }\n\n        let json: ICmsAssetsManifestJson;\n        try {\n            json = JSON.parse(file);\n        } catch (ex) {\n            throw new WebinyError({\n                message: ex.message,\n                code: \"MANIFEST_PARSE_FAILED\",\n                data: ex\n            });\n        }\n        if (!json.assets?.length) {\n            throw new WebinyError({\n                message: `Invalid manifest file \"${manifest}\". Missing \"assets\" property.`,\n                code: \"INVALID_MANIFEST_FILE\"\n            });\n        } else if (!json.size) {\n            throw new WebinyError({\n                message: `Invalid manifest file \"${manifest}\". Missing \"size\" property.`,\n                code: \"INVALID_MANIFEST_FILE\"\n            });\n        }\n        return json;\n    }\n\n    private takeNextAssetRecord(\n        manifest: ICmsAssetsManifestJson,\n        lastAsset: string | undefined\n    ): IResolvedAsset | null {\n        if (!lastAsset) {\n            return manifest.assets[0] as IResolvedAsset;\n        }\n        const lastIndex = manifest.assets.findIndex(asset => asset.id === lastAsset);\n        if (lastIndex === -1) {\n            return null;\n        }\n        const asset = manifest.assets[lastIndex + 1];\n        return asset || null;\n    }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,MAAA,GAAAC,OAAA;AAOA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,YAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAUO,MAAMI,0BAA0B,CAKvC;EAKWC,WAAWA,CAACC,MAAyC,EAAE;IAC1D,IAAI,CAACC,WAAW,GAAGD,MAAM,CAACC,WAAW;IACrC,IAAI,CAACC,MAAM,GAAGF,MAAM,CAACE,MAAM;IAC3B,IAAI,CAACC,YAAY,GAAGH,MAAM,CAACG,YAAY;EAC3C;EAEA,MAAaC,GAAGA,CAACJ,MAA+B,EAAsC;IAClF,MAAM;MAAEK,OAAO;MAAEC,QAAQ;MAAEC,KAAK;MAAEC,gBAAgB;MAAEC;IAAU,CAAC,GAAGT,MAAM;IAExE,MAAMU,eAAe,GAAGH,KAAK,CAACG,eAAe,IAAI,GAAG;IAEpD,IAAI,CAACH,KAAK,CAACI,OAAO,EAAE;MAChB,OAAOL,QAAQ,CAACM,KAAK,CAAC;QAClBC,OAAO,EAAG,iCAAgC;QAC1CC,IAAI,EAAE;MACV,CAAC,CAAC;IACN,CAAC,MAAM,IAAI,CAACP,KAAK,CAACQ,IAAI,EAAE;MACpB,OAAOT,QAAQ,CAACM,KAAK,CAAC;QAClBC,OAAO,EAAG,qCAAoC;QAC9CC,IAAI,EAAE;MACV,CAAC,CAAC;IACN,CAAC,MAAM,IAAIP,KAAK,CAACQ,IAAI,CAACC,IAAI,KAAKC,8BAAuB,CAACC,MAAM,EAAE;MAC3D,OAAOZ,QAAQ,CAACM,KAAK,CAAC;QAClBC,OAAO,EAAG,gCAA+BI,8BAAuB,CAACC,MAAO,cAAaX,KAAK,CAACQ,IAAI,CAACC,IAAK,IAAG;QACxGF,IAAI,EAAE;MACV,CAAC,CAAC;IACN;IAEA,MAAMK,YAAY,GAAG,MAAOC,EAAU,IAAuB;MACzD,IAAI;QACA,MAAMC,MAAM,GAAG,MAAMhB,OAAO,CAACiB,WAAW,CAACC,OAAO,CAACH,EAAE,CAAC;QACpD,OAAO,CAAC,CAACC,MAAM;MACnB,CAAC,CAAC,OAAOG,EAAE,EAAE;QACT,OAAO,KAAK;MAChB;IACJ,CAAC;IAED,MAAMH,MAAM,GAAGI,eAAe,CAAC;MAC3B,GAAGlB,KAAK;MACRmB,MAAM,EAAE;IACZ,CAAC,CAAC;IACF;AACR;AACA;IACQ,MAAMC,OAAO,GAAG,MAAM,IAAI,CAACzB,MAAM,CAAC0B,IAAI,CAACP,MAAM,CAACN,IAAI,CAACc,GAAG,CAAC;IACvD,IAAIF,OAAO,CAACG,MAAM,KAAK,CAAC,EAAE;MACtB,OAAOxB,QAAQ,CAACM,KAAK,CAAC;QAClBC,OAAO,EAAG,2CAA0C;QACpDC,IAAI,EAAE;MACV,CAAC,CAAC;IACN;;IAEA;AACR;AACA;AACA;IACQ,IAAIiB,QAAgC;IACpC,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACC,YAAY,CAACL,OAAO,EAAEN,MAAM,CAAC;IACvD,CAAC,CAAC,OAAOG,EAAE,EAAE;MACTS,OAAO,CAACrB,KAAK,CAACY,EAAE,CAAC;MACjB,OAAOlB,QAAQ,CAACM,KAAK,CAACY,EAAE,CAAC;IAC7B;IAEA,OAAO,IAAI,EAAE;MACT,IAAIhB,gBAAgB,CAAC,CAAC,EAAE;QACpB,OAAOF,QAAQ,CAAC4B,QAAQ,CAACb,MAAM,CAAC;MACpC,CAAC,MAAM,IAAIZ,SAAS,CAAC,CAAC,EAAE;QACpB,OAAOH,QAAQ,CAAC6B,OAAO,CAAC,CAAC;MAC7B,CAAC,MAAM,IAAId,MAAM,CAACK,MAAM,CAACI,MAAM,GAAGpB,eAAe,EAAE;QAC/C,OAAOJ,QAAQ,CAACM,KAAK,CAAC;UAClBC,OAAO,EAAG,sDAAqD;UAC/DC,IAAI,EAAE,iBAAiB;UACvBsB,IAAI,EAAE;YACFV,MAAM,EAAEL,MAAM,CAACK;UACnB;QACJ,CAAC,CAAC;MACN;MACA,MAAMW,MAAM,GAAG,IAAI,CAACC,mBAAmB,CAACP,QAAQ,EAAEV,MAAM,CAACkB,SAAS,CAAC;MACnE,IAAI,CAACF,MAAM,EAAE;QACT,OAAO/B,QAAQ,CAACkC,IAAI,CAAC,2BAA2B,CAAC;MACrD;MACAnB,MAAM,CAACkB,SAAS,GAAGF,MAAM,CAACjB,EAAE;MAC5B,MAAMqB,MAAM,GAAGd,OAAO,CAACe,IAAI,CAAC3B,IAAI,IAAIA,IAAI,CAAC4B,IAAI,KAAKN,MAAM,CAACR,GAAG,CAAC;MAC7D,IAAI,CAACY,MAAM,EAAE;QACTpB,MAAM,CAACK,MAAM,CAACkB,IAAI,CAAC;UACf7B,IAAI,EAAEsB,MAAM,CAACR,GAAG;UAChBhB,OAAO,EAAG;QACd,CAAC,CAAC;QACF;MACJ;MACA;AACZ;AACA;MACY,MAAMgC,gBAAgB,GAAG,MAAM,IAAI,CAAC5C,WAAW,CAAC6C,MAAM,CAACL,MAAM,CAACE,IAAI,CAAC;MACnE,IAAIE,gBAAgB,IAAI,CAACtC,KAAK,CAACwC,QAAQ,EAAE;QACrCd,OAAO,CAACe,GAAG,CACN,UAASX,MAAM,CAACjB,EAAG,OAAMqB,MAAM,CAACE,IAAK,mCAC1C,CAAC;QACD;MACJ;MACA;AACZ;AACA;MACY,MAAMG,MAAM,GAAG,MAAM3B,YAAY,CAACkB,MAAM,CAACjB,EAAE,CAAC;MAC5C,IAAI0B,MAAM,IAAI,CAACvC,KAAK,CAACwC,QAAQ,EAAE;QAC3Bd,OAAO,CAACe,GAAG,CACN,UAASX,MAAM,CAACjB,EAAG,OAAMqB,MAAM,CAACE,IAAK,qCAC1C,CAAC;QACD;MACJ;MACA;AACZ;AACA;MACY,IAAI,CAACE,gBAAgB,EAAE;QACnB,IAAI;UACA,MAAM9B,IAAI,GAAG,MAAM,IAAI,CAACZ,YAAY,CAAC8C,OAAO,CAAC;YACzCR,MAAM;YACNS,MAAM,EAAEb,MAAM,CAACR;UACnB,CAAC,CAAC;UACF,IAAI,CAACd,IAAI,EAAEoC,GAAG,EAAE;YACZ9B,MAAM,CAACK,MAAM,CAACkB,IAAI,CAAC;cACf7B,IAAI,EAAEsB,MAAM,CAACR,GAAG;cAChBhB,OAAO,EAAG,8BAA6B4B,MAAM,CAACE,IAAK,SAAQN,MAAM,CAACR,GAAI;YAC1E,CAAC,CAAC;YACF;UACJ;QACJ,CAAC,CAAC,OAAOL,EAAE,EAAE;UACTH,MAAM,CAACK,MAAM,CAACkB,IAAI,CAAC;YACf7B,IAAI,EAAEsB,MAAM,CAACR,GAAG;YAChBhB,OAAO,EAAEW,EAAE,CAACX;UAChB,CAAC,CAAC;UACF;QACJ;MACJ;MACA;AACZ;AACA;MACY,IAAIiC,MAAM,EAAE;QACR,IAAI;UACA,MAAMzC,OAAO,CAACiB,WAAW,CAAC8B,UAAU,CAACf,MAAM,CAACjB,EAAE,EAAE;YAC5C,GAAGiB;UACP,CAAC,CAAC;QACN,CAAC,CAAC,OAAOb,EAAE,EAAE;UACTH,MAAM,CAACK,MAAM,CAACkB,IAAI,CAAC;YACf7B,IAAI,EAAEsB,MAAM,CAACR,GAAG;YAChBhB,OAAO,EAAEW,EAAE,CAACX;UAChB,CAAC,CAAC;QACN;QACA;MACJ;MACA;AACZ;AACA;MACY,IAAI;QACA,MAAMR,OAAO,CAACiB,WAAW,CAAC+B,UAAU,CAAC;UACjC,GAAGhB,MAAM;UACTjB,EAAE,EAAEiB,MAAM,CAACjB,EAAE;UACbS,GAAG,EAAEQ,MAAM,CAACR,GAAG;UACfyB,IAAI,EAAEjB,MAAM,CAACiB,IAAI;UACjBtC,IAAI,EAAEqB,MAAM,CAACrB,IAAI;UACjBuC,IAAI,EAAElB,MAAM,CAACkB,IAAI;UACjBC,IAAI,EAAEnB,MAAM,CAACmB,IAAI;UACjBC,OAAO,EAAEpB,MAAM,CAACoB,OAAO;UACvBC,UAAU,EAAErB,MAAM,CAACqB,UAAU;UAC7BC,QAAQ,EAAEtB,MAAM,CAACsB,QAAQ;UACzBC,IAAI,EAAEvB,MAAM,CAACuB;QACjB,CAAC,CAAC;MACN,CAAC,CAAC,OAAOpC,EAAE,EAAE;QACTH,MAAM,CAACK,MAAM,CAACkB,IAAI,CAAC;UACf7B,IAAI,EAAEsB,MAAM,CAACR,GAAG;UAChBhB,OAAO,EAAEW,EAAE,CAACX;QAChB,CAAC,CAAC;MACN;IACJ;EACJ;EAEA,MAAcmB,YAAYA,CACtBL,OAAwB,EACxBpB,KAAQ,EACuB;IAC/B,IAAIwB,QAA4B,GAAGxB,KAAK,EAAEwB,QAAQ;IAElD,IAAI,CAACA,QAAQ,EAAE;MACX,MAAM8B,WAAW,GAAG,IAAAC,wBAAW,EAACvD,KAAK,CAACQ,IAAI,CAACc,GAAG,CAAC;MAC/C,MAAMY,MAAM,GAAGd,OAAO,CAACe,IAAI,CAAC3B,IAAI,IAAIA,IAAI,CAAC4B,IAAI,KAAKoB,wBAAa,CAAC;MAChE,IAAI,CAACtB,MAAM,EAAE;QACT,MAAM,IAAIuB,kBAAW,CAAC;UAClBnD,OAAO,EAAG,mDAAkD;UAC5DC,IAAI,EAAE;QACV,CAAC,CAAC;MACN;MACA,MAAMoC,MAAM,GAAI,aAAYW,WAAW,CAAClB,IAAK,IAAGF,MAAM,CAACE,IAAK,EAAC;MAC7D,IAAI;QACA,MAAM5B,IAAI,GAAG,MAAM,IAAI,CAACZ,YAAY,CAAC8C,OAAO,CAAC;UACzCR,MAAM;UACNS;QACJ,CAAC,CAAC;QACF,IAAI,CAACnC,IAAI,CAACoC,GAAG,EAAE;UACX,MAAM,IAAIc,KAAK,CAAE,8BAA6BxB,MAAM,CAACE,IAAK,SAAQO,MAAO,IAAG,CAAC;QACjF;QACAnB,QAAQ,GAAGhB,IAAI,CAACoC,GAAG;MACvB,CAAC,CAAC,OAAO3B,EAAE,EAAE;QACT,MAAM,IAAIwC,kBAAW,CAAC;UAClBnD,OAAO,EAAEW,EAAE,CAACX,OAAO;UACnBC,IAAI,EAAE;QACV,CAAC,CAAC;MACN;IACJ;IAEA,IAAIC,IAAmB;IACvB,IAAI;MACAA,IAAI,GAAG,MAAM,IAAI,CAACd,WAAW,CAAC2B,IAAI,CAACG,QAAQ,CAAC;MAC5C,IAAI,CAAChB,IAAI,EAAE;QACP,MAAM,IAAIiD,kBAAW,CAAC;UAClBnD,OAAO,EAAG,sCAAqCkB,QAAS,IAAG;UAC3DjB,IAAI,EAAE;QACV,CAAC,CAAC;MACN;IACJ,CAAC,CAAC,OAAOU,EAAE,EAAE;MACT,MAAM,IAAIwC,kBAAW,CAAC;QAClBnD,OAAO,EAAEW,EAAE,CAACX,OAAO;QACnBC,IAAI,EAAE,uBAAuB;QAC7BsB,IAAI,EAAEZ;MACV,CAAC,CAAC;IACN;IAEA,IAAI0C,IAA4B;IAChC,IAAI;MACAA,IAAI,GAAGC,IAAI,CAACC,KAAK,CAACrD,IAAI,CAAC;IAC3B,CAAC,CAAC,OAAOS,EAAE,EAAE;MACT,MAAM,IAAIwC,kBAAW,CAAC;QAClBnD,OAAO,EAAEW,EAAE,CAACX,OAAO;QACnBC,IAAI,EAAE,uBAAuB;QAC7BsB,IAAI,EAAEZ;MACV,CAAC,CAAC;IACN;IACA,IAAI,CAAC0C,IAAI,CAACG,MAAM,EAAEvC,MAAM,EAAE;MACtB,MAAM,IAAIkC,kBAAW,CAAC;QAClBnD,OAAO,EAAG,0BAAyBkB,QAAS,+BAA8B;QAC1EjB,IAAI,EAAE;MACV,CAAC,CAAC;IACN,CAAC,MAAM,IAAI,CAACoD,IAAI,CAACZ,IAAI,EAAE;MACnB,MAAM,IAAIU,kBAAW,CAAC;QAClBnD,OAAO,EAAG,0BAAyBkB,QAAS,6BAA4B;QACxEjB,IAAI,EAAE;MACV,CAAC,CAAC;IACN;IACA,OAAOoD,IAAI;EACf;EAEQ5B,mBAAmBA,CACvBP,QAAgC,EAChCQ,SAA6B,EACR;IACrB,IAAI,CAACA,SAAS,EAAE;MACZ,OAAOR,QAAQ,CAACsC,MAAM,CAAC,CAAC,CAAC;IAC7B;IACA,MAAMC,SAAS,GAAGvC,QAAQ,CAACsC,MAAM,CAACE,SAAS,CAACC,KAAK,IAAIA,KAAK,CAACpD,EAAE,KAAKmB,SAAS,CAAC;IAC5E,IAAI+B,SAAS,KAAK,CAAC,CAAC,EAAE;MAClB,OAAO,IAAI;IACf;IACA,MAAME,KAAK,GAAGzC,QAAQ,CAACsC,MAAM,CAACC,SAAS,GAAG,CAAC,CAAC;IAC5C,OAAOE,KAAK,IAAI,IAAI;EACxB;AACJ;AAACC,OAAA,CAAA3E,0BAAA,GAAAA,0BAAA","ignoreList":[]}

package/tasks/domain/importFromUrlProcessAssets/abstractions/ImportFromUrlProcessAssets.d.ts

@@ -0,0 +1,20 @@
+import type { ITaskResponseDoneResultOutput, ITaskResponseResult, ITaskRunParams } from "@webiny/tasks";
+import type { Context, ICmsImportExportValidatedValidFile } from "../../../../types";
+export type IImportFromUrlProcessAssetsInputFile = Pick<ICmsImportExportValidatedValidFile, "key" | "type">;
+export interface IImportFromUrlProcessAssetsInputError {
+    file: string;
+    message: string;
+}
+export interface IImportFromUrlProcessAssetsInput {
+    modelId: string;
+    file: IImportFromUrlProcessAssetsInputFile;
+    maxInsertErrors?: number;
+    override?: boolean;
+    manifest?: string;
+    lastAsset?: string;
+    errors?: IImportFromUrlProcessAssetsInputError[];
+}
+export type IImportFromUrlProcessAssetsOutput = ITaskResponseDoneResultOutput;
+export interface IImportFromUrlProcessAssets<C extends Context = Context, I extends IImportFromUrlProcessAssetsInput = IImportFromUrlProcessAssetsInput, O extends IImportFromUrlProcessAssetsOutput = IImportFromUrlProcessAssetsOutput> {
+    run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>>;
+}

package/tasks/domain/importFromUrlProcessAssets/abstractions/ImportFromUrlProcessAssets.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=ImportFromUrlProcessAssets.js.map

package/tasks/domain/importFromUrlProcessAssets/abstractions/ImportFromUrlProcessAssets.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ImportFromUrlProcessAssets.ts"],"sourcesContent":["import type {\n    ITaskResponseDoneResultOutput,\n    ITaskResponseResult,\n    ITaskRunParams\n} from \"@webiny/tasks\";\nimport type { Context, ICmsImportExportValidatedValidFile } from \"~/types\";\n\nexport type IImportFromUrlProcessAssetsInputFile = Pick<\n    ICmsImportExportValidatedValidFile,\n    \"key\" | \"type\"\n>;\n\nexport interface IImportFromUrlProcessAssetsInputError {\n    file: string;\n    message: string;\n}\n\nexport interface IImportFromUrlProcessAssetsInput {\n    modelId: string;\n    file: IImportFromUrlProcessAssetsInputFile;\n    maxInsertErrors?: number;\n    override?: boolean;\n    manifest?: string;\n    lastAsset?: string;\n    errors?: IImportFromUrlProcessAssetsInputError[];\n}\n\nexport type IImportFromUrlProcessAssetsOutput = ITaskResponseDoneResultOutput;\n\nexport interface IImportFromUrlProcessAssets<\n    C extends Context = Context,\n    I extends IImportFromUrlProcessAssetsInput = IImportFromUrlProcessAssetsInput,\n    O extends IImportFromUrlProcessAssetsOutput = IImportFromUrlProcessAssetsOutput\n> {\n    run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/domain/importFromUrlProcessEntries/ImportFromUrlProcessEntries.d.ts

@@ -0,0 +1,17 @@
+import type { Context } from "../../../types";
+import type { IImportFromUrlProcessEntries, IImportFromUrlProcessEntriesInput, IImportFromUrlProcessEntriesOutput } from "./abstractions/ImportFromUrlProcessEntries";
+import type { ITaskResponseResult, ITaskRunParams } from "@webiny/tasks";
+import type { IFileFetcher } from "../../utils/fileFetcher";
+import type { ICompressedFileReader, IDecompressor } from "../../utils/decompressor";
+export interface IImportFromUrlProcessEntriesParams {
+    fileFetcher: IFileFetcher;
+    reader: ICompressedFileReader;
+    decompressor: IDecompressor;
+}
+export declare class ImportFromUrlProcessEntries<C extends Context = Context, I extends IImportFromUrlProcessEntriesInput = IImportFromUrlProcessEntriesInput, O extends IImportFromUrlProcessEntriesOutput = IImportFromUrlProcessEntriesOutput> implements IImportFromUrlProcessEntries<C, I, O> {
+    private readonly fileFetcher;
+    private readonly reader;
+    private readonly decompressor;
+    constructor(params: IImportFromUrlProcessEntriesParams);
+    run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>>;
+}

package/tasks/domain/importFromUrlProcessEntries/ImportFromUrlProcessEntries.js

@@ -0,0 +1,82 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ImportFromUrlProcessEntries = void 0;
+var _types = require("../../../types");
+var _ImportFromUrlProcessEntriesDecompress = require("./ImportFromUrlProcessEntriesDecompress");
+var _ImportFromUrlProcessEntriesInsert = require("./ImportFromUrlProcessEntriesInsert");
+class ImportFromUrlProcessEntries {
+  constructor(params) {
+    this.fileFetcher = params.fileFetcher;
+    this.reader = params.reader;
+    this.decompressor = params.decompressor;
+  }
+  async run(params) {
+    const {
+      context,
+      response,
+      input
+    } = params;
+    if (!input.modelId) {
+      return response.error({
+        message: `Missing "modelId" in the input.`,
+        code: "MISSING_MODEL_ID"
+      });
+    } else if (!input.file) {
+      return response.error({
+        message: `No file found in the provided data.`,
+        code: "NO_FILE_FOUND"
+      });
+    } else if (input.file.type !== _types.CmsImportExportFileType.ENTRIES) {
+      return response.error({
+        message: `Invalid file type. Expected "${_types.CmsImportExportFileType.ENTRIES}" but got "${input.file.type}".`,
+        code: "INVALID_FILE_TYPE"
+      });
+    }
+    let entryManager;
+    try {
+      entryManager = await context.cms.getEntryManager(input.modelId);
+    } catch (ex) {
+      return response.error({
+        message: `Model "${input.modelId}" not found.`,
+        code: "MODEL_NOT_FOUND"
+      });
+    }
+    if (!input.decompress?.done) {
+      try {
+        const decompress = new _ImportFromUrlProcessEntriesDecompress.ImportFromUrlProcessEntriesDecompress({
+          reader: this.reader,
+          decompressor: this.decompressor
+        });
+        return await decompress.run(params);
+      } catch (ex) {
+        console.error(ex);
+        return response.error({
+          message: ex.message,
+          code: ex.code || "DECOMPRESS_ERROR",
+          data: ex.data,
+          stack: ex.stack
+        });
+      }
+    }
+    try {
+      const insert = new _ImportFromUrlProcessEntriesInsert.ImportFromUrlProcessEntriesInsert({
+        entryManager,
+        fileFetcher: this.fileFetcher
+      });
+      return await insert.run(params);
+    } catch (ex) {
+      return response.error({
+        message: ex.message,
+        code: ex.code || "DECOMPRESS_ERROR",
+        data: ex.data,
+        stack: ex.stack
+      });
+    }
+  }
+}
+exports.ImportFromUrlProcessEntries = ImportFromUrlProcessEntries;
+
+//# sourceMappingURL=ImportFromUrlProcessEntries.js.map

package/tasks/domain/importFromUrlProcessEntries/ImportFromUrlProcessEntries.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_types","require","_ImportFromUrlProcessEntriesDecompress","_ImportFromUrlProcessEntriesInsert","ImportFromUrlProcessEntries","constructor","params","fileFetcher","reader","decompressor","run","context","response","input","modelId","error","message","code","file","type","CmsImportExportFileType","ENTRIES","entryManager","cms","getEntryManager","ex","decompress","done","ImportFromUrlProcessEntriesDecompress","console","data","stack","insert","ImportFromUrlProcessEntriesInsert","exports"],"sources":["ImportFromUrlProcessEntries.ts"],"sourcesContent":["import type { Context } from \"~/types\";\nimport { CmsImportExportFileType } from \"~/types\";\nimport type {\n    IImportFromUrlProcessEntries,\n    IImportFromUrlProcessEntriesInput,\n    IImportFromUrlProcessEntriesOutput\n} from \"./abstractions/ImportFromUrlProcessEntries\";\nimport type { ITaskResponseResult, ITaskRunParams } from \"@webiny/tasks\";\nimport type { ICmsEntryManager } from \"@webiny/api-headless-cms/types\";\nimport { ImportFromUrlProcessEntriesDecompress } from \"~/tasks/domain/importFromUrlProcessEntries/ImportFromUrlProcessEntriesDecompress\";\nimport type { IFileFetcher } from \"~/tasks/utils/fileFetcher\";\nimport { ImportFromUrlProcessEntriesInsert } from \"./ImportFromUrlProcessEntriesInsert\";\nimport type { ICompressedFileReader, IDecompressor } from \"~/tasks/utils/decompressor\";\n\nexport interface IImportFromUrlProcessEntriesParams {\n    fileFetcher: IFileFetcher;\n    reader: ICompressedFileReader;\n    decompressor: IDecompressor;\n}\n\nexport class ImportFromUrlProcessEntries<\n    C extends Context = Context,\n    I extends IImportFromUrlProcessEntriesInput = IImportFromUrlProcessEntriesInput,\n    O extends IImportFromUrlProcessEntriesOutput = IImportFromUrlProcessEntriesOutput\n> implements IImportFromUrlProcessEntries<C, I, O>\n{\n    private readonly fileFetcher: IFileFetcher;\n    private readonly reader: ICompressedFileReader;\n    private readonly decompressor: IDecompressor;\n\n    public constructor(params: IImportFromUrlProcessEntriesParams) {\n        this.fileFetcher = params.fileFetcher;\n        this.reader = params.reader;\n        this.decompressor = params.decompressor;\n    }\n\n    public async run(params: ITaskRunParams<C, I, O>): Promise<ITaskResponseResult<I, O>> {\n        const { context, response, input } = params;\n\n        if (!input.modelId) {\n            return response.error({\n                message: `Missing \"modelId\" in the input.`,\n                code: \"MISSING_MODEL_ID\"\n            });\n        } else if (!input.file) {\n            return response.error({\n                message: `No file found in the provided data.`,\n                code: \"NO_FILE_FOUND\"\n            });\n        } else if (input.file.type !== CmsImportExportFileType.ENTRIES) {\n            return response.error({\n                message: `Invalid file type. Expected \"${CmsImportExportFileType.ENTRIES}\" but got \"${input.file.type}\".`,\n                code: \"INVALID_FILE_TYPE\"\n            });\n        }\n\n        let entryManager: ICmsEntryManager;\n        try {\n            entryManager = await context.cms.getEntryManager(input.modelId);\n        } catch (ex) {\n            return response.error({\n                message: `Model \"${input.modelId}\" not found.`,\n                code: \"MODEL_NOT_FOUND\"\n            });\n        }\n\n        if (!input.decompress?.done) {\n            try {\n                const decompress = new ImportFromUrlProcessEntriesDecompress<C, I, O>({\n                    reader: this.reader,\n                    decompressor: this.decompressor\n                });\n\n                return await decompress.run(params);\n            } catch (ex) {\n                console.error(ex);\n                return response.error({\n                    message: ex.message,\n                    code: ex.code || \"DECOMPRESS_ERROR\",\n                    data: ex.data,\n                    stack: ex.stack\n                });\n            }\n        }\n\n        try {\n            const insert = new ImportFromUrlProcessEntriesInsert<C, I, O>({\n                entryManager,\n                fileFetcher: this.fileFetcher\n            });\n            return await insert.run(params);\n        } catch (ex) {\n            return response.error({\n                message: ex.message,\n                code: ex.code || \"DECOMPRESS_ERROR\",\n                data: ex.data,\n                stack: ex.stack\n            });\n        }\n    }\n}\n"],"mappings":";;;;;;AACA,IAAAA,MAAA,GAAAC,OAAA;AAQA,IAAAC,sCAAA,GAAAD,OAAA;AAEA,IAAAE,kCAAA,GAAAF,OAAA;AASO,MAAMG,2BAA2B,CAKxC;EAKWC,WAAWA,CAACC,MAA0C,EAAE;IAC3D,IAAI,CAACC,WAAW,GAAGD,MAAM,CAACC,WAAW;IACrC,IAAI,CAACC,MAAM,GAAGF,MAAM,CAACE,MAAM;IAC3B,IAAI,CAACC,YAAY,GAAGH,MAAM,CAACG,YAAY;EAC3C;EAEA,MAAaC,GAAGA,CAACJ,MAA+B,EAAsC;IAClF,MAAM;MAAEK,OAAO;MAAEC,QAAQ;MAAEC;IAAM,CAAC,GAAGP,MAAM;IAE3C,IAAI,CAACO,KAAK,CAACC,OAAO,EAAE;MAChB,OAAOF,QAAQ,CAACG,KAAK,CAAC;QAClBC,OAAO,EAAG,iCAAgC;QAC1CC,IAAI,EAAE;MACV,CAAC,CAAC;IACN,CAAC,MAAM,IAAI,CAACJ,KAAK,CAACK,IAAI,EAAE;MACpB,OAAON,QAAQ,CAACG,KAAK,CAAC;QAClBC,OAAO,EAAG,qCAAoC;QAC9CC,IAAI,EAAE;MACV,CAAC,CAAC;IACN,CAAC,MAAM,IAAIJ,KAAK,CAACK,IAAI,CAACC,IAAI,KAAKC,8BAAuB,CAACC,OAAO,EAAE;MAC5D,OAAOT,QAAQ,CAACG,KAAK,CAAC;QAClBC,OAAO,EAAG,gCAA+BI,8BAAuB,CAACC,OAAQ,cAAaR,KAAK,CAACK,IAAI,CAACC,IAAK,IAAG;QACzGF,IAAI,EAAE;MACV,CAAC,CAAC;IACN;IAEA,IAAIK,YAA8B;IAClC,IAAI;MACAA,YAAY,GAAG,MAAMX,OAAO,CAACY,GAAG,CAACC,eAAe,CAACX,KAAK,CAACC,OAAO,CAAC;IACnE,CAAC,CAAC,OAAOW,EAAE,EAAE;MACT,OAAOb,QAAQ,CAACG,KAAK,CAAC;QAClBC,OAAO,EAAG,UAASH,KAAK,CAACC,OAAQ,cAAa;QAC9CG,IAAI,EAAE;MACV,CAAC,CAAC;IACN;IAEA,IAAI,CAACJ,KAAK,CAACa,UAAU,EAAEC,IAAI,EAAE;MACzB,IAAI;QACA,MAAMD,UAAU,GAAG,IAAIE,4EAAqC,CAAU;UAClEpB,MAAM,EAAE,IAAI,CAACA,MAAM;UACnBC,YAAY,EAAE,IAAI,CAACA;QACvB,CAAC,CAAC;QAEF,OAAO,MAAMiB,UAAU,CAAChB,GAAG,CAACJ,MAAM,CAAC;MACvC,CAAC,CAAC,OAAOmB,EAAE,EAAE;QACTI,OAAO,CAACd,KAAK,CAACU,EAAE,CAAC;QACjB,OAAOb,QAAQ,CAACG,KAAK,CAAC;UAClBC,OAAO,EAAES,EAAE,CAACT,OAAO;UACnBC,IAAI,EAAEQ,EAAE,CAACR,IAAI,IAAI,kBAAkB;UACnCa,IAAI,EAAEL,EAAE,CAACK,IAAI;UACbC,KAAK,EAAEN,EAAE,CAACM;QACd,CAAC,CAAC;MACN;IACJ;IAEA,IAAI;MACA,MAAMC,MAAM,GAAG,IAAIC,oEAAiC,CAAU;QAC1DX,YAAY;QACZf,WAAW,EAAE,IAAI,CAACA;MACtB,CAAC,CAAC;MACF,OAAO,MAAMyB,MAAM,CAACtB,GAAG,CAACJ,MAAM,CAAC;IACnC,CAAC,CAAC,OAAOmB,EAAE,EAAE;MACT,OAAOb,QAAQ,CAACG,KAAK,CAAC;QAClBC,OAAO,EAAES,EAAE,CAACT,OAAO;QACnBC,IAAI,EAAEQ,EAAE,CAACR,IAAI,IAAI,kBAAkB;QACnCa,IAAI,EAAEL,EAAE,CAACK,IAAI;QACbC,KAAK,EAAEN,EAAE,CAACM;MACd,CAAC,CAAC;IACN;EACJ;AACJ;AAACG,OAAA,CAAA9B,2BAAA,GAAAA,2BAAA","ignoreList":[]}

package/tasks/domain/importFromUrlProcessEntries/ImportFromUrlProcessEntriesDecompress.d.ts

@@ -0,0 +1,14 @@
+import type { ICompressedFileReader, IDecompressor } from "../../utils/decompressor";
+import type { IImportFromUrlProcessEntriesDecompress, IImportFromUrlProcessEntriesDecompressRunParams, IImportFromUrlProcessEntriesDecompressRunResult } from "./abstractions/ImportFromUrlProcessEntriesDecompress";
+import type { IImportFromUrlProcessEntriesInput, IImportFromUrlProcessEntriesOutput } from "./abstractions/ImportFromUrlProcessEntries";
+import type { Context } from "../../../types";
+export interface IImportFromUrlProcessEntriesDecompressParams {
+    reader: ICompressedFileReader;
+    decompressor: IDecompressor;
+}
+export declare class ImportFromUrlProcessEntriesDecompress<C extends Context = Context, I extends IImportFromUrlProcessEntriesInput = IImportFromUrlProcessEntriesInput, O extends IImportFromUrlProcessEntriesOutput = IImportFromUrlProcessEntriesOutput> implements IImportFromUrlProcessEntriesDecompress<C, I, O> {
+    private readonly reader;
+    private readonly decompressor;
+    constructor(params: IImportFromUrlProcessEntriesDecompressParams);
+    run(params: IImportFromUrlProcessEntriesDecompressRunParams<C, I, O>): Promise<IImportFromUrlProcessEntriesDecompressRunResult<I, O>>;
+}