@nrwl/nx-cloud 15.3.1 → 15.3.3-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/core/api/error-reporter.api.js +36 -1
- package/lib/core/api/run-group.api.js +73 -1
- package/lib/core/api/run-group.api.js.map +1 -1
- package/lib/core/commands/record-output.js +6 -2
- package/lib/core/commands/record-output.js.map +1 -1
- package/lib/core/commands/upload-and-show-run-details.js +2 -1
- package/lib/core/commands/upload-and-show-run-details.js.map +1 -1
- package/lib/core/file-storage/file-storage.js +1 -1
- package/lib/core/file-storage/file-storage.js.map +1 -1
- package/lib/core/runners/cloud-enabled/cloud-enabled-life-cycle.js +93 -1
- package/lib/core/runners/cloud-enabled/cloud-enabled.runner.js +320 -1
- package/lib/core/runners/cloud-enabled/cloud-enabled.runner.js.map +1 -1
- package/lib/core/runners/cloud-enabled/cloud-remote-cache.js +133 -1
- package/lib/core/runners/cloud-enabled/cloud-run.api.js +168 -1
- package/lib/core/runners/cloud-enabled/id-generator.js +16 -1
- package/lib/core/runners/distributed-agent/distributed-agent.api.js +93 -1
- package/lib/core/runners/distributed-agent/distributed-agent.impl.js +157 -1
- package/lib/core/runners/distributed-agent/execute-tasks.js +114 -1
- package/lib/core/runners/distributed-agent/invoke-tasks-using-nx-imperative-api.js +58 -1
- package/lib/core/runners/distributed-agent/invoke-tasks-using-run-many.js +97 -1
- package/lib/core/runners/distributed-execution/distributed-execution.api.d.ts +1 -1
- package/lib/core/runners/distributed-execution/distributed-execution.api.js +152 -1
- package/lib/core/runners/distributed-execution/distributed-execution.api.js.map +1 -1
- package/lib/core/runners/distributed-execution/distributed-execution.runner.js +118 -1
- package/lib/core/runners/distributed-execution/process-task.js +45 -1
- package/lib/core/runners/distributed-execution/process-tasks.js +67 -1
- package/lib/core/runners/distributed-execution/split-task-graph-into-stages.js +37 -1
- package/lib/core/runners/distributed-execution/task-graph-creator.js +100 -1
- package/lib/utilities/environment.js +8 -2
- package/lib/utilities/environment.js.map +1 -1
- package/lib/utilities/is-private-cloud.d.ts +1 -0
- package/lib/utilities/is-private-cloud.js +30 -0
- package/lib/utilities/is-private-cloud.js.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1,320 @@
|
|
|
1
|
-
const a1_0x3598=['runUrl','generateUniqueLinkId','__awaiter','warn','.commit','filter','getNxCacheDirectory','toString','ENCRYPTION_KEY','EndOfRunMessage','toISOString','CloudRemoteCache','ACCESS_TOKEN','taskId','subscribe','next','https://nx.app','agentRunningInDistributedExecution','../../terminal-output/message-reporter','store','join','cloud-enabled-runner','anyErrors','complete','DISTRIBUTED_TASK_EXECUTION_INTERNAL_ERROR_STATUS_CODE','../../file-storage/file-storage','./cloud-enabled-life-cycle','daemon','find','../../terminal-output/end-of-run-message','@nrwl/nx-cloud/lib/daemon/process-run-end','rxjs/internal/Subject','Task\x20with\x20hash\x20','VERBOSE_LOGGING','storedHashes','pathExists','waitForStoreRequestsToComplete','OutputObfuscator','all','tasks-hashes-','MessageReporter','message','\x20isn\x27t\x20recorded','../../../utilities/nx-imports','../../terminal-output/output-obfuscator','obfuscate','path','exit','getCIExecutionEnv','url','NX_CLOUD_DISTRIBUTED_EXECUTION_ID','Executed\x20tasks\x20with\x20hashes:\x20','/runs/','../../api/error-reporter.api','__esModule','Nx\x20Cloud\x20wasn\x27t\x20able\x20to\x20record\x20its\x20run.','getCIExecutionId','Nx\x20Cloud\x20Problems','env','stringify','endRun','getRunGroup','printMessages','./cloud-remote-cache','defineProperty','printCacheHitsMessage','ErrorReporterApi','skipNxCache','map','hash','startRun','assign','submitRunMetrics','delayedStoreRequests','local-cache-hit','CloudRunApi','lifeCycle','done','extractGitSha','cloudEnabledTasksRunner','Nx\x20Cloud\x20wasn\x27t\x20able\x20to\x20store\x20artifacts.','cacheStatus','FileStorage','parseCommand','getMachineInfo','error','../../../utilities/environment','push','requests','writeFileSync','forEach','CloudEnabledLifeCycle'];(function(_0x2df648,_0x3598d7){const _0x22bee6=function(_0x14476c){while(--_0x14476c){_0x2df648['push'](_0x2df648['shift']());}};_0x22bee6(++_0x3598d7);}(a1_0x3598,0x125));const a1_0x22be=function(_0x2df648,_0x3598d7){_0x2df648=_0x2df648-0x0;let _0x22bee6=a1_0x3598[_0x2df648];return _0x22bee6;};'use strict';var __awaiter=this&&this[a1_0x22be('0x4d')]||function(_0x15a58d,_0x3ca47e,_0xfc3ca5,_0x384c85){function _0x29382f(_0x5f0e09){return _0x5f0e09 instanceof _0xfc3ca5?_0x5f0e09:new _0xfc3ca5(function(_0x2d07ee){_0x2d07ee(_0x5f0e09);});}return new(_0xfc3ca5||(_0xfc3ca5=Promise))(function(_0x3c4ad2,_0x4a8b1e){function _0x2f6afd(_0x1c4e62){try{_0x1583f9(_0x384c85[a1_0x22be('0x5a')](_0x1c4e62));}catch(_0xa72699){_0x4a8b1e(_0xa72699);}}function _0x4a4c64(_0x178091){try{_0x1583f9(_0x384c85['throw'](_0x178091));}catch(_0x53571d){_0x4a8b1e(_0x53571d);}}function _0x1583f9(_0x51b76e){_0x51b76e[a1_0x22be('0x3c')]?_0x3c4ad2(_0x51b76e['value']):_0x29382f(_0x51b76e['value'])['then'](_0x2f6afd,_0x4a4c64);}_0x1583f9((_0x384c85=_0x384c85['apply'](_0x15a58d,_0x3ca47e||[]))[a1_0x22be('0x5a')]());});};Object[a1_0x22be('0x2f')](exports,a1_0x22be('0x25'),{'value':!![]});exports[a1_0x22be('0x3e')]=void 0x0;const message_reporter_1=require(a1_0x22be('0x1'));const end_of_run_message_1=require(a1_0x22be('0xc'));const output_obfuscator_1=require(a1_0x22be('0x1b'));const cloud_enabled_life_cycle_1=require(a1_0x22be('0x9'));const file_storage_1=require(a1_0x22be('0x8'));const e2e_encryption_1=require('../../file-storage/e2e-encryption');const environment_1=require(a1_0x22be('0x45'));const cloud_remote_cache_1=require(a1_0x22be('0x2e'));const cloud_run_api_1=require('./cloud-run.api');const fs_1=require('fs');const path=require(a1_0x22be('0x1d'));const path_1=require(a1_0x22be('0x1d'));const metric_logger_1=require('../../../utilities/metric-logger');const error_reporter_api_1=require(a1_0x22be('0x24'));const fs_extra_1=require('fs-extra');const id_generator_1=require('./id-generator');const {tasksRunner,output}=require(a1_0x22be('0x1a'));function createApi(_0x1fee84,_0x43d303,_0x5c0cfe){const _0x5eee67=(0x0,environment_1[a1_0x22be('0x43')])(_0x43d303);return new cloud_run_api_1[(a1_0x22be('0x3a'))](_0x1fee84,_0x5c0cfe,_0x43d303,_0x5eee67);}function storeTaskHashes(_0x2544b1,_0x1d984c,_0x267995){const _0xe0654e=JSON[a1_0x22be('0x2a')](_0x2544b1['map'](_0xe85e94=>({'taskId':_0xe85e94[a1_0x22be('0x58')],'hash':_0xe85e94['hash']})));if(environment_1[a1_0x22be('0x10')]){output['note']({'title':a1_0x22be('0x22')+_0xe0654e});}(0x0,fs_1[a1_0x22be('0x48')])(path[a1_0x22be('0x3')](_0x1d984c,a1_0x22be('0x16')+_0x267995),_0xe0654e);}function storeLocalCacheHits(_0x118be1,_0xf0ea36,_0x4a626e){const _0x112b04=_0x118be1['filter'](_0xb5b8c4=>_0xb5b8c4[a1_0x22be('0x40')]===a1_0x22be('0x39'))[a1_0x22be('0x33')](_0x2db97b=>_0x2db97b[a1_0x22be('0x34')]);_0x112b04[a1_0x22be('0x49')](_0x69d905=>_0xf0ea36[a1_0x22be('0x2')](_0x69d905,_0x4a626e));}function onComplete({daemon,options,fileStorage,remoteCache,api,outputObfuscator,runStartTime,messages,endOfRunMessage,taskExecutions,versionOfNxBefore133,inner,encryptionKey,storeInCurrentProcess,distributedExecutionId,runContext}){return __awaiter(this,void 0x0,void 0x0,function*(){const _0x1002f9=new Date()[a1_0x22be('0x55')]();const _0x5fa396=(0x0,environment_1['getBranch'])();const _0xcb8d4a={'command':outputObfuscator[a1_0x22be('0x1c')]((0x0,environment_1[a1_0x22be('0x42')])()),'startTime':runStartTime,'endTime':_0x1002f9,'distributedExecutionId':distributedExecutionId,'branch':_0x5fa396,'runGroup':(0x0,environment_1[a1_0x22be('0x2c')])(),'sha':_0x5fa396?(0x0,environment_1[a1_0x22be('0x3d')])():undefined,'inner':inner};const _0x5421af={'branch':_0x5fa396,'runGroup':(0x0,environment_1['getRunGroup'])(),'ciExecutionId':(0x0,environment_1[a1_0x22be('0x27')])(),'ciExecutionEnv':(0x0,environment_1[a1_0x22be('0x1f')])()};if(storeInCurrentProcess){if((0x0,environment_1['agentRunningInDistributedExecution'])(distributedExecutionId)){const _0xf281bf=(0x0,environment_1[a1_0x22be('0x51')])(options);storeTaskHashes(taskExecutions,_0xf281bf,distributedExecutionId);storeLocalCacheHits(taskExecutions,remoteCache,_0xf281bf);}try{yield remoteCache[a1_0x22be('0x13')]();}catch(_0x138f41){output[a1_0x22be('0x44')]({'title':a1_0x22be('0x3f')});messages[a1_0x22be('0x2d')]();return![];}for(const _0x211dba of fileStorage[a1_0x22be('0x11')]){const _0x5c21f2=taskExecutions[a1_0x22be('0xb')](_0x3b3a48=>_0x3b3a48[a1_0x22be('0x34')]===_0x211dba);if(!_0x5c21f2){throw new Error(a1_0x22be('0xf')+_0x211dba+a1_0x22be('0x19'));}_0x5c21f2['uploadedToStorage']=!![];}try{yield api[a1_0x22be('0x2b')](_0xcb8d4a,taskExecutions,_0x5421af);}catch(_0x5c25d2){output[a1_0x22be('0x44')]({'title':a1_0x22be('0x26')});messages[a1_0x22be('0x2d')]();return![];}yield(0x0,metric_logger_1[a1_0x22be('0x37')])(options);}else{try{const _0x4a32fa=environment_1[a1_0x22be('0x57')]?environment_1[a1_0x22be('0x57')]:options['accessToken'];const _0x47b6af=(0x0,id_generator_1[a1_0x22be('0x4c')])();yield daemon['processInBackground'](a1_0x22be('0xd'),{'encryptionKey':encryptionKey,'runnerOptions':Object[a1_0x22be('0x36')](Object[a1_0x22be('0x36')]({},options),{'accessToken':_0x4a32fa}),'delayedStoreRequests':remoteCache[a1_0x22be('0x38')],'ciExecutionContext':_0x5421af,'runEnd':{'runData':_0xcb8d4a,'taskExecutions':taskExecutions,'linkId':_0x47b6af}});runContext[a1_0x22be('0x4b')]=(options[a1_0x22be('0x20')]||a1_0x22be('0x5b'))+a1_0x22be('0x23')+_0x47b6af;}catch(_0x184a1c){output[a1_0x22be('0x4e')]({'title':a1_0x22be('0x28'),'bodyLines':[_0x184a1c[a1_0x22be('0x18')]||_0x184a1c[a1_0x22be('0x52')]()]});return![];}}if(versionOfNxBefore133){setTimeout(()=>{messages[a1_0x22be('0x2d')]();if(!messages[a1_0x22be('0x5')]&&!inner){endOfRunMessage[a1_0x22be('0x30')]();}},0x0);}else{messages['printMessages']();if(!messages['anyErrors']&&!inner){endOfRunMessage[a1_0x22be('0x30')]();}}return!![];});}function createLifeCycle(_0x213e10,_0x2fe2cf,_0x23a0b4,_0x2c89bd){const _0x514133=new cloud_enabled_life_cycle_1[(a1_0x22be('0x4a'))](_0x213e10,(0x0,environment_1[a1_0x22be('0x51')])(_0x2fe2cf),!![],_0x2fe2cf['cacheableOperations']||[],_0x23a0b4,_0x2c89bd);try{const {CompositeLifeCycle}=require(a1_0x22be('0x1a'));if(!CompositeLifeCycle)return _0x514133;return new CompositeLifeCycle([_0x2fe2cf[a1_0x22be('0x3b')],_0x514133]);}catch(_0x196e8b){return _0x514133;}}function fetchUrlsForKnownHashesUpfront(_0xa67940,_0x4b5b57,_0x45466e,_0x18691d,_0x2d6232){return __awaiter(this,void 0x0,void 0x0,function*(){if(_0x18691d[a1_0x22be('0x32')])return;let _0x4cd039=_0x45466e[a1_0x22be('0x33')](_0x1e2709=>_0x1e2709[a1_0x22be('0x34')])[a1_0x22be('0x50')](_0x3e7858=>!!_0x3e7858);const _0x424523=(0x0,environment_1['getNxCacheDirectory'])(_0x18691d);const _0x1123f2=yield Promise[a1_0x22be('0x15')](_0x4cd039[a1_0x22be('0x33')](_0x582525=>{const _0x37b46a=(0x0,path_1['join'])(_0x424523,_0x582525+a1_0x22be('0x4f'));return(0x0,fs_extra_1[a1_0x22be('0x12')])(_0x37b46a);}));const _0x4fc15b=[];for(let _0x2da5c5=0x0;_0x2da5c5<_0x1123f2['length'];++_0x2da5c5){if(!_0x1123f2[_0x2da5c5]){_0x4fc15b[a1_0x22be('0x46')](_0x4cd039[_0x2da5c5]);}}if(_0x4fc15b['length']>0x0){const _0x1c4365=_0xa67940[a1_0x22be('0x35')](_0x2d6232,_0x4fc15b);for(const _0x213e72 of _0x4fc15b){_0x4b5b57[a1_0x22be('0x47')][_0x213e72]=_0x1c4365;}}});}function cloudEnabledTasksRunner(_0x2c4feb,_0x2fecec,_0x4d5b0d,_0x273c5b=![]){var _0x4128af;const _0x48570b=process[a1_0x22be('0x29')][a1_0x22be('0x21')];const _0x38d418={'statuses':{},'scheduledTasks':[],'requests':{},'allTasks':_0x2c4feb};const _0x26f8da=_0x2fecec['lifeCycle']===undefined;const _0x49e91d=[];const _0x1a1225=new message_reporter_1[(a1_0x22be('0x17'))](_0x2fecec);const _0x2d3a0f=createApi(_0x1a1225,_0x2fecec,_0x38d418);const _0x1b4762=new end_of_run_message_1[(a1_0x22be('0x54'))](_0x38d418,_0x49e91d,_0x48570b);const _0x528499=new output_obfuscator_1[(a1_0x22be('0x14'))](_0x2fecec['maskedProperties']);const _0x19e63f=new Date()['toISOString']();const _0x39be7e=createLifeCycle(_0x38d418,_0x2fecec,_0x528499,_0x49e91d);const _0x34eff7=environment_1[a1_0x22be('0x53')]||_0x2fecec['encryptionKey'];const _0x4881af=new e2e_encryption_1['E2EEncryption'](_0x34eff7);const _0x9fe864=new error_reporter_api_1[(a1_0x22be('0x31'))](_0x2fecec);const _0x542c59=(0x0,environment_1[a1_0x22be('0x0')])(_0x48570b)||!((_0x4128af=_0x4d5b0d[a1_0x22be('0xa')])===null||_0x4128af===void 0x0?void 0x0:_0x4128af['enabled']());const _0x512f1f=new file_storage_1[(a1_0x22be('0x41'))](_0x4881af,_0x9fe864,_0x2fecec,a1_0x22be('0x4'));const _0x4f838b=new cloud_remote_cache_1[(a1_0x22be('0x56'))](_0x1a1225,_0x2d3a0f,_0x38d418,_0x512f1f,_0x48570b,_0x542c59);fetchUrlsForKnownHashesUpfront(_0x2d3a0f,_0x38d418,_0x2c4feb,_0x2fecec,_0x48570b);delete process['env'][a1_0x22be('0x21')];const _0x4ccef6=tasksRunner(_0x2c4feb,Object[a1_0x22be('0x36')](Object[a1_0x22be('0x36')]({},_0x2fecec),{'remoteCache':_0x4f838b,'lifeCycle':_0x39be7e}),_0x4d5b0d);if(_0x4ccef6[a1_0x22be('0x59')]){const {Subject}=require(a1_0x22be('0xe'));const _0x49137e=new Subject();_0x4ccef6[a1_0x22be('0x59')]({'next':_0x1b78fd=>_0x49137e[a1_0x22be('0x5a')](_0x1b78fd),'error':_0x35510e=>_0x49137e[a1_0x22be('0x44')](_0x35510e),'complete':()=>__awaiter(this,void 0x0,void 0x0,function*(){const _0xee27cf=yield onComplete({'daemon':_0x4d5b0d['daemon'],'options':_0x2fecec,'fileStorage':_0x512f1f,'remoteCache':_0x4f838b,'api':_0x2d3a0f,'outputObfuscator':_0x528499,'runStartTime':_0x19e63f,'messages':_0x1a1225,'endOfRunMessage':_0x1b4762,'taskExecutions':_0x49e91d,'versionOfNxBefore133':_0x26f8da,'inner':_0x273c5b,'encryptionKey':_0x34eff7,'storeInCurrentProcess':_0x542c59,'runContext':_0x38d418,'distributedExecutionId':_0x48570b});if(!_0xee27cf&&(0x0,environment_1[a1_0x22be('0x0')])(_0x48570b)){process['exit'](environment_1[a1_0x22be('0x7')]);}_0x49137e[a1_0x22be('0x6')]();})});return _0x49137e;}else{return _0x4ccef6['then'](_0x3fbe28=>__awaiter(this,void 0x0,void 0x0,function*(){const _0x4aa8d7=yield onComplete({'daemon':_0x4d5b0d['daemon'],'options':_0x2fecec,'fileStorage':_0x512f1f,'remoteCache':_0x4f838b,'api':_0x2d3a0f,'outputObfuscator':_0x528499,'runStartTime':_0x19e63f,'messages':_0x1a1225,'endOfRunMessage':_0x1b4762,'taskExecutions':_0x49e91d,'versionOfNxBefore133':_0x26f8da,'inner':_0x273c5b,'encryptionKey':_0x34eff7,'storeInCurrentProcess':_0x542c59,'runContext':_0x38d418,'distributedExecutionId':_0x48570b});if(!_0x4aa8d7&&(0x0,environment_1['agentRunningInDistributedExecution'])(_0x48570b)){process[a1_0x22be('0x1e')](environment_1[a1_0x22be('0x7')]);}return _0x3fbe28;}))['catch'](_0xa76176=>__awaiter(this,void 0x0,void 0x0,function*(){const _0x4ff253=yield onComplete({'daemon':_0x4d5b0d[a1_0x22be('0xa')],'options':_0x2fecec,'fileStorage':_0x512f1f,'remoteCache':_0x4f838b,'api':_0x2d3a0f,'outputObfuscator':_0x528499,'runStartTime':_0x19e63f,'messages':_0x1a1225,'endOfRunMessage':_0x1b4762,'taskExecutions':_0x49e91d,'versionOfNxBefore133':_0x26f8da,'inner':_0x273c5b,'encryptionKey':_0x34eff7,'storeInCurrentProcess':_0x542c59,'runContext':_0x38d418,'distributedExecutionId':_0x48570b});if(!_0x4ff253&&(0x0,environment_1['agentRunningInDistributedExecution'])(_0x48570b)){process[a1_0x22be('0x1e')](environment_1[a1_0x22be('0x7')]);}throw _0xa76176;}));}}exports[a1_0x22be('0x3e')]=cloudEnabledTasksRunner;
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.cloudEnabledTasksRunner = void 0;
|
|
13
|
+
const message_reporter_1 = require("../../terminal-output/message-reporter");
|
|
14
|
+
const end_of_run_message_1 = require("../../terminal-output/end-of-run-message");
|
|
15
|
+
const output_obfuscator_1 = require("../../terminal-output/output-obfuscator");
|
|
16
|
+
const cloud_enabled_life_cycle_1 = require("./cloud-enabled-life-cycle");
|
|
17
|
+
const file_storage_1 = require("../../file-storage/file-storage");
|
|
18
|
+
const e2e_encryption_1 = require("../../file-storage/e2e-encryption");
|
|
19
|
+
const environment_1 = require("../../../utilities/environment");
|
|
20
|
+
const cloud_remote_cache_1 = require("./cloud-remote-cache");
|
|
21
|
+
const cloud_run_api_1 = require("./cloud-run.api");
|
|
22
|
+
const fs_1 = require("fs");
|
|
23
|
+
const path = require("path");
|
|
24
|
+
const path_1 = require("path");
|
|
25
|
+
const metric_logger_1 = require("../../../utilities/metric-logger");
|
|
26
|
+
const error_reporter_api_1 = require("../../api/error-reporter.api");
|
|
27
|
+
const fs_extra_1 = require("fs-extra");
|
|
28
|
+
const id_generator_1 = require("./id-generator");
|
|
29
|
+
const { tasksRunner, output } = require('../../../utilities/nx-imports');
|
|
30
|
+
function createApi(errors, options, runContext) {
|
|
31
|
+
const machineInfo = (0, environment_1.getMachineInfo)(options);
|
|
32
|
+
return new cloud_run_api_1.CloudRunApi(errors, runContext, options, machineInfo);
|
|
33
|
+
}
|
|
34
|
+
function storeTaskHashes(taskExecutions, directory, distributedExecutionId) {
|
|
35
|
+
const hashes = JSON.stringify(taskExecutions.map((t) => ({
|
|
36
|
+
taskId: t.taskId,
|
|
37
|
+
hash: t.hash,
|
|
38
|
+
})));
|
|
39
|
+
if (environment_1.VERBOSE_LOGGING) {
|
|
40
|
+
output.note({
|
|
41
|
+
title: `Executed tasks with hashes: ${hashes}`,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
(0, fs_1.writeFileSync)(path.join(directory, `tasks-hashes-${distributedExecutionId}`), hashes);
|
|
45
|
+
}
|
|
46
|
+
/*
|
|
47
|
+
* It can happen that a task is executed multiple times on an agent, say for DTE1 and DTE2.
|
|
48
|
+
* When it runs for DTE2, it will be a local cache hit, so it will be marked as completed.
|
|
49
|
+
* However, Nx won't upload an artifact to S3 in this case. Usually this isn't a problem,
|
|
50
|
+
* cause the first DTE would have uploaded it. But if you are using a DTE, with a read-only token,
|
|
51
|
+
* the uploaded artifact will have a different scoped hash.
|
|
52
|
+
*
|
|
53
|
+
* To address this: we are finding all local cache hits, and upload everything that has a put url.
|
|
54
|
+
*/
|
|
55
|
+
function storeLocalCacheHits(taskExecutions, remoteCache, nxCacheDirectory) {
|
|
56
|
+
const hashes = taskExecutions
|
|
57
|
+
.filter((t) => t.cacheStatus === 'local-cache-hit')
|
|
58
|
+
.map((t) => t.hash);
|
|
59
|
+
hashes.forEach((hash) => remoteCache.store(hash, nxCacheDirectory));
|
|
60
|
+
}
|
|
61
|
+
function onComplete({ daemon, options, fileStorage, remoteCache, api, outputObfuscator, runStartTime, messages, endOfRunMessage, taskExecutions, versionOfNxBefore133, inner, encryptionKey, storeInCurrentProcess, distributedExecutionId, runContext, }) {
|
|
62
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
63
|
+
const runEndTime = new Date().toISOString();
|
|
64
|
+
const branch = (0, environment_1.getBranch)();
|
|
65
|
+
const runData = {
|
|
66
|
+
command: outputObfuscator.obfuscate((0, environment_1.parseCommand)()),
|
|
67
|
+
startTime: runStartTime,
|
|
68
|
+
endTime: runEndTime,
|
|
69
|
+
distributedExecutionId,
|
|
70
|
+
branch: branch,
|
|
71
|
+
runGroup: (0, environment_1.getRunGroup)(),
|
|
72
|
+
// only set sha if branch is set because we invoke a separate process,
|
|
73
|
+
// which adds a few millis
|
|
74
|
+
sha: branch ? (0, environment_1.extractGitSha)() : undefined,
|
|
75
|
+
inner: inner,
|
|
76
|
+
};
|
|
77
|
+
const ciExecutionContext = {
|
|
78
|
+
branch: branch,
|
|
79
|
+
runGroup: (0, environment_1.getRunGroup)(),
|
|
80
|
+
ciExecutionId: (0, environment_1.getCIExecutionId)(),
|
|
81
|
+
ciExecutionEnv: (0, environment_1.getCIExecutionEnv)(),
|
|
82
|
+
};
|
|
83
|
+
if (storeInCurrentProcess) {
|
|
84
|
+
if ((0, environment_1.agentRunningInDistributedExecution)(distributedExecutionId)) {
|
|
85
|
+
const nxCacheDirectory = (0, environment_1.getNxCacheDirectory)(options);
|
|
86
|
+
storeTaskHashes(taskExecutions, nxCacheDirectory, distributedExecutionId);
|
|
87
|
+
storeLocalCacheHits(taskExecutions, remoteCache, nxCacheDirectory);
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
yield remoteCache.waitForStoreRequestsToComplete();
|
|
91
|
+
}
|
|
92
|
+
catch (e) {
|
|
93
|
+
output.error({
|
|
94
|
+
title: `Nx Cloud wasn't able to store artifacts.`,
|
|
95
|
+
});
|
|
96
|
+
messages.printMessages();
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
for (const hash of fileStorage.storedHashes) {
|
|
100
|
+
const matchingTask = taskExecutions.find((t) => t.hash === hash);
|
|
101
|
+
if (!matchingTask) {
|
|
102
|
+
throw new Error(`Task with hash ${hash} isn't recorded`);
|
|
103
|
+
}
|
|
104
|
+
matchingTask.uploadedToStorage = true;
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
yield api.endRun(runData, taskExecutions, ciExecutionContext);
|
|
108
|
+
}
|
|
109
|
+
catch (e) {
|
|
110
|
+
output.error({
|
|
111
|
+
title: `Nx Cloud wasn't able to record its run.`,
|
|
112
|
+
});
|
|
113
|
+
messages.printMessages();
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
yield (0, metric_logger_1.submitRunMetrics)(options);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
try {
|
|
120
|
+
const accessToken = environment_1.ACCESS_TOKEN ? environment_1.ACCESS_TOKEN : options.accessToken;
|
|
121
|
+
const linkId = (0, id_generator_1.generateUniqueLinkId)();
|
|
122
|
+
yield daemon.processInBackground('@nrwl/nx-cloud/lib/daemon/process-run-end', {
|
|
123
|
+
encryptionKey,
|
|
124
|
+
runnerOptions: Object.assign(Object.assign({}, options), { accessToken }),
|
|
125
|
+
delayedStoreRequests: remoteCache.delayedStoreRequests,
|
|
126
|
+
ciExecutionContext,
|
|
127
|
+
runEnd: {
|
|
128
|
+
runData,
|
|
129
|
+
taskExecutions,
|
|
130
|
+
linkId,
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
runContext.runUrl = `${options.url || 'https://nx.app'}/runs/${linkId}`;
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
output.warn({
|
|
137
|
+
title: `Nx Cloud Problems`,
|
|
138
|
+
bodyLines: [e.message || e.toString()],
|
|
139
|
+
});
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// this workaround is required because prior to Nx 13.3 we printed
|
|
144
|
+
// the end message after the runner completes, so we need to wait for the
|
|
145
|
+
// message to appear in the right place
|
|
146
|
+
if (versionOfNxBefore133) {
|
|
147
|
+
setTimeout(() => {
|
|
148
|
+
messages.printMessages();
|
|
149
|
+
if (!messages.anyErrors && !inner) {
|
|
150
|
+
endOfRunMessage.printCacheHitsMessage();
|
|
151
|
+
}
|
|
152
|
+
}, 0);
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
messages.printMessages();
|
|
156
|
+
if (!messages.anyErrors && !inner) {
|
|
157
|
+
endOfRunMessage.printCacheHitsMessage();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return true;
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
function createLifeCycle(runContext, options, outputObfuscator, tasks) {
|
|
164
|
+
const cloudEnabledLifeCycle = new cloud_enabled_life_cycle_1.CloudEnabledLifeCycle(runContext, (0, environment_1.getNxCacheDirectory)(options), true, options.cacheableOperations || [], outputObfuscator, tasks);
|
|
165
|
+
try {
|
|
166
|
+
const { CompositeLifeCycle } = require('../../../utilities/nx-imports');
|
|
167
|
+
if (!CompositeLifeCycle)
|
|
168
|
+
return cloudEnabledLifeCycle;
|
|
169
|
+
return new CompositeLifeCycle([options.lifeCycle, cloudEnabledLifeCycle]);
|
|
170
|
+
}
|
|
171
|
+
catch (e) {
|
|
172
|
+
return cloudEnabledLifeCycle;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function fetchUrlsForKnownHashesUpfront(api, runContext, tasks, options, distributedExecutionId) {
|
|
176
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
177
|
+
if (options.skipNxCache)
|
|
178
|
+
return;
|
|
179
|
+
let hashes = tasks.map((t) => t.hash).filter((h) => !!h);
|
|
180
|
+
const cachePath = (0, environment_1.getNxCacheDirectory)(options);
|
|
181
|
+
// filter out the urls that don't have artifact stored locally
|
|
182
|
+
const exists = yield Promise.all(hashes.map((hash) => {
|
|
183
|
+
const tdCommit = (0, path_1.join)(cachePath, `${hash}.commit`);
|
|
184
|
+
return (0, fs_extra_1.pathExists)(tdCommit);
|
|
185
|
+
}));
|
|
186
|
+
const missingHashes = [];
|
|
187
|
+
for (let i = 0; i < exists.length; ++i) {
|
|
188
|
+
if (!exists[i]) {
|
|
189
|
+
missingHashes.push(hashes[i]);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
if (missingHashes.length > 0) {
|
|
193
|
+
const startRunRequest = api.startRun(distributedExecutionId, missingHashes);
|
|
194
|
+
for (const hash of missingHashes) {
|
|
195
|
+
runContext.requests[hash] = startRunRequest;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
function cloudEnabledTasksRunner(tasks, options, context, inner = false) {
|
|
201
|
+
var _a;
|
|
202
|
+
const distributedExecutionId = process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_ID;
|
|
203
|
+
const runContext = {
|
|
204
|
+
statuses: {},
|
|
205
|
+
scheduledTasks: [],
|
|
206
|
+
requests: {},
|
|
207
|
+
allTasks: tasks,
|
|
208
|
+
};
|
|
209
|
+
const versionOfNxBefore133 = options.lifeCycle === undefined;
|
|
210
|
+
const taskExecutions = [];
|
|
211
|
+
const messages = new message_reporter_1.MessageReporter(options);
|
|
212
|
+
const api = createApi(messages, options, runContext);
|
|
213
|
+
const endOfRunMessage = new end_of_run_message_1.EndOfRunMessage(runContext, taskExecutions, distributedExecutionId);
|
|
214
|
+
const outputObfuscator = new output_obfuscator_1.OutputObfuscator(options.maskedProperties);
|
|
215
|
+
const runStartTime = new Date().toISOString();
|
|
216
|
+
const lifeCycle = createLifeCycle(runContext, options, outputObfuscator, taskExecutions);
|
|
217
|
+
const encryptionKey = environment_1.ENCRYPTION_KEY || options.encryptionKey;
|
|
218
|
+
const encryption = new e2e_encryption_1.E2EEncryption(encryptionKey);
|
|
219
|
+
const errorApi = new error_reporter_api_1.ErrorReporterApi(options);
|
|
220
|
+
const storeInCurrentProcess = (0, environment_1.agentRunningInDistributedExecution)(distributedExecutionId) ||
|
|
221
|
+
!((_a = context.daemon) === null || _a === void 0 ? void 0 : _a.enabled());
|
|
222
|
+
const fileStorage = new file_storage_1.FileStorage(encryption, errorApi, options, 'cloud-enabled-runner');
|
|
223
|
+
const remoteCache = new cloud_remote_cache_1.CloudRemoteCache(messages, api, runContext, fileStorage, distributedExecutionId, storeInCurrentProcess);
|
|
224
|
+
// we don't await for it cause this step is optional
|
|
225
|
+
fetchUrlsForKnownHashesUpfront(api, runContext, tasks, options, distributedExecutionId);
|
|
226
|
+
// have to reset it so we don't capture inner tasks
|
|
227
|
+
delete process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_ID;
|
|
228
|
+
const res = tasksRunner(tasks, Object.assign(Object.assign({}, options), { remoteCache, lifeCycle }), context);
|
|
229
|
+
// observable -> legacy
|
|
230
|
+
if (res.subscribe) {
|
|
231
|
+
const { Subject } = require('rxjs/internal/Subject');
|
|
232
|
+
const wrappedRes = new Subject();
|
|
233
|
+
res.subscribe({
|
|
234
|
+
next: (value) => wrappedRes.next(value),
|
|
235
|
+
error: (err) => wrappedRes.error(err),
|
|
236
|
+
complete: () => __awaiter(this, void 0, void 0, function* () {
|
|
237
|
+
const success = yield onComplete({
|
|
238
|
+
daemon: context.daemon,
|
|
239
|
+
options,
|
|
240
|
+
fileStorage,
|
|
241
|
+
remoteCache,
|
|
242
|
+
api,
|
|
243
|
+
outputObfuscator,
|
|
244
|
+
runStartTime,
|
|
245
|
+
messages,
|
|
246
|
+
endOfRunMessage,
|
|
247
|
+
taskExecutions,
|
|
248
|
+
versionOfNxBefore133,
|
|
249
|
+
inner,
|
|
250
|
+
encryptionKey,
|
|
251
|
+
storeInCurrentProcess,
|
|
252
|
+
runContext,
|
|
253
|
+
distributedExecutionId,
|
|
254
|
+
});
|
|
255
|
+
if (!success &&
|
|
256
|
+
(0, environment_1.agentRunningInDistributedExecution)(distributedExecutionId)) {
|
|
257
|
+
process.exit(environment_1.DISTRIBUTED_TASK_EXECUTION_INTERNAL_ERROR_STATUS_CODE);
|
|
258
|
+
}
|
|
259
|
+
wrappedRes.complete();
|
|
260
|
+
}),
|
|
261
|
+
});
|
|
262
|
+
return wrappedRes;
|
|
263
|
+
// promise
|
|
264
|
+
}
|
|
265
|
+
else {
|
|
266
|
+
return res
|
|
267
|
+
.then((r) => __awaiter(this, void 0, void 0, function* () {
|
|
268
|
+
const success = yield onComplete({
|
|
269
|
+
daemon: context.daemon,
|
|
270
|
+
options,
|
|
271
|
+
fileStorage,
|
|
272
|
+
remoteCache,
|
|
273
|
+
api,
|
|
274
|
+
outputObfuscator,
|
|
275
|
+
runStartTime,
|
|
276
|
+
messages,
|
|
277
|
+
endOfRunMessage,
|
|
278
|
+
taskExecutions,
|
|
279
|
+
versionOfNxBefore133,
|
|
280
|
+
inner,
|
|
281
|
+
encryptionKey,
|
|
282
|
+
storeInCurrentProcess,
|
|
283
|
+
runContext,
|
|
284
|
+
distributedExecutionId,
|
|
285
|
+
});
|
|
286
|
+
if (!success &&
|
|
287
|
+
(0, environment_1.agentRunningInDistributedExecution)(distributedExecutionId)) {
|
|
288
|
+
process.exit(environment_1.DISTRIBUTED_TASK_EXECUTION_INTERNAL_ERROR_STATUS_CODE);
|
|
289
|
+
}
|
|
290
|
+
return r;
|
|
291
|
+
}))
|
|
292
|
+
.catch((e) => __awaiter(this, void 0, void 0, function* () {
|
|
293
|
+
const success = yield onComplete({
|
|
294
|
+
daemon: context.daemon,
|
|
295
|
+
options,
|
|
296
|
+
fileStorage,
|
|
297
|
+
remoteCache,
|
|
298
|
+
api,
|
|
299
|
+
outputObfuscator,
|
|
300
|
+
runStartTime,
|
|
301
|
+
messages,
|
|
302
|
+
endOfRunMessage,
|
|
303
|
+
taskExecutions,
|
|
304
|
+
versionOfNxBefore133,
|
|
305
|
+
inner,
|
|
306
|
+
encryptionKey,
|
|
307
|
+
storeInCurrentProcess,
|
|
308
|
+
runContext,
|
|
309
|
+
distributedExecutionId,
|
|
310
|
+
});
|
|
311
|
+
if (!success &&
|
|
312
|
+
(0, environment_1.agentRunningInDistributedExecution)(distributedExecutionId)) {
|
|
313
|
+
process.exit(environment_1.DISTRIBUTED_TASK_EXECUTION_INTERNAL_ERROR_STATUS_CODE);
|
|
314
|
+
}
|
|
315
|
+
throw e;
|
|
316
|
+
}));
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
exports.cloudEnabledTasksRunner = cloudEnabledTasksRunner;
|
|
320
|
+
//# sourceMappingURL=cloud-enabled.runner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cloud-enabled.runner.js","sourceRoot":"","sources":["../../../../../../../../libs/nx-packages/nx-cloud/lib/core/runners/cloud-enabled/cloud-enabled.runner.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,
|
|
1
|
+
{"version":3,"file":"cloud-enabled.runner.js","sourceRoot":"","sources":["../../../../../../../../libs/nx-packages/nx-cloud/lib/core/runners/cloud-enabled/cloud-enabled.runner.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,6EAAyE;AACzE,iFAA2E;AAC3E,+EAA2E;AAC3E,yEAAmE;AACnE,kEAA8D;AAC9D,sEAAkE;AAClE,gEAcwC;AACxC,6DAAwD;AACxD,mDAA8C;AAE9C,2BAAmC;AACnC,6BAA6B;AAC7B,+BAA4B;AAC5B,oEAAoE;AACpE,qEAAgE;AAChE,uCAAsC;AACtC,iDAAsD;AAEtD,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;AAEzE,SAAS,SAAS,CAChB,MAAuB,EACvB,OAA+B,EAC/B,UAAsB;IAEtB,MAAM,WAAW,GAAG,IAAA,4BAAc,EAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,IAAI,2BAAW,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,eAAe,CACtB,cAA+B,EAC/B,SAAiB,EACjB,sBAA8B;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAC3B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,IAAI,EAAE,CAAC,CAAC,IAAI;KACb,CAAC,CAAC,CACJ,CAAC;IAEF,IAAI,6BAAe,EAAE;QACnB,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,+BAA+B,MAAM,EAAE;SAC/C,CAAC,CAAC;KACJ;IAED,IAAA,kBAAa,EACX,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,sBAAsB,EAAE,CAAC,EAC9D,MAAM,CACP,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAC1B,cAA+B,EAC/B,WAA6B,EAC7B,gBAAgB;IAEhB,MAAM,MAAM,GAAG,cAAc;SAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,iBAAiB,CAAC;SAClD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAe,UAAU,CAAC,EACxB,MAAM,EACN,OAAO,EACP,WAAW,EACX,WAAW,EACX,GAAG,EACH,gBAAgB,EAChB,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,KAAK,EACL,aAAa,EACb,qBAAqB,EACrB,sBAAsB,EACtB,UAAU,GAkBX;;QACC,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAA,uBAAS,GAAE,CAAC;QAC3B,MAAM,OAAO,GAAG;YACd,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,IAAA,0BAAY,GAAE,CAAC;YACnD,SAAS,EAAE,YAAY;YACvB,OAAO,EAAE,UAAU;YACnB,sBAAsB;YACtB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,IAAA,yBAAW,GAAE;YACvB,sEAAsE;YACtE,0BAA0B;YAC1B,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,IAAA,2BAAa,GAAE,CAAC,CAAC,CAAC,SAAS;YACzC,KAAK,EAAE,KAAK;SACb,CAAC;QAEF,MAAM,kBAAkB,GAAG;YACzB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,IAAA,yBAAW,GAAE;YACvB,aAAa,EAAE,IAAA,8BAAgB,GAAE;YACjC,cAAc,EAAE,IAAA,+BAAiB,GAAE;SACpC,CAAC;QAEF,IAAI,qBAAqB,EAAE;YACzB,IAAI,IAAA,gDAAkC,EAAC,sBAAsB,CAAC,EAAE;gBAC9D,MAAM,gBAAgB,GAAG,IAAA,iCAAmB,EAAC,OAAO,CAAC,CAAC;gBACtD,eAAe,CAAC,cAAc,EAAE,gBAAgB,EAAE,sBAAsB,CAAC,CAAC;gBAC1E,mBAAmB,CAAC,cAAc,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;aACpE;YAED,IAAI;gBACF,MAAM,WAAW,CAAC,8BAA8B,EAAE,CAAC;aACpD;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC;oBACX,KAAK,EAAE,0CAA0C;iBAClD,CAAC,CAAC;gBACH,QAAQ,CAAC,aAAa,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;aACd;YAED,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,YAAY,EAAE;gBAC3C,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;gBACjE,IAAI,CAAC,YAAY,EAAE;oBACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,iBAAiB,CAAC,CAAC;iBAC1D;gBACD,YAAY,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACvC;YAED,IAAI;gBACF,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,kBAAkB,CAAC,CAAC;aAC/D;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC;oBACX,KAAK,EAAE,yCAAyC;iBACjD,CAAC,CAAC;gBACH,QAAQ,CAAC,aAAa,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;aACd;YAED,MAAM,IAAA,gCAAgB,EAAC,OAAO,CAAC,CAAC;SACjC;aAAM;YACL,IAAI;gBACF,MAAM,WAAW,GAAG,0BAAY,CAAC,CAAC,CAAC,0BAAY,CAAC,CAAC,CAAC,OAAO,CAAC,WAAY,CAAC;gBACvE,MAAM,MAAM,GAAG,IAAA,mCAAoB,GAAE,CAAC;gBACtC,MAAM,MAAM,CAAC,mBAAmB,CAC9B,2CAA2C,EAC3C;oBACE,aAAa;oBACb,aAAa,kCAAO,OAAO,KAAE,WAAW,GAAE;oBAC1C,oBAAoB,EAAE,WAAW,CAAC,oBAAoB;oBACtD,kBAAkB;oBAClB,MAAM,EAAE;wBACN,OAAO;wBACP,cAAc;wBACd,MAAM;qBACP;iBACF,CACF,CAAC;gBACF,UAAU,CAAC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,gBAAgB,SAAS,MAAM,EAAE,CAAC;aACzE;YAAC,OAAO,CAAM,EAAE;gBACf,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,mBAAmB;oBAC1B,SAAS,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACvC,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;aACd;SACF;QAED,kEAAkE;QAClE,yEAAyE;QACzE,uCAAuC;QACvC,IAAI,oBAAoB,EAAE;YACxB,UAAU,CAAC,GAAG,EAAE;gBACd,QAAQ,CAAC,aAAa,EAAE,CAAC;gBACzB,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,KAAK,EAAE;oBACjC,eAAe,CAAC,qBAAqB,EAAE,CAAC;iBACzC;YACH,CAAC,EAAE,CAAC,CAAC,CAAC;SACP;aAAM;YACL,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,KAAK,EAAE;gBACjC,eAAe,CAAC,qBAAqB,EAAE,CAAC;aACzC;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CAAA;AAED,SAAS,eAAe,CACtB,UAAsB,EACtB,OAA+B,EAC/B,gBAAkC,EAClC,KAAsB;IAEtB,MAAM,qBAAqB,GAAG,IAAI,gDAAqB,CACrD,UAAU,EACV,IAAA,iCAAmB,EAAC,OAAO,CAAC,EAC5B,IAAI,EACJ,OAAO,CAAC,mBAAmB,IAAI,EAAE,EACjC,gBAAgB,EAChB,KAAK,CACN,CAAC;IAEF,IAAI;QACF,MAAM,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;QACxE,IAAI,CAAC,kBAAkB;YAAE,OAAO,qBAAqB,CAAC;QACtD,OAAO,IAAI,kBAAkB,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,CAAC;KAC3E;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,qBAAqB,CAAC;KAC9B;AACH,CAAC;AAED,SAAe,8BAA8B,CAC3C,GAAgB,EAChB,UAAsB,EACtB,KAAa,EACb,OAA+B,EAC/B,sBAA0C;;QAE1C,IAAI,OAAO,CAAC,WAAW;YAAE,OAAO;QAEhC,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,IAAA,iCAAmB,EAAC,OAAO,CAAC,CAAC;QAC/C,8DAA8D;QAC9D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAClB,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,GAAG,IAAI,SAAS,CAAC,CAAC;YACnD,OAAO,IAAA,qBAAU,EAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,aAAa,GAAG,EAAc,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBACd,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aAC/B;SACF;QAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;YAC5B,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,EAAE,aAAa,CAAC,CAAC;YAC5E,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE;gBAChC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;aAC7C;SACF;IACH,CAAC;CAAA;AAED,SAAgB,uBAAuB,CACrC,KAAa,EACb,OAA+B,EAC/B,OAAY,EACZ,QAAiB,KAAK;;IAEtB,MAAM,sBAAsB,GAC1B,OAAO,CAAC,GAAG,CAAC,iCAAmC,CAAC;IAClD,MAAM,UAAU,GAAG;QACjB,QAAQ,EAAE,EAAE;QACZ,cAAc,EAAE,EAAE;QAClB,QAAQ,EAAE,EAAE;QACZ,QAAQ,EAAE,KAAK;KACF,CAAC;IAEhB,MAAM,oBAAoB,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC;IAC7D,MAAM,cAAc,GAAG,EAAqB,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,kCAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,eAAe,GAAG,IAAI,oCAAe,CACzC,UAAU,EACV,cAAc,EACd,sBAAsB,CACvB,CAAC;IAEF,MAAM,gBAAgB,GAAG,IAAI,oCAAgB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAExE,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,eAAe,CAC/B,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,cAAc,CACf,CAAC;IACF,MAAM,aAAa,GAAG,4BAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,8BAAa,CAAC,aAAa,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,qBAAqB,GACzB,IAAA,gDAAkC,EAAC,sBAAsB,CAAC;QAC1D,CAAC,CAAA,MAAA,OAAO,CAAC,MAAM,0CAAE,OAAO,EAAE,CAAA,CAAC;IAC7B,MAAM,WAAW,GAAG,IAAI,0BAAW,CACjC,UAAU,EACV,QAAQ,EACR,OAAO,EACP,sBAAsB,CACvB,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,qCAAgB,CACtC,QAAQ,EACR,GAAG,EACH,UAAU,EACV,WAAW,EACX,sBAAsB,EACtB,qBAAqB,CACtB,CAAC;IAEF,oDAAoD;IACpD,8BAA8B,CAC5B,GAAG,EACH,UAAU,EACV,KAAK,EACL,OAAO,EACP,sBAAsB,CACvB,CAAC;IAEF,mDAAmD;IACnD,OAAO,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC;IACrD,MAAM,GAAG,GAAG,WAAW,CACrB,KAAK,kCACA,OAAO,KAAE,WAAW,EAAE,SAAS,KACpC,OAAO,CACD,CAAC;IACT,uBAAuB;IACvB,IAAK,GAAW,CAAC,SAAS,EAAE;QAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,OAAO,EAAS,CAAC;QACxC,GAAG,CAAC,SAAS,CAAC;YACZ,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;YACvC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC;YACrC,QAAQ,EAAE,GAAS,EAAE;gBACnB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,OAAO;oBACP,WAAW;oBACX,WAAW;oBACX,GAAG;oBACH,gBAAgB;oBAChB,YAAY;oBACZ,QAAQ;oBACR,eAAe;oBACf,cAAc;oBACd,oBAAoB;oBACpB,KAAK;oBACL,aAAa;oBACb,qBAAqB;oBACrB,UAAU;oBACV,sBAAsB;iBACvB,CAAC,CAAC;gBACH,IACE,CAAC,OAAO;oBACR,IAAA,gDAAkC,EAAC,sBAAsB,CAAC,EAC1D;oBACA,OAAO,CAAC,IAAI,CAAC,mEAAqD,CAAC,CAAC;iBACrE;gBACD,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,CAAC,CAAA;SACF,CAAC,CAAC;QACH,OAAO,UAAU,CAAC;QAClB,UAAU;KACX;SAAM;QACL,OAAO,GAAG;aACP,IAAI,CAAC,CAAO,CAAC,EAAE,EAAE;YAChB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO;gBACP,WAAW;gBACX,WAAW;gBACX,GAAG;gBACH,gBAAgB;gBAChB,YAAY;gBACZ,QAAQ;gBACR,eAAe;gBACf,cAAc;gBACd,oBAAoB;gBACpB,KAAK;gBACL,aAAa;gBACb,qBAAqB;gBACrB,UAAU;gBACV,sBAAsB;aACvB,CAAC,CAAC;YACH,IACE,CAAC,OAAO;gBACR,IAAA,gDAAkC,EAAC,sBAAsB,CAAC,EAC1D;gBACA,OAAO,CAAC,IAAI,CAAC,mEAAqD,CAAC,CAAC;aACrE;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAA,CAAC;aACD,KAAK,CAAC,CAAO,CAAC,EAAE,EAAE;YACjB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO;gBACP,WAAW;gBACX,WAAW;gBACX,GAAG;gBACH,gBAAgB;gBAChB,YAAY;gBACZ,QAAQ;gBACR,eAAe;gBACf,cAAc;gBACd,oBAAoB;gBACpB,KAAK;gBACL,aAAa;gBACb,qBAAqB;gBACrB,UAAU;gBACV,sBAAsB;aACvB,CAAC,CAAC;YACH,IACE,CAAC,OAAO;gBACR,IAAA,gDAAkC,EAAC,sBAAsB,CAAC,EAC1D;gBACA,OAAO,CAAC,IAAI,CAAC,mEAAqD,CAAC,CAAC;aACrE;YACD,MAAM,CAAC,CAAC;QACV,CAAC,CAAA,CAAC,CAAC;KACN;AACH,CAAC;AAtKD,0DAsKC"}
|
|
@@ -1 +1,133 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CloudRemoteCache = void 0;
|
|
13
|
+
const environment_1 = require("../../../utilities/environment");
|
|
14
|
+
const { output } = require('../../../utilities/nx-imports');
|
|
15
|
+
class CloudRemoteCache {
|
|
16
|
+
constructor(messages, api, runContext, fileStorage, distributedExecutionId, storeInCurrentProcess) {
|
|
17
|
+
this.messages = messages;
|
|
18
|
+
this.api = api;
|
|
19
|
+
this.runContext = runContext;
|
|
20
|
+
this.fileStorage = fileStorage;
|
|
21
|
+
this.distributedExecutionId = distributedExecutionId;
|
|
22
|
+
this.storeInCurrentProcess = storeInCurrentProcess;
|
|
23
|
+
this.storeRequests = [];
|
|
24
|
+
this.delayedStoreRequests = [];
|
|
25
|
+
}
|
|
26
|
+
retrieve(hash, cacheDirectory) {
|
|
27
|
+
var _a;
|
|
28
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
29
|
+
if (this.messages.cacheError)
|
|
30
|
+
return false;
|
|
31
|
+
const hashUrls = yield this.hashUrls(hash);
|
|
32
|
+
if (!hashUrls || !hashUrls.get) {
|
|
33
|
+
if (environment_1.VERBOSE_LOGGING) {
|
|
34
|
+
output.note({ title: `Nx Cloud: Cache miss ${hash}.` });
|
|
35
|
+
}
|
|
36
|
+
this.runContext.statuses[hash] = 'cache-miss';
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
yield this.fileStorage.retrieve(hash, hashUrls.get, cacheDirectory);
|
|
41
|
+
this.runContext.statuses[hash] = 'remote-cache-hit';
|
|
42
|
+
return true;
|
|
43
|
+
}
|
|
44
|
+
catch (ee) {
|
|
45
|
+
const e = (_a = ee.axiosException) !== null && _a !== void 0 ? _a : ee;
|
|
46
|
+
// TODO: vsavkin remove this handling Oct 2023
|
|
47
|
+
// 404 indicates that something is broken, but older versions
|
|
48
|
+
// of the api can result in such errors
|
|
49
|
+
// we ignore them for now
|
|
50
|
+
if (e.response && e.response.status === 404) {
|
|
51
|
+
if (environment_1.VERBOSE_LOGGING) {
|
|
52
|
+
output.note({ title: `Nx Cloud: Cache miss ${hash}. Status 404.` });
|
|
53
|
+
}
|
|
54
|
+
// cache miss. print nothing
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
this.messages.cacheError = this.messages.extractErrorMessage(e, 'storage');
|
|
58
|
+
}
|
|
59
|
+
this.runContext.statuses[hash] = 'cache-miss';
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
store(hash, cacheDirectory) {
|
|
65
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
66
|
+
if (this.messages.cacheError)
|
|
67
|
+
return false;
|
|
68
|
+
const res = Promise.resolve().then(() => __awaiter(this, void 0, void 0, function* () {
|
|
69
|
+
var _a;
|
|
70
|
+
const hashUrls = yield this.hashUrls(hash);
|
|
71
|
+
if (!hashUrls)
|
|
72
|
+
return false;
|
|
73
|
+
if (!hashUrls.put) {
|
|
74
|
+
if (environment_1.VERBOSE_LOGGING) {
|
|
75
|
+
output.note({
|
|
76
|
+
title: `Nx Cloud: Skipping storing ${hash}.`,
|
|
77
|
+
bodyLines: [
|
|
78
|
+
`There are several reasons why this can happen.`,
|
|
79
|
+
`Maybe you are using a read-only token or the artifact has already being uploaded.`,
|
|
80
|
+
],
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
if (!this.storeInCurrentProcess) {
|
|
86
|
+
this.delayedStoreRequests.push({ hash: hash, url: hashUrls.put });
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
yield this.fileStorage.store(hash, hashUrls.put, cacheDirectory);
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
catch (ee) {
|
|
94
|
+
const e = (_a = ee.axiosException) !== null && _a !== void 0 ? _a : ee;
|
|
95
|
+
this.messages.cacheError = this.messages.extractErrorMessage(e, 'storage');
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
}));
|
|
99
|
+
this.storeRequests.push(res);
|
|
100
|
+
return res;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
hashUrls(hash) {
|
|
104
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
105
|
+
if (hash in this.runContext.requests) {
|
|
106
|
+
return (yield this.runContext.requests[hash])[hash];
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const scheduledTaskHashesWithoutRequests = this.runContext.scheduledTasks
|
|
110
|
+
.filter((t) => !this.runContext.requests[t.hash])
|
|
111
|
+
.map((t) => t.hash);
|
|
112
|
+
if (scheduledTaskHashesWithoutRequests.indexOf(hash) === -1) {
|
|
113
|
+
scheduledTaskHashesWithoutRequests.push(hash);
|
|
114
|
+
}
|
|
115
|
+
const request = this.api.startRun(this.distributedExecutionId, scheduledTaskHashesWithoutRequests);
|
|
116
|
+
scheduledTaskHashesWithoutRequests.forEach((taskHash) => {
|
|
117
|
+
this.runContext.requests[taskHash] = request;
|
|
118
|
+
});
|
|
119
|
+
return (yield request)[hash];
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
waitForStoreRequestsToComplete() {
|
|
124
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
125
|
+
const res = yield Promise.all(this.storeRequests).then((r) => r.reduce((a, b) => a && b, true));
|
|
126
|
+
if (!res) {
|
|
127
|
+
throw new Error(`Error when storing artifacts`);
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
exports.CloudRemoteCache = CloudRemoteCache;
|
|
133
|
+
//# sourceMappingURL=cloud-remote-cache.js.map
|