@sasjs/adapter 4.4.0 → 4.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SASViyaApiClient.d.ts +11 -6
- package/SASjs.d.ts +1 -1
- package/api/viya/executeScript.d.ts +1 -1
- package/api/viya/pollJobState.d.ts +33 -1
- package/index-dev.js +2 -2
- package/index-dev.js.LICENSE.txt +31 -2
- package/index.js +1 -1
- package/index.js.LICENSE.txt +31 -2
- package/minified_sas9.js +1 -1
- package/node/SASViyaApiClient.d.ts +11 -6
- package/node/SASjs.d.ts +1 -1
- package/node/api/viya/executeScript.d.ts +1 -1
- package/node/api/viya/pollJobState.d.ts +33 -1
- package/node/file/generateTableUploadForm.d.ts +1 -1
- package/node/index.js +1 -1
- package/node/index.js.LICENSE.txt +31 -0
- package/node/types/PollOptions.d.ts +3 -1
- package/node/utils/getFormData.d.ts +2 -0
- package/node/utils/index.d.ts +1 -0
- package/package.json +3 -3
- package/src/SASViyaApiClient.ts +30 -7
- package/src/SASjs.ts +1 -1
- package/src/api/viya/executeScript.ts +3 -3
- package/src/api/viya/pollJobState.ts +160 -81
- package/src/file/generateFileUploadForm.ts +6 -2
- package/src/job-execution/SasjsJobExecutor.ts +2 -3
- package/src/job-execution/WebJobExecutor.ts +4 -4
- package/src/types/PollOptions.ts +5 -2
- package/src/utils/getFormData.ts +5 -0
- package/src/utils/index.ts +1 -0
- package/types/PollOptions.d.ts +3 -1
- package/utils/getFormData.d.ts +2 -0
- package/utils/index.d.ts +1 -0
- package/utils/spec/getFormData.spec.d.ts +1 -0
|
@@ -29,6 +29,37 @@
|
|
|
29
29
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
+
/*!
|
|
33
|
+
* Copyright (c) 2015-2020, Salesforce.com, Inc.
|
|
34
|
+
* All rights reserved.
|
|
35
|
+
*
|
|
36
|
+
* Redistribution and use in source and binary forms, with or without
|
|
37
|
+
* modification, are permitted provided that the following conditions are met:
|
|
38
|
+
*
|
|
39
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
40
|
+
* this list of conditions and the following disclaimer.
|
|
41
|
+
*
|
|
42
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
43
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
44
|
+
* and/or other materials provided with the distribution.
|
|
45
|
+
*
|
|
46
|
+
* 3. Neither the name of Salesforce.com nor the names of its contributors may
|
|
47
|
+
* be used to endorse or promote products derived from this software without
|
|
48
|
+
* specific prior written permission.
|
|
49
|
+
*
|
|
50
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
51
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
52
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
53
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
54
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
55
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
56
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
57
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
58
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
59
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
60
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
61
|
+
*/
|
|
62
|
+
|
|
32
63
|
/*!
|
|
33
64
|
* Copyright (c) 2018, Salesforce.com, Inc.
|
|
34
65
|
* All rights reserved.
|
package/node/utils/index.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"@types/jest": "27.4.0",
|
|
50
50
|
"@types/mime": "2.0.3",
|
|
51
51
|
"@types/pem": "1.9.6",
|
|
52
|
-
"@types/tough-cookie": "4.0.
|
|
52
|
+
"@types/tough-cookie": "4.0.2",
|
|
53
53
|
"copyfiles": "2.4.1",
|
|
54
54
|
"cp": "0.2.0",
|
|
55
55
|
"cypress": "7.7.0",
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
"axios-cookiejar-support": "1.0.1",
|
|
83
83
|
"form-data": "4.0.0",
|
|
84
84
|
"https": "1.0.0",
|
|
85
|
-
"tough-cookie": "4.
|
|
85
|
+
"tough-cookie": "4.1.3"
|
|
86
86
|
},
|
|
87
|
-
"version": "4.
|
|
87
|
+
"version": "4.5.1"
|
|
88
88
|
}
|
package/src/SASViyaApiClient.ts
CHANGED
|
@@ -29,6 +29,12 @@ import { executeScript } from './api/viya/executeScript'
|
|
|
29
29
|
import { getAccessTokenForViya } from './auth/getAccessTokenForViya'
|
|
30
30
|
import { refreshTokensForViya } from './auth/refreshTokensForViya'
|
|
31
31
|
|
|
32
|
+
interface JobExecutionResult {
|
|
33
|
+
result?: { result: object }
|
|
34
|
+
log?: string
|
|
35
|
+
error?: object
|
|
36
|
+
}
|
|
37
|
+
|
|
32
38
|
/**
|
|
33
39
|
* A client for interfacing with the SAS Viya REST API.
|
|
34
40
|
*
|
|
@@ -270,7 +276,7 @@ export class SASViyaApiClient {
|
|
|
270
276
|
* @param debug - when set to true, the log will be returned.
|
|
271
277
|
* @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code).
|
|
272
278
|
* @param waitForResult - when set to true, function will return the session
|
|
273
|
-
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: {
|
|
279
|
+
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }. More information available at src/api/viya/pollJobState.ts.
|
|
274
280
|
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
|
|
275
281
|
* @param variables - an object that represents macro variables.
|
|
276
282
|
*/
|
|
@@ -621,7 +627,7 @@ export class SASViyaApiClient {
|
|
|
621
627
|
* @param accessToken - an optional access token for an authorized user.
|
|
622
628
|
* @param waitForResult - a boolean indicating if the function should wait for a result.
|
|
623
629
|
* @param expectWebout - a boolean indicating whether to expect a _webout response.
|
|
624
|
-
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: {
|
|
630
|
+
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }. More information available at src/api/viya/pollJobState.ts.
|
|
625
631
|
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
|
|
626
632
|
* @param variables - an object that represents macro variables.
|
|
627
633
|
*/
|
|
@@ -732,11 +738,13 @@ export class SASViyaApiClient {
|
|
|
732
738
|
debug: boolean,
|
|
733
739
|
data?: any,
|
|
734
740
|
authConfig?: AuthConfig
|
|
735
|
-
) {
|
|
741
|
+
): Promise<JobExecutionResult> {
|
|
736
742
|
let access_token = (authConfig || {}).access_token
|
|
743
|
+
|
|
737
744
|
if (authConfig) {
|
|
738
745
|
;({ access_token } = await getTokens(this.requestClient, authConfig))
|
|
739
746
|
}
|
|
747
|
+
|
|
740
748
|
if (isRelativePath(sasJob) && !this.rootFolderName) {
|
|
741
749
|
throw new Error(
|
|
742
750
|
'Relative paths cannot be used without specifying a root folder name.'
|
|
@@ -749,6 +757,7 @@ export class SASViyaApiClient {
|
|
|
749
757
|
const fullFolderPath = isRelativePath(sasJob)
|
|
750
758
|
? `${this.rootFolderName}/${folderPath}`
|
|
751
759
|
: folderPath
|
|
760
|
+
|
|
752
761
|
await this.populateFolderMap(fullFolderPath, access_token)
|
|
753
762
|
|
|
754
763
|
const jobFolder = this.folderMap.get(fullFolderPath)
|
|
@@ -765,9 +774,8 @@ export class SASViyaApiClient {
|
|
|
765
774
|
files = await this.uploadTables(data, access_token)
|
|
766
775
|
}
|
|
767
776
|
|
|
768
|
-
if (!jobToExecute)
|
|
769
|
-
|
|
770
|
-
}
|
|
777
|
+
if (!jobToExecute) throw new Error(`Job was not found.`)
|
|
778
|
+
|
|
771
779
|
const jobDefinitionLink = jobToExecute?.links.find(
|
|
772
780
|
(l) => l.rel === 'getResource'
|
|
773
781
|
)?.href
|
|
@@ -807,16 +815,19 @@ export class SASViyaApiClient {
|
|
|
807
815
|
jobDefinition,
|
|
808
816
|
arguments: jobArguments
|
|
809
817
|
}
|
|
818
|
+
|
|
810
819
|
const { result: postedJob } = await this.requestClient.post<Job>(
|
|
811
820
|
`${this.serverUrl}/jobExecution/jobs?_action=wait`,
|
|
812
821
|
postJobRequestBody,
|
|
813
822
|
access_token
|
|
814
823
|
)
|
|
824
|
+
|
|
815
825
|
const jobStatus = await this.pollJobState(postedJob, authConfig).catch(
|
|
816
826
|
(err) => {
|
|
817
827
|
throw prefixMessage(err, 'Error while polling job status. ')
|
|
818
828
|
}
|
|
819
829
|
)
|
|
830
|
+
|
|
820
831
|
const { result: currentJob } = await this.requestClient.get<Job>(
|
|
821
832
|
`${this.serverUrl}/jobExecution/jobs/${postedJob.id}`,
|
|
822
833
|
access_token
|
|
@@ -827,6 +838,7 @@ export class SASViyaApiClient {
|
|
|
827
838
|
|
|
828
839
|
const resultLink = currentJob.results['_webout.json']
|
|
829
840
|
const logLink = currentJob.links.find((l) => l.rel === 'log')
|
|
841
|
+
|
|
830
842
|
if (resultLink) {
|
|
831
843
|
jobResult = await this.requestClient.get<any>(
|
|
832
844
|
`${this.serverUrl}${resultLink}/content`,
|
|
@@ -834,11 +846,13 @@ export class SASViyaApiClient {
|
|
|
834
846
|
'text/plain'
|
|
835
847
|
)
|
|
836
848
|
}
|
|
849
|
+
|
|
837
850
|
if (debug && logLink) {
|
|
838
851
|
log = await this.requestClient
|
|
839
852
|
.get<any>(`${this.serverUrl}${logLink.href}/content`, access_token)
|
|
840
853
|
.then((res: any) => res.result.items.map((i: any) => i.line).join('\n'))
|
|
841
854
|
}
|
|
855
|
+
|
|
842
856
|
if (jobStatus === 'failed') {
|
|
843
857
|
throw new JobExecutionError(
|
|
844
858
|
currentJob.error?.errorCode,
|
|
@@ -846,7 +860,16 @@ export class SASViyaApiClient {
|
|
|
846
860
|
log
|
|
847
861
|
)
|
|
848
862
|
}
|
|
849
|
-
|
|
863
|
+
|
|
864
|
+
const executionResult: JobExecutionResult = {
|
|
865
|
+
result: jobResult?.result,
|
|
866
|
+
log
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
const { error } = currentJob
|
|
870
|
+
if (error) executionResult.error = error
|
|
871
|
+
|
|
872
|
+
return executionResult
|
|
850
873
|
}
|
|
851
874
|
|
|
852
875
|
private async populateFolderMap(folderPath: string, accessToken?: string) {
|
package/src/SASjs.ts
CHANGED
|
@@ -851,7 +851,7 @@ export default class SASjs {
|
|
|
851
851
|
* @param authConfig - a valid client, secret, refresh and access tokens that are authorised to execute compute jobs.
|
|
852
852
|
* The access token is not required when the user is authenticated via the browser.
|
|
853
853
|
* @param waitForResult - a boolean that indicates whether the function needs to wait for execution to complete.
|
|
854
|
-
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: {
|
|
854
|
+
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }. More information available at src/api/viya/pollJobState.ts.
|
|
855
855
|
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
|
|
856
856
|
* @param variables - an object that represents macro variables.
|
|
857
857
|
*/
|
|
@@ -12,7 +12,7 @@ import { RequestClient } from '../../request/RequestClient'
|
|
|
12
12
|
import { SessionManager } from '../../SessionManager'
|
|
13
13
|
import { isRelativePath, fetchLogByChunks } from '../../utils'
|
|
14
14
|
import { formatDataForRequest } from '../../utils/formatDataForRequest'
|
|
15
|
-
import { pollJobState } from './pollJobState'
|
|
15
|
+
import { pollJobState, JobState } from './pollJobState'
|
|
16
16
|
import { uploadTables } from './uploadTables'
|
|
17
17
|
|
|
18
18
|
/**
|
|
@@ -25,7 +25,7 @@ import { uploadTables } from './uploadTables'
|
|
|
25
25
|
* @param debug - when set to true, the log will be returned.
|
|
26
26
|
* @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code).
|
|
27
27
|
* @param waitForResult - when set to true, function will return the session
|
|
28
|
-
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: {
|
|
28
|
+
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }. More information available at src/api/viya/pollJobState.ts.
|
|
29
29
|
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
|
|
30
30
|
* @param variables - an object that represents macro variables.
|
|
31
31
|
*/
|
|
@@ -228,7 +228,7 @@ export async function executeScript(
|
|
|
228
228
|
)
|
|
229
229
|
}
|
|
230
230
|
|
|
231
|
-
if (jobStatus ===
|
|
231
|
+
if (jobStatus === JobState.Failed || jobStatus === JobState.Error) {
|
|
232
232
|
throw new ComputeJobExecutionError(currentJob, log)
|
|
233
233
|
}
|
|
234
234
|
|
|
@@ -1,29 +1,88 @@
|
|
|
1
1
|
import { AuthConfig } from '@sasjs/utils/types'
|
|
2
|
-
import { Job, PollOptions } from '../..'
|
|
2
|
+
import { Job, PollOptions, PollStrategy } from '../..'
|
|
3
3
|
import { getTokens } from '../../auth/getTokens'
|
|
4
4
|
import { RequestClient } from '../../request/RequestClient'
|
|
5
5
|
import { JobStatePollError } from '../../types/errors'
|
|
6
6
|
import { Link, WriteStream } from '../../types'
|
|
7
7
|
import { delay, isNode } from '../../utils'
|
|
8
8
|
|
|
9
|
+
export enum JobState {
|
|
10
|
+
Completed = 'completed',
|
|
11
|
+
Running = 'running',
|
|
12
|
+
Pending = 'pending',
|
|
13
|
+
Unavailable = 'unavailable',
|
|
14
|
+
NoState = '',
|
|
15
|
+
Failed = 'failed',
|
|
16
|
+
Error = 'error'
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Polls job status using default or provided poll options.
|
|
21
|
+
* @param requestClient - the pre-configured HTTP request client.
|
|
22
|
+
* @param postedJob - the relative or absolute path to the job.
|
|
23
|
+
* @param debug - sets the _debug flag in the job arguments.
|
|
24
|
+
* @param authConfig - an access token, refresh token, client and secret for an authorized user.
|
|
25
|
+
* @param pollOptions - an object containing maxPollCount, pollInterval, streamLog and logFolderPath. It will override the first default poll options in poll strategy if provided.
|
|
26
|
+
* Example pollOptions:
|
|
27
|
+
* {
|
|
28
|
+
* maxPollCount: 200,
|
|
29
|
+
* pollInterval: 300,
|
|
30
|
+
* streamLog: true, // optional, equals to false by default.
|
|
31
|
+
* pollStrategy?: // optional array of poll options that should be applied after 'maxPollCount' of the provided poll options is reached. If not provided the default (see example below) poll strategy will be used.
|
|
32
|
+
* }
|
|
33
|
+
* Example pollStrategy (values used from default poll strategy):
|
|
34
|
+
* [
|
|
35
|
+
* { maxPollCount: 200, pollInterval: 300 }, // approximately ~2 mins (including time to get response (~300ms))
|
|
36
|
+
* { maxPollCount: 300, pollInterval: 3000 }, // approximately ~5.5 mins (including time to get response (~300ms))
|
|
37
|
+
* { maxPollCount: 500, pollInterval: 30000 }, // approximately ~50.5 mins (including time to get response (~300ms))
|
|
38
|
+
* { maxPollCount: 3400, pollInterval: 60000 } // approximately ~3015 mins (~125 hours) (including time to get response (~300ms))
|
|
39
|
+
* ]
|
|
40
|
+
* @returns - a promise which resolves with a job state
|
|
41
|
+
*/
|
|
9
42
|
export async function pollJobState(
|
|
10
43
|
requestClient: RequestClient,
|
|
11
44
|
postedJob: Job,
|
|
12
45
|
debug: boolean,
|
|
13
46
|
authConfig?: AuthConfig,
|
|
14
47
|
pollOptions?: PollOptions
|
|
15
|
-
) {
|
|
48
|
+
): Promise<JobState> {
|
|
16
49
|
const logger = process.logger || console
|
|
17
50
|
|
|
18
|
-
|
|
19
|
-
|
|
51
|
+
const streamLog = pollOptions?.streamLog || false
|
|
52
|
+
|
|
53
|
+
const defaultPollStrategy: PollStrategy = [
|
|
54
|
+
{ maxPollCount: 200, pollInterval: 300 },
|
|
55
|
+
{ maxPollCount: 300, pollInterval: 3000 },
|
|
56
|
+
{ maxPollCount: 500, pollInterval: 30000 },
|
|
57
|
+
{ maxPollCount: 3400, pollInterval: 60000 }
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
let pollStrategy: PollStrategy
|
|
61
|
+
|
|
62
|
+
if (pollOptions !== undefined) {
|
|
63
|
+
pollStrategy = [pollOptions]
|
|
64
|
+
|
|
65
|
+
let { pollStrategy: providedPollStrategy } = pollOptions
|
|
66
|
+
|
|
67
|
+
if (providedPollStrategy !== undefined) {
|
|
68
|
+
validatePollStrategies(providedPollStrategy)
|
|
69
|
+
|
|
70
|
+
// INFO: sort by 'maxPollCount'
|
|
71
|
+
providedPollStrategy = providedPollStrategy.sort(
|
|
72
|
+
(strategyA: PollOptions, strategyB: PollOptions) =>
|
|
73
|
+
strategyA.maxPollCount - strategyB.maxPollCount
|
|
74
|
+
)
|
|
20
75
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
76
|
+
pollStrategy = [...pollStrategy, ...providedPollStrategy]
|
|
77
|
+
} else {
|
|
78
|
+
pollStrategy = [...pollStrategy, ...defaultPollStrategy]
|
|
79
|
+
}
|
|
80
|
+
} else {
|
|
81
|
+
pollStrategy = defaultPollStrategy
|
|
25
82
|
}
|
|
26
83
|
|
|
84
|
+
let defaultPollOptions: PollOptions = pollStrategy.splice(0, 1)[0]
|
|
85
|
+
|
|
27
86
|
pollOptions = { ...defaultPollOptions, ...(pollOptions || {}) }
|
|
28
87
|
|
|
29
88
|
const stateLink = postedJob.links.find((l: any) => l.rel === 'state')
|
|
@@ -31,10 +90,10 @@ export async function pollJobState(
|
|
|
31
90
|
throw new Error(`Job state link was not found.`)
|
|
32
91
|
}
|
|
33
92
|
|
|
34
|
-
let currentState = await getJobState(
|
|
93
|
+
let currentState: JobState = await getJobState(
|
|
35
94
|
requestClient,
|
|
36
95
|
postedJob,
|
|
37
|
-
|
|
96
|
+
JobState.NoState,
|
|
38
97
|
debug,
|
|
39
98
|
authConfig
|
|
40
99
|
).catch((err) => {
|
|
@@ -42,74 +101,72 @@ export async function pollJobState(
|
|
|
42
101
|
`Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`,
|
|
43
102
|
err
|
|
44
103
|
)
|
|
45
|
-
|
|
104
|
+
|
|
105
|
+
return JobState.Unavailable
|
|
46
106
|
})
|
|
47
107
|
|
|
48
108
|
let pollCount = 0
|
|
49
109
|
|
|
50
|
-
if (currentState ===
|
|
110
|
+
if (currentState === JobState.Completed) {
|
|
51
111
|
return Promise.resolve(currentState)
|
|
52
112
|
}
|
|
53
113
|
|
|
54
114
|
let logFileStream
|
|
55
|
-
if (
|
|
115
|
+
if (streamLog && isNode()) {
|
|
56
116
|
const { getFileStream } = require('./getFileStream')
|
|
117
|
+
|
|
57
118
|
logFileStream = await getFileStream(postedJob, pollOptions.logFolderPath)
|
|
58
119
|
}
|
|
59
120
|
|
|
60
|
-
// Poll up to the first 100 times with the specified poll interval
|
|
61
121
|
let result = await doPoll(
|
|
62
122
|
requestClient,
|
|
63
123
|
postedJob,
|
|
64
124
|
currentState,
|
|
65
125
|
debug,
|
|
66
126
|
pollCount,
|
|
127
|
+
pollOptions,
|
|
67
128
|
authConfig,
|
|
68
|
-
|
|
69
|
-
...pollOptions,
|
|
70
|
-
maxPollCount:
|
|
71
|
-
pollOptions.maxPollCount <= 100 ? pollOptions.maxPollCount : 100
|
|
72
|
-
},
|
|
129
|
+
streamLog,
|
|
73
130
|
logFileStream
|
|
74
131
|
)
|
|
75
132
|
|
|
76
133
|
currentState = result.state
|
|
77
134
|
pollCount = result.pollCount
|
|
78
135
|
|
|
79
|
-
if (
|
|
136
|
+
if (
|
|
137
|
+
!needsRetry(currentState) ||
|
|
138
|
+
(pollCount >= pollOptions.maxPollCount && !pollStrategy.length)
|
|
139
|
+
) {
|
|
80
140
|
return currentState
|
|
81
141
|
}
|
|
82
142
|
|
|
83
|
-
// If we get to this point, this is a long-running job that needs longer polling.
|
|
84
|
-
// We will resume polling with a bigger interval
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
pollInterval: 60000,
|
|
88
|
-
streamLog: false
|
|
89
|
-
}
|
|
90
|
-
if (pollOptions) {
|
|
91
|
-
longJobPollOptions.streamLog = pollOptions.streamLog
|
|
92
|
-
longJobPollOptions.logFolderPath = pollOptions.logFolderPath
|
|
93
|
-
}
|
|
143
|
+
// INFO: If we get to this point, this is a long-running job that needs longer polling.
|
|
144
|
+
// We will resume polling with a bigger interval according to the next polling strategy
|
|
145
|
+
while (pollStrategy.length && needsRetry(currentState)) {
|
|
146
|
+
defaultPollOptions = pollStrategy.splice(0, 1)[0]
|
|
94
147
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
currentState,
|
|
99
|
-
debug,
|
|
100
|
-
pollCount,
|
|
101
|
-
authConfig,
|
|
102
|
-
longJobPollOptions,
|
|
103
|
-
logFileStream
|
|
104
|
-
)
|
|
148
|
+
if (pollOptions) {
|
|
149
|
+
defaultPollOptions.logFolderPath = pollOptions.logFolderPath
|
|
150
|
+
}
|
|
105
151
|
|
|
106
|
-
|
|
107
|
-
|
|
152
|
+
result = await doPoll(
|
|
153
|
+
requestClient,
|
|
154
|
+
postedJob,
|
|
155
|
+
currentState,
|
|
156
|
+
debug,
|
|
157
|
+
pollCount,
|
|
158
|
+
defaultPollOptions,
|
|
159
|
+
authConfig,
|
|
160
|
+
streamLog,
|
|
161
|
+
logFileStream
|
|
162
|
+
)
|
|
108
163
|
|
|
109
|
-
|
|
110
|
-
|
|
164
|
+
currentState = result.state
|
|
165
|
+
pollCount = result.pollCount
|
|
111
166
|
}
|
|
112
167
|
|
|
168
|
+
if (logFileStream) logFileStream.end()
|
|
169
|
+
|
|
113
170
|
return currentState
|
|
114
171
|
}
|
|
115
172
|
|
|
@@ -119,17 +176,13 @@ const getJobState = async (
|
|
|
119
176
|
currentState: string,
|
|
120
177
|
debug: boolean,
|
|
121
178
|
authConfig?: AuthConfig
|
|
122
|
-
) => {
|
|
123
|
-
const stateLink = job.links.find((l: any) => l.rel === 'state')
|
|
124
|
-
if (!stateLink) {
|
|
125
|
-
throw new Error(`Job state link was not found.`)
|
|
126
|
-
}
|
|
179
|
+
): Promise<JobState> => {
|
|
180
|
+
const stateLink = job.links.find((l: any) => l.rel === 'state')!
|
|
127
181
|
|
|
128
182
|
if (needsRetry(currentState)) {
|
|
129
183
|
let tokens
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
}
|
|
184
|
+
|
|
185
|
+
if (authConfig) tokens = await getTokens(requestClient, authConfig)
|
|
133
186
|
|
|
134
187
|
const { result: jobState } = await requestClient
|
|
135
188
|
.get<string>(
|
|
@@ -143,48 +196,38 @@ const getJobState = async (
|
|
|
143
196
|
throw new JobStatePollError(job.id, err)
|
|
144
197
|
})
|
|
145
198
|
|
|
146
|
-
return jobState.trim()
|
|
199
|
+
return jobState.trim() as JobState
|
|
147
200
|
} else {
|
|
148
|
-
return currentState
|
|
201
|
+
return currentState as JobState
|
|
149
202
|
}
|
|
150
203
|
}
|
|
151
204
|
|
|
152
205
|
const needsRetry = (state: string) =>
|
|
153
|
-
state ===
|
|
154
|
-
state ===
|
|
155
|
-
state ===
|
|
156
|
-
state ===
|
|
206
|
+
state === JobState.Running ||
|
|
207
|
+
state === JobState.NoState ||
|
|
208
|
+
state === JobState.Pending ||
|
|
209
|
+
state === JobState.Unavailable
|
|
157
210
|
|
|
158
211
|
const doPoll = async (
|
|
159
212
|
requestClient: RequestClient,
|
|
160
213
|
postedJob: Job,
|
|
161
|
-
currentState:
|
|
214
|
+
currentState: JobState,
|
|
162
215
|
debug: boolean,
|
|
163
216
|
pollCount: number,
|
|
217
|
+
pollOptions: PollOptions,
|
|
164
218
|
authConfig?: AuthConfig,
|
|
165
|
-
|
|
219
|
+
streamLog?: boolean,
|
|
166
220
|
logStream?: WriteStream
|
|
167
|
-
): Promise<{ state:
|
|
168
|
-
|
|
169
|
-
|
|
221
|
+
): Promise<{ state: JobState; pollCount: number }> => {
|
|
222
|
+
const { maxPollCount, pollInterval } = pollOptions
|
|
223
|
+
const logger = process.logger || console
|
|
224
|
+
const stateLink = postedJob.links.find((l: Link) => l.rel === 'state')!
|
|
170
225
|
let maxErrorCount = 5
|
|
171
226
|
let errorCount = 0
|
|
172
227
|
let state = currentState
|
|
173
|
-
let printedState =
|
|
228
|
+
let printedState = JobState.NoState
|
|
174
229
|
let startLogLine = 0
|
|
175
230
|
|
|
176
|
-
const logger = process.logger || console
|
|
177
|
-
|
|
178
|
-
if (pollOptions) {
|
|
179
|
-
pollInterval = pollOptions.pollInterval || pollInterval
|
|
180
|
-
maxPollCount = pollOptions.maxPollCount || maxPollCount
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
const stateLink = postedJob.links.find((l: Link) => l.rel === 'state')
|
|
184
|
-
if (!stateLink) {
|
|
185
|
-
throw new Error(`Job state link was not found.`)
|
|
186
|
-
}
|
|
187
|
-
|
|
188
231
|
while (needsRetry(state) && pollCount <= maxPollCount) {
|
|
189
232
|
state = await getJobState(
|
|
190
233
|
requestClient,
|
|
@@ -194,21 +237,24 @@ const doPoll = async (
|
|
|
194
237
|
authConfig
|
|
195
238
|
).catch((err) => {
|
|
196
239
|
errorCount++
|
|
240
|
+
|
|
197
241
|
if (pollCount >= maxPollCount || errorCount >= maxErrorCount) {
|
|
198
242
|
throw err
|
|
199
243
|
}
|
|
244
|
+
|
|
200
245
|
logger.error(
|
|
201
246
|
`Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`,
|
|
202
247
|
err
|
|
203
248
|
)
|
|
204
|
-
|
|
249
|
+
|
|
250
|
+
return JobState.Unavailable
|
|
205
251
|
})
|
|
206
252
|
|
|
207
253
|
pollCount++
|
|
208
254
|
|
|
209
255
|
const jobHref = postedJob.links.find((l: Link) => l.rel === 'self')!.href
|
|
210
256
|
|
|
211
|
-
if (
|
|
257
|
+
if (streamLog) {
|
|
212
258
|
const { result: job } = await requestClient.get<Job>(
|
|
213
259
|
jobHref,
|
|
214
260
|
authConfig?.access_token
|
|
@@ -238,12 +284,45 @@ const doPoll = async (
|
|
|
238
284
|
printedState = state
|
|
239
285
|
}
|
|
240
286
|
|
|
241
|
-
if (state
|
|
287
|
+
if (state !== JobState.Unavailable && errorCount > 0) {
|
|
242
288
|
errorCount = 0
|
|
243
289
|
}
|
|
244
290
|
|
|
245
|
-
|
|
291
|
+
if (state !== JobState.Completed) {
|
|
292
|
+
await delay(pollInterval)
|
|
293
|
+
}
|
|
246
294
|
}
|
|
247
295
|
|
|
248
296
|
return { state, pollCount }
|
|
249
297
|
}
|
|
298
|
+
|
|
299
|
+
const validatePollStrategies = (strategy: PollStrategy) => {
|
|
300
|
+
const throwError = (message?: string, pollOptions?: PollOptions) => {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`Poll strategies are not valid.${message ? ` ${message}` : ''}${
|
|
303
|
+
pollOptions
|
|
304
|
+
? ` Invalid poll strategy: \n${JSON.stringify(pollOptions, null, 2)}`
|
|
305
|
+
: ''
|
|
306
|
+
}`
|
|
307
|
+
)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
strategy.forEach((pollOptions: PollOptions, i: number) => {
|
|
311
|
+
const { maxPollCount, pollInterval } = pollOptions
|
|
312
|
+
|
|
313
|
+
if (maxPollCount < 1) {
|
|
314
|
+
throwError(`'maxPollCount' has to be greater than 0.`, pollOptions)
|
|
315
|
+
} else if (i !== 0) {
|
|
316
|
+
const previousPollOptions = strategy[i - 1]
|
|
317
|
+
|
|
318
|
+
if (maxPollCount <= previousPollOptions.maxPollCount) {
|
|
319
|
+
throwError(
|
|
320
|
+
`'maxPollCount' has to be greater than 'maxPollCount' in previous poll strategy.`,
|
|
321
|
+
pollOptions
|
|
322
|
+
)
|
|
323
|
+
}
|
|
324
|
+
} else if (pollInterval < 1) {
|
|
325
|
+
throwError(`'pollInterval' has to be greater than 0.`, pollOptions)
|
|
326
|
+
}
|
|
327
|
+
})
|
|
328
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as NodeFormData from 'form-data'
|
|
2
2
|
import { convertToCSV } from '../utils/convertToCsv'
|
|
3
|
+
import { isNode } from '../utils'
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* One of the approaches SASjs takes to send tables-formatted JSON (see README)
|
|
@@ -26,12 +27,15 @@ export const generateFileUploadForm = (
|
|
|
26
27
|
)
|
|
27
28
|
}
|
|
28
29
|
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
// INFO: unfortunately it is not possible to check if formData is instance of NodeFormData or FormData because it will return true for both
|
|
31
|
+
if (isNode()) {
|
|
32
|
+
// INFO: environment is Node and formData is instance of NodeFormData
|
|
33
|
+
;(formData as NodeFormData).append(name, csv, {
|
|
31
34
|
filename: `${name}.csv`,
|
|
32
35
|
contentType: 'application/csv'
|
|
33
36
|
})
|
|
34
37
|
} else {
|
|
38
|
+
// INFO: environment is Browser and formData is instance of FormData
|
|
35
39
|
const file = new Blob([csv], {
|
|
36
40
|
type: 'application/csv'
|
|
37
41
|
})
|
|
@@ -10,8 +10,8 @@ import {
|
|
|
10
10
|
LoginRequiredError
|
|
11
11
|
} from '../types/errors'
|
|
12
12
|
import { generateFileUploadForm } from '../file/generateFileUploadForm'
|
|
13
|
-
|
|
14
13
|
import { RequestClient } from '../request/RequestClient'
|
|
14
|
+
import { getFormData } from '../utils'
|
|
15
15
|
|
|
16
16
|
import {
|
|
17
17
|
isRelativePath,
|
|
@@ -53,8 +53,7 @@ export class SasjsJobExecutor extends BaseJobExecutor {
|
|
|
53
53
|
* Use the available form data object (FormData in Browser, NodeFormData in
|
|
54
54
|
* Node)
|
|
55
55
|
*/
|
|
56
|
-
let formData =
|
|
57
|
-
typeof FormData === 'undefined' ? new NodeFormData() : new FormData()
|
|
56
|
+
let formData = getFormData()
|
|
58
57
|
|
|
59
58
|
if (data) {
|
|
60
59
|
// file upload approach
|