@8ms/helpers 2.1.4 → 2.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy_npmjs.yml +19 -14
- package/_class/BaseClass.js +43 -52
- package/_class/BaseNamespace.js +17 -20
- package/_class/index.js +2 -7
- package/adverity/getJobs.js +7 -11
- package/adverity/index.js +51 -54
- package/adverity/server.js +1 -5
- package/api/ApiResponseClass.js +84 -88
- package/api/functions.js +13 -22
- package/api/index.js +3 -19
- package/api/types.js +5 -9
- package/array/index.js +3 -11
- package/atInternet/index.js +10 -13
- package/aws/ec2/AwsEc2Namespace.js +34 -74
- package/aws/ec2/server.js +5 -9
- package/aws/ecs/AwsEcsNamespace.js +110 -150
- package/aws/ecs/server.js +5 -9
- package/aws/glue/AwsGlueNamespace.js +26 -66
- package/aws/glue/server.js +5 -9
- package/aws/isResponse200.js +3 -7
- package/aws/lambda/AwsLambdaNamespace.js +62 -102
- package/aws/lambda/server.js +6 -11
- package/aws/s3/AwsS3Namespace.js +347 -390
- package/aws/s3/payload.js +29 -32
- package/aws/s3/server.js +9 -14
- package/aws/server.js +5 -10
- package/aws/ses/AwsSesNamespace.js +36 -76
- package/aws/ses/SimpleEmail.js +267 -304
- package/aws/ses/server.js +6 -11
- package/aws/sqs/AwsSqsNamespace.js +49 -89
- package/aws/sqs/server.js +5 -9
- package/aws/ssm/AwsSsmNamespace.js +43 -79
- package/aws/ssm/server.js +5 -9
- package/axios/deleteRequest.js +8 -15
- package/axios/get.js +8 -15
- package/axios/index.js +3 -9
- package/axios/post.js +8 -15
- package/boolean/index.js +3 -10
- package/brightData/serpApi/buildGoogleSerpUrl.js +1 -5
- package/brightData/serpApi/buildGoogleTrendsUrl.js +1 -5
- package/brightData/serpApi/getAsyncRequestId.js +8 -12
- package/brightData/serpApi/getAsyncResults.js +8 -12
- package/brightData/serpApi/getRealtime.js +3 -7
- package/brightData/serpApi/server.js +6 -14
- package/brightData/server.js +2 -7
- package/brightData/webScraperIde/getBatch.js +3 -7
- package/brightData/webScraperIde/getRealtime.js +11 -15
- package/brightData/webScraperIde/server.js +2 -7
- package/cache/server.js +2 -7
- package/cache/test/cache.test.js +4 -6
- package/crud/index.js +1 -4
- package/crypto/getDecrypt.js +2 -39
- package/crypto/getEncrypt.js +2 -39
- package/crypto/getRandom.js +1 -5
- package/crypto/getSha256.js +4 -41
- package/crypto/index.js +4 -11
- package/date/calculation.js +70 -99
- package/date/financialYear.js +16 -23
- package/date/format.js +40 -61
- package/date/index.js +4 -20
- package/date/type.js +4 -7
- package/environment/index.js +15 -25
- package/eskimi/getAgeGroup.js +3 -7
- package/eskimi/getData.js +3 -7
- package/eskimi/getDevice.js +3 -7
- package/eskimi/getGender.js +3 -7
- package/eskimi/server.js +23 -26
- package/file/index.js +8 -46
- package/geo/countries.js +1 -4
- package/geo/index.js +2 -7
- package/geo/languages.js +1 -4
- package/google/bigQuery/GoogleBigQueryNamespace.js +108 -148
- package/google/bigQuery/loadData.js +1 -5
- package/google/bigQuery/server.js +8 -14
- package/google/server.js +1 -5
- package/google/sheets/GoogleSheetsNamespace.js +34 -74
- package/google/sheets/getAssociatedData.js +1 -5
- package/google/sheets/server.js +8 -14
- package/google/storage/GoogleCloudStorageNamespace.js +62 -102
- package/google/storage/server.js +6 -10
- package/googleAds/GoogleAdsNamespace.js +28 -68
- package/googleAds/keywordPlanner/server.js +32 -35
- package/googleAds/server.js +6 -10
- package/googlePageSpeed/GooglePageSpeedNamespace.js +21 -28
- package/googlePageSpeed/server.js +8 -12
- package/googleSearchIncidents/server.js +28 -32
- package/greenDomain/server.js +17 -21
- package/inngest/server.js +1 -4
- package/json/getJsonNewline.js +3 -7
- package/json/index.js +2 -7
- package/json/isJson.js +1 -5
- package/littleWarden/LittleWardenNamespace.js +24 -31
- package/littleWarden/getUrlStatus.js +3 -7
- package/littleWarden/server.js +14 -22
- package/lumar/api/buildRequest.js +14 -18
- package/lumar/api/getData.js +3 -7
- package/lumar/api/initClient.js +3 -7
- package/lumar/api/server.js +6 -12
- package/lumar/graphql/columns.js +1 -4
- package/lumar/graphql/getData.js +3 -7
- package/lumar/graphql/initClient.js +3 -7
- package/lumar/graphql/queries/crawl.js +1 -2
- package/lumar/graphql/queries/getCrawls.js +1 -3
- package/lumar/graphql/queries/getReportDifferences.js +3 -5
- package/lumar/graphql/queries/getRows.js +1 -3
- package/lumar/graphql/queries/getTotals.js +1 -3
- package/lumar/graphql/queries/row.js +1 -2
- package/lumar/graphql/reportTemplates.js +1 -4
- package/lumar/graphql/server.js +7 -15
- package/myTarget/server.js +3 -7
- package/nextAuth/index.js +2 -6
- package/nextAuth/isSessionReady.js +1 -5
- package/nextJs/client/LazyLoad.js +5 -12
- package/nextJs/client.js +1 -5
- package/nextJs/index.js +2 -7
- package/number/format.js +3 -7
- package/number/formatCurrency.js +1 -5
- package/number/getDecimal.js +3 -7
- package/number/getNumber.js +3 -7
- package/number/getPercentIncrease.js +4 -8
- package/number/getSafeDivide.js +6 -10
- package/number/index.js +6 -15
- package/object/index.js +1 -5
- package/object/replaceKeys.js +3 -7
- package/onePassword/OnePasswordNamespace.js +36 -76
- package/onePassword/server.js +3 -7
- package/openAi/OpenAiNamespace.js +32 -39
- package/openAi/server.js +6 -10
- package/package.json +136 -120
- package/prisma/PrismaNamespace.d.ts +3 -3
- package/prisma/PrismaNamespace.js +55 -81
- package/prisma/getDecimal.js +5 -12
- package/prisma/server.d.ts +13 -0
- package/prisma/server.js +6 -9
- package/snapchat/SnapchatNamespace.js +20 -27
- package/snapchat/server.js +6 -10
- package/sorting/byNumberAscending.js +1 -5
- package/sorting/byNumberDescending.js +1 -5
- package/sorting/byStringAscending.js +1 -5
- package/sorting/byStringDescending.js +1 -5
- package/sorting/index.js +4 -11
- package/stream/server.js +1 -5
- package/stream/sort.js +1 -5
- package/string/getCapitalised.js +3 -7
- package/string/getClean.js +17 -21
- package/string/getFolder.js +3 -10
- package/string/getProperCase.js +4 -8
- package/string/getString.js +8 -15
- package/string/getStringFromStream.js +1 -5
- package/string/getUnescaped.js +3 -10
- package/string/getWithoutAccents.js +1 -5
- package/string/getWithoutHtmlTags.js +1 -5
- package/string/getWithoutPunctuation.js +1 -5
- package/string/getWithoutUnderscores.js +1 -5
- package/string/getWithoutWhitespaces.js +1 -5
- package/string/index.js +12 -27
- package/string/reservedWords.js +5 -9
- package/swr/index.js +9 -18
- package/upTimeRobot/UpTimeRobotNamespace.js +32 -42
- package/upTimeRobot/server.js +11 -15
- package/url/index.js +12 -23
- package/url/server.js +1 -5
- package/url/writeUrlContents.js +6 -46
- package/util/defaultTo.js +5 -12
- package/util/getBrotliCompressed.js +3 -10
- package/util/getBrotliDecompressed.js +3 -10
- package/util/getClean.js +5 -9
- package/util/getError.js +1 -5
- package/util/getGzipCompressed.js +3 -10
- package/util/getGzipDecompressed.js +3 -10
- package/util/getWithoutHash.js +1 -5
- package/util/getWithoutParameter.js +1 -5
- package/util/index.js +7 -17
- package/util/isUndefined.js +3 -7
- package/util/promiseChunks.js +6 -10
- package/util/server.js +4 -11
- package/util/sleep.js +1 -5
- package/webWorker/index.js +1 -4
- package/xml/getXml.js +3 -7
- package/.yarn/install-state.gz +0 -0
- package/.yarn/releases/yarn-4.9.4.cjs +0 -942
- package/.yarnrc.yml +0 -3
- package/aws/athenaExpress/AwsAthenaNamespace.d.ts +0 -8
- package/aws/athenaExpress/AwsAthenaNamespace.js +0 -82
- package/aws/athenaExpress/server.d.ts +0 -10
- package/aws/athenaExpress/server.js +0 -40
package/aws/s3/AwsS3Namespace.js
CHANGED
|
@@ -1,407 +1,364 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
1
|
+
import { BaseNamespace } from "../../_class";
|
|
2
|
+
import { isResponse200 } from "../server";
|
|
3
|
+
import { getFolder, getStringFromStream } from "../../string";
|
|
4
|
+
import { getBrotliCompressed, getBrotliDecompressed, getGzipCompressed, getGzipDecompressed } from "../../util/server";
|
|
5
|
+
import { readFileDefault } from "./server";
|
|
6
|
+
import cloneDeep from "lodash/cloneDeep";
|
|
7
|
+
import { getToday, getYmdHisString } from "../../date";
|
|
8
|
+
import get from "axios";
|
|
9
|
+
export class AwsS3Namespace extends BaseNamespace {
|
|
10
|
+
ensureInit = async () => {
|
|
11
|
+
if (!this.client) {
|
|
12
|
+
try {
|
|
13
|
+
const { S3Client } = await import("@aws-sdk/client-s3");
|
|
14
|
+
this.client = new S3Client(this.config);
|
|
15
|
+
}
|
|
16
|
+
catch (e) {
|
|
17
|
+
throw new Error("AWS S3 Client not installed");
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
copy = async (fromBucket, fromKey, toBucket, toKey) => {
|
|
22
|
+
await this.ensureInit();
|
|
23
|
+
const { CopyObjectCommand } = await import("@aws-sdk/client-s3");
|
|
24
|
+
await this.client.send(new CopyObjectCommand({
|
|
25
|
+
Bucket: toBucket,
|
|
26
|
+
CopySource: encodeURIComponent(`${fromBucket}/${fromKey}`),
|
|
27
|
+
Key: toKey,
|
|
28
|
+
}));
|
|
29
|
+
};
|
|
30
|
+
deleteFile = async (bucket, key) => {
|
|
31
|
+
await this.ensureInit();
|
|
32
|
+
const { DeleteObjectCommand } = await import("@aws-sdk/client-s3");
|
|
33
|
+
await this.client.send(new DeleteObjectCommand({
|
|
34
|
+
Bucket: bucket,
|
|
35
|
+
Key: key,
|
|
36
|
+
}));
|
|
26
37
|
};
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
38
|
+
deleteFiles = async (bucket, keys) => {
|
|
39
|
+
await this.ensureInit();
|
|
40
|
+
const { DeleteObjectsCommand } = await import("@aws-sdk/client-s3");
|
|
41
|
+
await this.client.send(new DeleteObjectsCommand({
|
|
42
|
+
Bucket: bucket,
|
|
43
|
+
Delete: {
|
|
44
|
+
Objects: keys.map(key => ({
|
|
45
|
+
Key: key,
|
|
46
|
+
})),
|
|
47
|
+
},
|
|
48
|
+
}));
|
|
49
|
+
};
|
|
50
|
+
deleteFolder = async (bucket, folder) => {
|
|
51
|
+
await this.ensureInit();
|
|
52
|
+
const { ListObjectsV2Command, DeleteObjectsCommand } = await import("@aws-sdk/client-s3");
|
|
53
|
+
const cleanFolder = getFolder(folder);
|
|
54
|
+
// Get a list of all files to be deleted
|
|
55
|
+
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
56
|
+
Bucket: bucket,
|
|
57
|
+
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
58
|
+
}));
|
|
59
|
+
// No files to be deleted
|
|
60
|
+
if (apiResponse.Contents.length === 0) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
// Map each of the files
|
|
64
|
+
const keys = apiResponse.Contents.map(file => ({ Key: file.Key }));
|
|
65
|
+
// Delete the files
|
|
66
|
+
await this.client.send(new DeleteObjectsCommand({
|
|
67
|
+
Bucket: bucket,
|
|
68
|
+
Delete: {
|
|
69
|
+
Objects: keys,
|
|
70
|
+
},
|
|
71
|
+
}));
|
|
72
|
+
// If we didn't get all files, run it again
|
|
73
|
+
if (apiResponse.IsTruncated) {
|
|
74
|
+
await this.deleteFolder(bucket, folder);
|
|
75
|
+
}
|
|
33
76
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
catch (e) {
|
|
58
|
-
throw new Error("AWS S3 Client not installed");
|
|
59
|
-
}
|
|
77
|
+
/**
|
|
78
|
+
* List the files in the given bucket and option folder
|
|
79
|
+
*/
|
|
80
|
+
listFiles = async (bucket, folder = "") => {
|
|
81
|
+
await this.ensureInit();
|
|
82
|
+
const { ListObjectsV2Command } = await import("@aws-sdk/client-s3");
|
|
83
|
+
const cleanFolder = getFolder(folder);
|
|
84
|
+
let response = [];
|
|
85
|
+
let valid = [];
|
|
86
|
+
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
87
|
+
Bucket: bucket,
|
|
88
|
+
Delimiter: "/",
|
|
89
|
+
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
90
|
+
}));
|
|
91
|
+
// Ensure the data exists
|
|
92
|
+
if (undefined !== apiResponse["Contents"]) {
|
|
93
|
+
// Return an array of files
|
|
94
|
+
valid = apiResponse["Contents"].filter(content => {
|
|
95
|
+
const keyExploded = content["Key"].split("/");
|
|
96
|
+
return keyExploded[keyExploded.length - 1].indexOf(".") > -1;
|
|
97
|
+
});
|
|
98
|
+
if (valid.length) {
|
|
99
|
+
response = valid.map(entry => entry["Key"]);
|
|
60
100
|
}
|
|
101
|
+
}
|
|
102
|
+
return response;
|
|
103
|
+
};
|
|
104
|
+
/**
|
|
105
|
+
* Fetch all the files within a given directory recursively
|
|
106
|
+
*/
|
|
107
|
+
filesIteratively = async (bucket, folder) => {
|
|
108
|
+
await this.ensureInit();
|
|
109
|
+
const response = await this.iterateFolders(bucket, folder);
|
|
110
|
+
return response;
|
|
111
|
+
};
|
|
112
|
+
/**
|
|
113
|
+
* Sub function
|
|
114
|
+
*/
|
|
115
|
+
iterateFolders = async (bucket, folder) => {
|
|
116
|
+
let response = [];
|
|
117
|
+
const folders = await this.listFolders(bucket, folder);
|
|
118
|
+
for (let i = 0; i < folders.length; i++) {
|
|
119
|
+
const subFolderFiles = await this.listFolders(bucket, `${folder}/${folders[i]}`);
|
|
120
|
+
const files = await this.listFiles(bucket, `${folder}/${folders[i]}`);
|
|
121
|
+
response = [
|
|
122
|
+
...response,
|
|
123
|
+
...files,
|
|
124
|
+
...subFolderFiles,
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
return response;
|
|
128
|
+
};
|
|
129
|
+
/**
|
|
130
|
+
* List the files in the given bucket and option folder.
|
|
131
|
+
*/
|
|
132
|
+
listFolders = async (bucket, folder) => {
|
|
133
|
+
await this.ensureInit();
|
|
134
|
+
const { ListObjectsV2Command } = await import("@aws-sdk/client-s3");
|
|
135
|
+
const cleanFolder = getFolder(folder);
|
|
136
|
+
let response = [];
|
|
137
|
+
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
138
|
+
Bucket: bucket,
|
|
139
|
+
Delimiter: "/",
|
|
140
|
+
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
141
|
+
}));
|
|
142
|
+
// Ensure the data exists
|
|
143
|
+
if (undefined !== apiResponse["CommonPrefixes"]) {
|
|
144
|
+
// Return an array of folders removing the trailing folder
|
|
145
|
+
response = apiResponse["CommonPrefixes"].map(prefix => {
|
|
146
|
+
// Remove the trailing slash then split by slash
|
|
147
|
+
const fileFolders = prefix.Prefix.slice(0, -1)
|
|
148
|
+
.split("/");
|
|
149
|
+
// Return the last folder without a trailing slash
|
|
150
|
+
return fileFolders[fileFolders.length - 1];
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
return response;
|
|
154
|
+
};
|
|
155
|
+
move = async (fromBucket, fromKey, toBucket, toKey) => {
|
|
156
|
+
await this.ensureInit();
|
|
157
|
+
const { CopyObjectCommand, DeleteObjectCommand } = await import("@aws-sdk/client-s3");
|
|
158
|
+
await this.client.send(new CopyObjectCommand({
|
|
159
|
+
Bucket: toBucket,
|
|
160
|
+
CopySource: encodeURIComponent(`${fromBucket}/${fromKey}`),
|
|
161
|
+
Key: toKey,
|
|
162
|
+
}));
|
|
163
|
+
await this.client.send(new DeleteObjectCommand({
|
|
164
|
+
Bucket: fromBucket,
|
|
165
|
+
Key: fromKey,
|
|
166
|
+
}));
|
|
167
|
+
};
|
|
168
|
+
readBrotli = async (bucket, key, brotliOptions = {}, awsS3Options = {}) => {
|
|
169
|
+
await this.ensureInit();
|
|
170
|
+
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
171
|
+
// Cache exists and is still within our caching timeframe
|
|
172
|
+
if (undefined !== apiResponse.body) {
|
|
173
|
+
apiResponse.body = await getBrotliDecompressed(apiResponse.body, brotliOptions);
|
|
174
|
+
}
|
|
175
|
+
return apiResponse;
|
|
176
|
+
};
|
|
177
|
+
readJsonBrotli = async (bucket, key, brotliOptions = {}, awsS3Options = {}) => {
|
|
178
|
+
await this.ensureInit();
|
|
179
|
+
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
180
|
+
if (undefined !== apiResponse.body) {
|
|
181
|
+
apiResponse.body = await getBrotliDecompressed(apiResponse.body, brotliOptions);
|
|
182
|
+
apiResponse.body = JSON.parse(apiResponse.body) || null;
|
|
183
|
+
}
|
|
184
|
+
return apiResponse;
|
|
185
|
+
};
|
|
186
|
+
readBuffer = async (bucket, key) => {
|
|
187
|
+
await this.ensureInit();
|
|
188
|
+
const { GetObjectCommand } = await import("@aws-sdk/client-s3");
|
|
189
|
+
let response = {
|
|
190
|
+
...readFileDefault,
|
|
191
|
+
buffer: undefined,
|
|
61
192
|
};
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
193
|
+
const apiResponse = await this.client.send(new GetObjectCommand({
|
|
194
|
+
Bucket: bucket,
|
|
195
|
+
Key: key,
|
|
196
|
+
}));
|
|
197
|
+
if (isResponse200(apiResponse)) {
|
|
198
|
+
const chunks = [];
|
|
199
|
+
// Store the response
|
|
200
|
+
response.buffer = await new Promise((resolve, reject) => {
|
|
201
|
+
// @ts-ignore
|
|
202
|
+
apiResponse.Body.on("data", (chunk) => chunks.push(chunk));
|
|
203
|
+
// @ts-ignore
|
|
204
|
+
apiResponse.Body.on("error", (err) => reject(err));
|
|
205
|
+
// @ts-ignore
|
|
206
|
+
apiResponse.Body.on("end", () => resolve(Buffer.concat(chunks)));
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
throw new Error("Error reading file.");
|
|
211
|
+
}
|
|
212
|
+
return response;
|
|
213
|
+
};
|
|
214
|
+
fileExists = async (bucket, key) => {
|
|
215
|
+
await this.ensureInit();
|
|
216
|
+
let response = false;
|
|
217
|
+
try {
|
|
218
|
+
const { HeadObjectCommand } = await import("@aws-sdk/client-s3");
|
|
219
|
+
// Fetch from S3
|
|
220
|
+
const apiResponse = await this.client.send(new HeadObjectCommand({
|
|
75
221
|
Bucket: bucket,
|
|
76
222
|
Key: key,
|
|
77
223
|
}));
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const {
|
|
94
|
-
|
|
95
|
-
// Get a list of all files to be deleted
|
|
96
|
-
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
97
|
-
Bucket: bucket,
|
|
98
|
-
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
99
|
-
}));
|
|
100
|
-
// No files to be deleted
|
|
101
|
-
if (apiResponse.Contents.length === 0) {
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
// Map each of the files
|
|
105
|
-
const keys = apiResponse.Contents.map(file => ({ Key: file.Key }));
|
|
106
|
-
// Delete the files
|
|
107
|
-
await this.client.send(new DeleteObjectsCommand({
|
|
108
|
-
Bucket: bucket,
|
|
109
|
-
Delete: {
|
|
110
|
-
Objects: keys,
|
|
111
|
-
},
|
|
112
|
-
}));
|
|
113
|
-
// If we didn't get all files, run it again
|
|
114
|
-
if (apiResponse.IsTruncated) {
|
|
115
|
-
await this.deleteFolder(bucket, folder);
|
|
116
|
-
}
|
|
117
|
-
};
|
|
118
|
-
/**
|
|
119
|
-
* List the files in the given bucket and option folder
|
|
120
|
-
*/
|
|
121
|
-
this.listFiles = async (bucket, folder = "") => {
|
|
122
|
-
await this.ensureInit();
|
|
123
|
-
const { ListObjectsV2Command } = await Promise.resolve().then(() => __importStar(require("@aws-sdk/client-s3")));
|
|
124
|
-
const cleanFolder = (0, string_1.getFolder)(folder);
|
|
125
|
-
let response = [];
|
|
126
|
-
let valid = [];
|
|
127
|
-
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
128
|
-
Bucket: bucket,
|
|
129
|
-
Delimiter: "/",
|
|
130
|
-
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
131
|
-
}));
|
|
132
|
-
// Ensure the data exists
|
|
133
|
-
if (undefined !== apiResponse["Contents"]) {
|
|
134
|
-
// Return an array of files
|
|
135
|
-
valid = apiResponse["Contents"].filter(content => {
|
|
136
|
-
const keyExploded = content["Key"].split("/");
|
|
137
|
-
return keyExploded[keyExploded.length - 1].indexOf(".") > -1;
|
|
138
|
-
});
|
|
139
|
-
if (valid.length) {
|
|
140
|
-
response = valid.map(entry => entry["Key"]);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
return response;
|
|
144
|
-
};
|
|
145
|
-
/**
|
|
146
|
-
* Fetch all the files within a given directory recursively
|
|
147
|
-
*/
|
|
148
|
-
this.filesIteratively = async (bucket, folder) => {
|
|
149
|
-
await this.ensureInit();
|
|
150
|
-
const response = await this.iterateFolders(bucket, folder);
|
|
151
|
-
return response;
|
|
152
|
-
};
|
|
153
|
-
/**
|
|
154
|
-
* Sub function
|
|
155
|
-
*/
|
|
156
|
-
this.iterateFolders = async (bucket, folder) => {
|
|
157
|
-
let response = [];
|
|
158
|
-
const folders = await this.listFolders(bucket, folder);
|
|
159
|
-
for (let i = 0; i < folders.length; i++) {
|
|
160
|
-
const subFolderFiles = await this.listFolders(bucket, `${folder}/${folders[i]}`);
|
|
161
|
-
const files = await this.listFiles(bucket, `${folder}/${folders[i]}`);
|
|
162
|
-
response = [
|
|
163
|
-
...response,
|
|
164
|
-
...files,
|
|
165
|
-
...subFolderFiles,
|
|
166
|
-
];
|
|
167
|
-
}
|
|
168
|
-
return response;
|
|
169
|
-
};
|
|
170
|
-
/**
|
|
171
|
-
* List the files in the given bucket and option folder.
|
|
172
|
-
*/
|
|
173
|
-
this.listFolders = async (bucket, folder) => {
|
|
174
|
-
await this.ensureInit();
|
|
175
|
-
const { ListObjectsV2Command } = await Promise.resolve().then(() => __importStar(require("@aws-sdk/client-s3")));
|
|
176
|
-
const cleanFolder = (0, string_1.getFolder)(folder);
|
|
177
|
-
let response = [];
|
|
178
|
-
const apiResponse = await this.client.send(new ListObjectsV2Command({
|
|
179
|
-
Bucket: bucket,
|
|
180
|
-
Delimiter: "/",
|
|
181
|
-
Prefix: ("" === cleanFolder ? "" : cleanFolder + "/"),
|
|
182
|
-
}));
|
|
183
|
-
// Ensure the data exists
|
|
184
|
-
if (undefined !== apiResponse["CommonPrefixes"]) {
|
|
185
|
-
// Return an array of folders removing the trailing folder
|
|
186
|
-
response = apiResponse["CommonPrefixes"].map(prefix => {
|
|
187
|
-
// Remove the trailing slash then split by slash
|
|
188
|
-
const fileFolders = prefix.Prefix.slice(0, -1)
|
|
189
|
-
.split("/");
|
|
190
|
-
// Return the last folder without a trailing slash
|
|
191
|
-
return fileFolders[fileFolders.length - 1];
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
return response;
|
|
195
|
-
};
|
|
196
|
-
this.move = async (fromBucket, fromKey, toBucket, toKey) => {
|
|
197
|
-
await this.ensureInit();
|
|
198
|
-
const { CopyObjectCommand, DeleteObjectCommand } = await Promise.resolve().then(() => __importStar(require("@aws-sdk/client-s3")));
|
|
199
|
-
await this.client.send(new CopyObjectCommand({
|
|
200
|
-
Bucket: toBucket,
|
|
201
|
-
CopySource: encodeURIComponent(`${fromBucket}/${fromKey}`),
|
|
202
|
-
Key: toKey,
|
|
203
|
-
}));
|
|
204
|
-
await this.client.send(new DeleteObjectCommand({
|
|
205
|
-
Bucket: fromBucket,
|
|
206
|
-
Key: fromKey,
|
|
207
|
-
}));
|
|
208
|
-
};
|
|
209
|
-
this.readBrotli = async (bucket, key, brotliOptions = {}, awsS3Options = {}) => {
|
|
210
|
-
await this.ensureInit();
|
|
211
|
-
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
212
|
-
// Cache exists and is still within our caching timeframe
|
|
213
|
-
if (undefined !== apiResponse.body) {
|
|
214
|
-
apiResponse.body = await (0, server_2.getBrotliDecompressed)(apiResponse.body, brotliOptions);
|
|
215
|
-
}
|
|
216
|
-
return apiResponse;
|
|
217
|
-
};
|
|
218
|
-
this.readJsonBrotli = async (bucket, key, brotliOptions = {}, awsS3Options = {}) => {
|
|
219
|
-
await this.ensureInit();
|
|
220
|
-
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
221
|
-
if (undefined !== apiResponse.body) {
|
|
222
|
-
apiResponse.body = await (0, server_2.getBrotliDecompressed)(apiResponse.body, brotliOptions);
|
|
223
|
-
apiResponse.body = JSON.parse(apiResponse.body) || null;
|
|
224
|
-
}
|
|
225
|
-
return apiResponse;
|
|
226
|
-
};
|
|
227
|
-
this.readBuffer = async (bucket, key) => {
|
|
228
|
-
await this.ensureInit();
|
|
229
|
-
const { GetObjectCommand } = await Promise.resolve().then(() => __importStar(require("@aws-sdk/client-s3")));
|
|
230
|
-
let response = {
|
|
231
|
-
...server_3.readFileDefault,
|
|
232
|
-
buffer: undefined,
|
|
233
|
-
};
|
|
224
|
+
// If the status is 200 then it does exist
|
|
225
|
+
response = isResponse200(apiResponse);
|
|
226
|
+
}
|
|
227
|
+
catch (exception) {
|
|
228
|
+
response = false;
|
|
229
|
+
}
|
|
230
|
+
return response;
|
|
231
|
+
};
|
|
232
|
+
readFile = async (bucket, key, options = {}) => {
|
|
233
|
+
await this.ensureInit();
|
|
234
|
+
let response = cloneDeep(readFileDefault);
|
|
235
|
+
const today = getToday();
|
|
236
|
+
const midnight = getToday(true);
|
|
237
|
+
try {
|
|
238
|
+
response.now = today.toUnixInteger();
|
|
239
|
+
const { GetObjectCommand } = await import("@aws-sdk/client-s3");
|
|
240
|
+
//console.log('GetObjectCommand', GetObjectCommand);
|
|
234
241
|
const apiResponse = await this.client.send(new GetObjectCommand({
|
|
235
242
|
Bucket: bucket,
|
|
236
243
|
Key: key,
|
|
244
|
+
...options,
|
|
237
245
|
}));
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
//
|
|
241
|
-
response.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
// @ts-ignore
|
|
245
|
-
apiResponse.Body.on("error", (err) => reject(err));
|
|
246
|
-
// @ts-ignore
|
|
247
|
-
apiResponse.Body.on("end", () => resolve(Buffer.concat(chunks)));
|
|
248
|
-
});
|
|
249
|
-
}
|
|
250
|
-
else {
|
|
251
|
-
throw new Error("Error reading file.");
|
|
252
|
-
}
|
|
253
|
-
return response;
|
|
254
|
-
};
|
|
255
|
-
this.fileExists = async (bucket, key) => {
|
|
256
|
-
await this.ensureInit();
|
|
257
|
-
let response = false;
|
|
258
|
-
try {
|
|
259
|
-
const { HeadObjectCommand } = await Promise.resolve().then(() => __importStar(require("@aws-sdk/client-s3")));
|
|
260
|
-
// Fetch from S3
|
|
261
|
-
const apiResponse = await this.client.send(new HeadObjectCommand({
|
|
262
|
-
Bucket: bucket,
|
|
263
|
-
Key: key,
|
|
264
|
-
}));
|
|
265
|
-
// If the status is 200 then it does exist
|
|
266
|
-
response = (0, server_1.isResponse200)(apiResponse);
|
|
267
|
-
}
|
|
268
|
-
catch (exception) {
|
|
269
|
-
response = false;
|
|
246
|
+
//console.log('apiResponse', apiResponse);
|
|
247
|
+
if (isResponse200(apiResponse)) {
|
|
248
|
+
// Convert the date to unix
|
|
249
|
+
response.modified = getYmdHisString(apiResponse.LastModified);
|
|
250
|
+
// Convert stream into a string
|
|
251
|
+
response.body = await getStringFromStream(apiResponse.Body);
|
|
270
252
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
// Convert the date to unix
|
|
290
|
-
response.modified = (0, date_1.getYmdHisString)(apiResponse.LastModified);
|
|
291
|
-
// Convert stream into a string
|
|
292
|
-
response.body = await (0, string_1.getStringFromStream)(apiResponse.Body);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
catch (error) {
|
|
296
|
-
response.body = null;
|
|
297
|
-
response.error = error;
|
|
298
|
-
}
|
|
299
|
-
return response;
|
|
300
|
-
};
|
|
301
|
-
this.readJson = async (bucket, key) => {
|
|
302
|
-
await this.ensureInit();
|
|
303
|
-
const apiResponse = await this.readFile(bucket, key);
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
response.body = null;
|
|
256
|
+
response.error = error;
|
|
257
|
+
}
|
|
258
|
+
return response;
|
|
259
|
+
};
|
|
260
|
+
readJson = async (bucket, key) => {
|
|
261
|
+
await this.ensureInit();
|
|
262
|
+
const apiResponse = await this.readFile(bucket, key);
|
|
263
|
+
apiResponse.body = JSON.parse(apiResponse.body) || null;
|
|
264
|
+
return apiResponse;
|
|
265
|
+
};
|
|
266
|
+
readJsonGzip = async (bucket, key, gzipOptions = {}, awsS3Options = {}) => {
|
|
267
|
+
await this.ensureInit();
|
|
268
|
+
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
269
|
+
if (undefined !== apiResponse.body) {
|
|
270
|
+
apiResponse.body = await getGzipDecompressed(apiResponse.body, gzipOptions);
|
|
304
271
|
apiResponse.body = JSON.parse(apiResponse.body) || null;
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
272
|
+
}
|
|
273
|
+
return apiResponse;
|
|
274
|
+
};
|
|
275
|
+
readGzip = async (bucket, key, gzipOptions = {}, awsS3Options = {}) => {
|
|
276
|
+
await this.ensureInit();
|
|
277
|
+
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
278
|
+
if (undefined !== apiResponse.body) {
|
|
279
|
+
apiResponse.body = await getGzipDecompressed(apiResponse.body, gzipOptions);
|
|
280
|
+
}
|
|
281
|
+
return apiResponse;
|
|
282
|
+
};
|
|
283
|
+
writeBrotli = async (bucket, key, data, brotliOptions = {}, awsS3Options = {}) => {
|
|
284
|
+
await this.ensureInit();
|
|
285
|
+
let finalData = data;
|
|
286
|
+
finalData = await getBrotliCompressed(finalData, brotliOptions);
|
|
287
|
+
const apiResponse = await this.writeFile(bucket, key, finalData, awsS3Options);
|
|
288
|
+
return apiResponse;
|
|
289
|
+
};
|
|
290
|
+
writeFile = async (bucket, key, data, options = {}) => {
|
|
291
|
+
await this.ensureInit();
|
|
292
|
+
const { PutObjectCommand } = await import("@aws-sdk/client-s3");
|
|
293
|
+
const apiResponse = await this.client.send(new PutObjectCommand({
|
|
294
|
+
Bucket: bucket,
|
|
295
|
+
Body: data,
|
|
296
|
+
Key: key,
|
|
297
|
+
...options,
|
|
298
|
+
}));
|
|
299
|
+
return apiResponse;
|
|
300
|
+
};
|
|
301
|
+
writeJson = async (bucket, key, data, options = {}) => {
|
|
302
|
+
await this.ensureInit();
|
|
303
|
+
const apiResponse = await this.writeFile(bucket, key, JSON.stringify(data), options);
|
|
304
|
+
return apiResponse;
|
|
305
|
+
};
|
|
306
|
+
writeJsonGzip = async (bucket, key, data, gzipOptions = {}, awsS3Options = {}) => {
|
|
307
|
+
await this.ensureInit();
|
|
308
|
+
let finalData = JSON.stringify(data);
|
|
309
|
+
finalData = await getGzipCompressed(finalData, gzipOptions);
|
|
310
|
+
const apiResponse = await this.writeFile(bucket, key, finalData, awsS3Options);
|
|
311
|
+
return apiResponse;
|
|
312
|
+
};
|
|
313
|
+
writeGzip = async (bucket, key, data, gzipOptions = {}, awsS3Options = {}) => {
|
|
314
|
+
await this.ensureInit();
|
|
315
|
+
let finalData = data;
|
|
316
|
+
finalData = await getGzipCompressed(finalData, gzipOptions);
|
|
317
|
+
const apiResponse = await this.writeFile(bucket, key, finalData, awsS3Options);
|
|
318
|
+
return apiResponse;
|
|
319
|
+
};
|
|
320
|
+
/**
|
|
321
|
+
* Get a presigned post URL so we can upload directly to S3 via frontend.
|
|
322
|
+
*/
|
|
323
|
+
presignedPost = async (props) => {
|
|
324
|
+
await this.ensureInit();
|
|
325
|
+
return new Promise(async (resolve, reject) => {
|
|
326
|
+
let response = null;
|
|
327
|
+
const formData = new FormData();
|
|
328
|
+
Object.entries({ fields: props.fields, file: props.file })
|
|
329
|
+
.forEach(([key, value]) => {
|
|
330
|
+
formData.append(key, value);
|
|
331
|
+
});
|
|
332
|
+
const upload = await fetch(props.url, {
|
|
333
|
+
method: "POST",
|
|
334
|
+
body: formData,
|
|
335
|
+
});
|
|
336
|
+
// Uploaded to S3
|
|
337
|
+
if (upload.ok) {
|
|
338
|
+
response = `https://${props.bucket}.s3.${this.config.region}.amazonaws.com/${props.key}`;
|
|
339
|
+
resolve(response);
|
|
313
340
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
const apiResponse = await this.readFile(bucket, key, awsS3Options);
|
|
319
|
-
if (undefined !== apiResponse.body) {
|
|
320
|
-
apiResponse.body = await (0, server_2.getGzipDecompressed)(apiResponse.body, gzipOptions);
|
|
341
|
+
// Failed to upload to S3
|
|
342
|
+
else {
|
|
343
|
+
response = null;
|
|
344
|
+
reject(response);
|
|
321
345
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
Key: key,
|
|
338
|
-
...options,
|
|
339
|
-
}));
|
|
340
|
-
return apiResponse;
|
|
341
|
-
};
|
|
342
|
-
this.writeJson = async (bucket, key, data, options = {}) => {
|
|
343
|
-
await this.ensureInit();
|
|
344
|
-
const apiResponse = await this.writeFile(bucket, key, JSON.stringify(data), options);
|
|
345
|
-
return apiResponse;
|
|
346
|
-
};
|
|
347
|
-
this.writeJsonGzip = async (bucket, key, data, gzipOptions = {}, awsS3Options = {}) => {
|
|
348
|
-
await this.ensureInit();
|
|
349
|
-
let finalData = JSON.stringify(data);
|
|
350
|
-
finalData = await (0, server_2.getGzipCompressed)(finalData, gzipOptions);
|
|
351
|
-
const apiResponse = await this.writeFile(bucket, key, finalData, awsS3Options);
|
|
352
|
-
return apiResponse;
|
|
353
|
-
};
|
|
354
|
-
this.writeGzip = async (bucket, key, data, gzipOptions = {}, awsS3Options = {}) => {
|
|
355
|
-
await this.ensureInit();
|
|
356
|
-
let finalData = data;
|
|
357
|
-
finalData = await (0, server_2.getGzipCompressed)(finalData, gzipOptions);
|
|
358
|
-
const apiResponse = await this.writeFile(bucket, key, finalData, awsS3Options);
|
|
359
|
-
return apiResponse;
|
|
360
|
-
};
|
|
361
|
-
/**
|
|
362
|
-
* Get a presigned post URL so we can upload directly to S3 via frontend.
|
|
363
|
-
*/
|
|
364
|
-
this.presignedPost = async (props) => {
|
|
365
|
-
await this.ensureInit();
|
|
366
|
-
return new Promise(async (resolve, reject) => {
|
|
367
|
-
let response = null;
|
|
368
|
-
const formData = new FormData();
|
|
369
|
-
Object.entries({ fields: props.fields, file: props.file })
|
|
370
|
-
.forEach(([key, value]) => {
|
|
371
|
-
formData.append(key, value);
|
|
372
|
-
});
|
|
373
|
-
const upload = await fetch(props.url, {
|
|
374
|
-
method: "POST",
|
|
375
|
-
body: formData,
|
|
376
|
-
});
|
|
377
|
-
// Uploaded to S3
|
|
378
|
-
if (upload.ok) {
|
|
379
|
-
response = `https://${props.bucket}.s3.${this.config.region}.amazonaws.com/${props.key}`;
|
|
380
|
-
resolve(response);
|
|
381
|
-
}
|
|
382
|
-
// Failed to upload to S3
|
|
383
|
-
else {
|
|
384
|
-
response = null;
|
|
385
|
-
reject(response);
|
|
386
|
-
}
|
|
387
|
-
});
|
|
388
|
-
};
|
|
389
|
-
/**
|
|
390
|
-
* Download a file and write to S3.
|
|
391
|
-
*/
|
|
392
|
-
this.urlContents = async (bucket, key, url) => {
|
|
393
|
-
await this.ensureInit();
|
|
394
|
-
return (0, axios_1.default)(url, {
|
|
395
|
-
responseType: "arraybuffer",
|
|
396
|
-
responseEncoding: "binary",
|
|
397
|
-
})
|
|
398
|
-
.then(async (response) => {
|
|
399
|
-
await this.writeFile(bucket, response.data, key, {
|
|
400
|
-
ContentType: response.headers["content-type"],
|
|
401
|
-
ContentLength: response.headers["content-length"],
|
|
402
|
-
});
|
|
346
|
+
});
|
|
347
|
+
};
|
|
348
|
+
/**
|
|
349
|
+
* Download a file and write to S3.
|
|
350
|
+
*/
|
|
351
|
+
urlContents = async (bucket, key, url) => {
|
|
352
|
+
await this.ensureInit();
|
|
353
|
+
return get(url, {
|
|
354
|
+
responseType: "arraybuffer",
|
|
355
|
+
responseEncoding: "binary",
|
|
356
|
+
})
|
|
357
|
+
.then(async (response) => {
|
|
358
|
+
await this.writeFile(bucket, response.data, key, {
|
|
359
|
+
ContentType: response.headers["content-type"],
|
|
360
|
+
ContentLength: response.headers["content-length"],
|
|
403
361
|
});
|
|
404
|
-
};
|
|
405
|
-
}
|
|
362
|
+
});
|
|
363
|
+
};
|
|
406
364
|
}
|
|
407
|
-
exports.AwsS3Namespace = AwsS3Namespace;
|