importer-storage 1.0.15 → 1.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +50 -3
- package/dist/index.global.js +50 -3
- package/dist/index.js +50 -3
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -61,6 +61,7 @@ var import_dotenv = require("dotenv");
|
|
|
61
61
|
var import_axios = __toESM(require("axios"), 1);
|
|
62
62
|
var import_jimp = require("jimp");
|
|
63
63
|
var import_https_proxy_agent = require("https-proxy-agent");
|
|
64
|
+
var import_perf_hooks = require("perf_hooks");
|
|
64
65
|
(0, import_dotenv.config)();
|
|
65
66
|
if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
|
|
66
67
|
throw new Error("'AWS_ACCESS_KEY_ID' not set");
|
|
@@ -97,14 +98,26 @@ var S3Storage = class {
|
|
|
97
98
|
const result = [];
|
|
98
99
|
const errored = [];
|
|
99
100
|
const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
|
|
101
|
+
if (options.logging) {
|
|
102
|
+
console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
|
|
103
|
+
console.time(`[SaveFiles] Total time`);
|
|
104
|
+
}
|
|
100
105
|
for (const { key, filePath } of keys) {
|
|
106
|
+
const start = import_perf_hooks.performance.now();
|
|
107
|
+
if (options.logging) {
|
|
108
|
+
console.info(`
|
|
109
|
+
[Processing Start] key=${key}, filePath=${filePath}`);
|
|
110
|
+
}
|
|
101
111
|
try {
|
|
102
112
|
const s3Key = `${options.savePath}/${key}`;
|
|
113
|
+
if (options.logging) {
|
|
114
|
+
console.info(`[S3 Key] ${s3Key}`);
|
|
115
|
+
}
|
|
103
116
|
let stream = void 0;
|
|
104
117
|
let buffer = void 0;
|
|
105
118
|
if (isHttp(filePath)) {
|
|
106
119
|
if (options.logging) {
|
|
107
|
-
console.info(`[Start
|
|
120
|
+
console.info(`[Download Start] ${filePath}`);
|
|
108
121
|
}
|
|
109
122
|
const axiosResponse = yield import_axios.default.get(filePath, {
|
|
110
123
|
responseType: "arraybuffer",
|
|
@@ -112,19 +125,33 @@ var S3Storage = class {
|
|
|
112
125
|
httpsAgent: proxyAgent
|
|
113
126
|
});
|
|
114
127
|
if (options.logging) {
|
|
115
|
-
console.info(`[Success
|
|
128
|
+
console.info(`[Download Success] ${filePath}`);
|
|
116
129
|
}
|
|
117
130
|
buffer = Buffer.from(axiosResponse.data);
|
|
118
131
|
} else {
|
|
132
|
+
if (options.logging) {
|
|
133
|
+
console.info(`[Read from FS] ${filePath}`);
|
|
134
|
+
}
|
|
119
135
|
stream = (0, import_fs.createReadStream)(filePath);
|
|
120
136
|
}
|
|
121
137
|
const tagString = options.tags.map(
|
|
122
138
|
(tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
|
|
123
139
|
).join("&");
|
|
124
140
|
if (!stream && !buffer) {
|
|
141
|
+
if (options.logging) {
|
|
142
|
+
console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
|
|
143
|
+
}
|
|
125
144
|
throw new Error("buffer or stream not set");
|
|
126
145
|
}
|
|
146
|
+
if (options.logging) {
|
|
147
|
+
console.info(`[Image Processing Start] ${key}`);
|
|
148
|
+
}
|
|
127
149
|
const image = buffer ? yield import_jimp.Jimp.fromBuffer(buffer) : yield import_jimp.Jimp.read(filePath);
|
|
150
|
+
if (options.logging) {
|
|
151
|
+
console.info(
|
|
152
|
+
`[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
|
|
153
|
+
);
|
|
154
|
+
}
|
|
128
155
|
const command = new import_client_s3.PutObjectCommand({
|
|
129
156
|
Body: buffer || stream,
|
|
130
157
|
Bucket: BUCKET_NAME,
|
|
@@ -142,16 +169,36 @@ var S3Storage = class {
|
|
|
142
169
|
filePath,
|
|
143
170
|
uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
|
|
144
171
|
});
|
|
172
|
+
if (options.logging) {
|
|
173
|
+
console.info(`[Uploading] ${key} to S3...`);
|
|
174
|
+
}
|
|
145
175
|
yield s3.send(command);
|
|
176
|
+
if (options.logging) {
|
|
177
|
+
console.info(`[Upload Success] ${key}`);
|
|
178
|
+
}
|
|
146
179
|
} catch (e) {
|
|
147
180
|
const error = e;
|
|
148
|
-
|
|
181
|
+
if (options.logging) {
|
|
182
|
+
console.error(`[Upload Error] key=${key}, error=${error.message}`);
|
|
183
|
+
}
|
|
149
184
|
errored.push({
|
|
150
185
|
key,
|
|
151
186
|
filePath,
|
|
152
187
|
message: error.message
|
|
153
188
|
});
|
|
154
189
|
}
|
|
190
|
+
const end = import_perf_hooks.performance.now();
|
|
191
|
+
if (options.logging) {
|
|
192
|
+
console.info(
|
|
193
|
+
`[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
if (options.logging) {
|
|
198
|
+
console.timeEnd(`[SaveFiles] Total time`);
|
|
199
|
+
console.info(
|
|
200
|
+
`[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
|
|
201
|
+
);
|
|
155
202
|
}
|
|
156
203
|
return { result, errored };
|
|
157
204
|
});
|
package/dist/index.global.js
CHANGED
|
@@ -76710,6 +76710,7 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
|
|
|
76710
76710
|
|
|
76711
76711
|
// src/logic/storage.ts
|
|
76712
76712
|
var import_https_proxy_agent = __toESM(require_dist2(), 1);
|
|
76713
|
+
var import_perf_hooks = __require("perf_hooks");
|
|
76713
76714
|
(0, import_dotenv.config)();
|
|
76714
76715
|
if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
|
|
76715
76716
|
throw new Error("'AWS_ACCESS_KEY_ID' not set");
|
|
@@ -76746,14 +76747,26 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
|
|
|
76746
76747
|
const result = [];
|
|
76747
76748
|
const errored = [];
|
|
76748
76749
|
const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
|
|
76750
|
+
if (options.logging) {
|
|
76751
|
+
console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
|
|
76752
|
+
console.time(`[SaveFiles] Total time`);
|
|
76753
|
+
}
|
|
76749
76754
|
for (const { key, filePath } of keys) {
|
|
76755
|
+
const start = import_perf_hooks.performance.now();
|
|
76756
|
+
if (options.logging) {
|
|
76757
|
+
console.info(`
|
|
76758
|
+
[Processing Start] key=${key}, filePath=${filePath}`);
|
|
76759
|
+
}
|
|
76750
76760
|
try {
|
|
76751
76761
|
const s3Key = `${options.savePath}/${key}`;
|
|
76762
|
+
if (options.logging) {
|
|
76763
|
+
console.info(`[S3 Key] ${s3Key}`);
|
|
76764
|
+
}
|
|
76752
76765
|
let stream5 = void 0;
|
|
76753
76766
|
let buffer = void 0;
|
|
76754
76767
|
if (isHttp(filePath)) {
|
|
76755
76768
|
if (options.logging) {
|
|
76756
|
-
console.info(`[Start
|
|
76769
|
+
console.info(`[Download Start] ${filePath}`);
|
|
76757
76770
|
}
|
|
76758
76771
|
const axiosResponse = yield axios_default.get(filePath, {
|
|
76759
76772
|
responseType: "arraybuffer",
|
|
@@ -76761,19 +76774,33 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
|
|
|
76761
76774
|
httpsAgent: proxyAgent
|
|
76762
76775
|
});
|
|
76763
76776
|
if (options.logging) {
|
|
76764
|
-
console.info(`[Success
|
|
76777
|
+
console.info(`[Download Success] ${filePath}`);
|
|
76765
76778
|
}
|
|
76766
76779
|
buffer = Buffer.from(axiosResponse.data);
|
|
76767
76780
|
} else {
|
|
76781
|
+
if (options.logging) {
|
|
76782
|
+
console.info(`[Read from FS] ${filePath}`);
|
|
76783
|
+
}
|
|
76768
76784
|
stream5 = (0, import_fs8.createReadStream)(filePath);
|
|
76769
76785
|
}
|
|
76770
76786
|
const tagString = options.tags.map(
|
|
76771
76787
|
(tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
|
|
76772
76788
|
).join("&");
|
|
76773
76789
|
if (!stream5 && !buffer) {
|
|
76790
|
+
if (options.logging) {
|
|
76791
|
+
console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
|
|
76792
|
+
}
|
|
76774
76793
|
throw new Error("buffer or stream not set");
|
|
76775
76794
|
}
|
|
76795
|
+
if (options.logging) {
|
|
76796
|
+
console.info(`[Image Processing Start] ${key}`);
|
|
76797
|
+
}
|
|
76776
76798
|
const image2 = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
|
|
76799
|
+
if (options.logging) {
|
|
76800
|
+
console.info(
|
|
76801
|
+
`[Image Loaded] key=${key}, width=${image2.width}, height=${image2.height}`
|
|
76802
|
+
);
|
|
76803
|
+
}
|
|
76777
76804
|
const command = new PutObjectCommand({
|
|
76778
76805
|
Body: buffer || stream5,
|
|
76779
76806
|
Bucket: BUCKET_NAME,
|
|
@@ -76791,16 +76818,36 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
|
|
|
76791
76818
|
filePath,
|
|
76792
76819
|
uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
|
|
76793
76820
|
});
|
|
76821
|
+
if (options.logging) {
|
|
76822
|
+
console.info(`[Uploading] ${key} to S3...`);
|
|
76823
|
+
}
|
|
76794
76824
|
yield s32.send(command);
|
|
76825
|
+
if (options.logging) {
|
|
76826
|
+
console.info(`[Upload Success] ${key}`);
|
|
76827
|
+
}
|
|
76795
76828
|
} catch (e5) {
|
|
76796
76829
|
const error = e5;
|
|
76797
|
-
|
|
76830
|
+
if (options.logging) {
|
|
76831
|
+
console.error(`[Upload Error] key=${key}, error=${error.message}`);
|
|
76832
|
+
}
|
|
76798
76833
|
errored.push({
|
|
76799
76834
|
key,
|
|
76800
76835
|
filePath,
|
|
76801
76836
|
message: error.message
|
|
76802
76837
|
});
|
|
76803
76838
|
}
|
|
76839
|
+
const end = import_perf_hooks.performance.now();
|
|
76840
|
+
if (options.logging) {
|
|
76841
|
+
console.info(
|
|
76842
|
+
`[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
|
|
76843
|
+
);
|
|
76844
|
+
}
|
|
76845
|
+
}
|
|
76846
|
+
if (options.logging) {
|
|
76847
|
+
console.timeEnd(`[SaveFiles] Total time`);
|
|
76848
|
+
console.info(
|
|
76849
|
+
`[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
|
|
76850
|
+
);
|
|
76804
76851
|
}
|
|
76805
76852
|
return { result, errored };
|
|
76806
76853
|
});
|
package/dist/index.js
CHANGED
|
@@ -32,6 +32,7 @@ import { config } from "dotenv";
|
|
|
32
32
|
import axios from "axios";
|
|
33
33
|
import { Jimp } from "jimp";
|
|
34
34
|
import { HttpsProxyAgent } from "https-proxy-agent";
|
|
35
|
+
import { performance } from "perf_hooks";
|
|
35
36
|
config();
|
|
36
37
|
if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
|
|
37
38
|
throw new Error("'AWS_ACCESS_KEY_ID' not set");
|
|
@@ -68,14 +69,26 @@ var S3Storage = class {
|
|
|
68
69
|
const result = [];
|
|
69
70
|
const errored = [];
|
|
70
71
|
const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
|
|
72
|
+
if (options.logging) {
|
|
73
|
+
console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
|
|
74
|
+
console.time(`[SaveFiles] Total time`);
|
|
75
|
+
}
|
|
71
76
|
for (const { key, filePath } of keys) {
|
|
77
|
+
const start = performance.now();
|
|
78
|
+
if (options.logging) {
|
|
79
|
+
console.info(`
|
|
80
|
+
[Processing Start] key=${key}, filePath=${filePath}`);
|
|
81
|
+
}
|
|
72
82
|
try {
|
|
73
83
|
const s3Key = `${options.savePath}/${key}`;
|
|
84
|
+
if (options.logging) {
|
|
85
|
+
console.info(`[S3 Key] ${s3Key}`);
|
|
86
|
+
}
|
|
74
87
|
let stream = void 0;
|
|
75
88
|
let buffer = void 0;
|
|
76
89
|
if (isHttp(filePath)) {
|
|
77
90
|
if (options.logging) {
|
|
78
|
-
console.info(`[Start
|
|
91
|
+
console.info(`[Download Start] ${filePath}`);
|
|
79
92
|
}
|
|
80
93
|
const axiosResponse = yield axios.get(filePath, {
|
|
81
94
|
responseType: "arraybuffer",
|
|
@@ -83,19 +96,33 @@ var S3Storage = class {
|
|
|
83
96
|
httpsAgent: proxyAgent
|
|
84
97
|
});
|
|
85
98
|
if (options.logging) {
|
|
86
|
-
console.info(`[Success
|
|
99
|
+
console.info(`[Download Success] ${filePath}`);
|
|
87
100
|
}
|
|
88
101
|
buffer = Buffer.from(axiosResponse.data);
|
|
89
102
|
} else {
|
|
103
|
+
if (options.logging) {
|
|
104
|
+
console.info(`[Read from FS] ${filePath}`);
|
|
105
|
+
}
|
|
90
106
|
stream = createReadStream(filePath);
|
|
91
107
|
}
|
|
92
108
|
const tagString = options.tags.map(
|
|
93
109
|
(tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
|
|
94
110
|
).join("&");
|
|
95
111
|
if (!stream && !buffer) {
|
|
112
|
+
if (options.logging) {
|
|
113
|
+
console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
|
|
114
|
+
}
|
|
96
115
|
throw new Error("buffer or stream not set");
|
|
97
116
|
}
|
|
117
|
+
if (options.logging) {
|
|
118
|
+
console.info(`[Image Processing Start] ${key}`);
|
|
119
|
+
}
|
|
98
120
|
const image = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
|
|
121
|
+
if (options.logging) {
|
|
122
|
+
console.info(
|
|
123
|
+
`[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
|
|
124
|
+
);
|
|
125
|
+
}
|
|
99
126
|
const command = new PutObjectCommand({
|
|
100
127
|
Body: buffer || stream,
|
|
101
128
|
Bucket: BUCKET_NAME,
|
|
@@ -113,16 +140,36 @@ var S3Storage = class {
|
|
|
113
140
|
filePath,
|
|
114
141
|
uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
|
|
115
142
|
});
|
|
143
|
+
if (options.logging) {
|
|
144
|
+
console.info(`[Uploading] ${key} to S3...`);
|
|
145
|
+
}
|
|
116
146
|
yield s3.send(command);
|
|
147
|
+
if (options.logging) {
|
|
148
|
+
console.info(`[Upload Success] ${key}`);
|
|
149
|
+
}
|
|
117
150
|
} catch (e) {
|
|
118
151
|
const error = e;
|
|
119
|
-
|
|
152
|
+
if (options.logging) {
|
|
153
|
+
console.error(`[Upload Error] key=${key}, error=${error.message}`);
|
|
154
|
+
}
|
|
120
155
|
errored.push({
|
|
121
156
|
key,
|
|
122
157
|
filePath,
|
|
123
158
|
message: error.message
|
|
124
159
|
});
|
|
125
160
|
}
|
|
161
|
+
const end = performance.now();
|
|
162
|
+
if (options.logging) {
|
|
163
|
+
console.info(
|
|
164
|
+
`[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (options.logging) {
|
|
169
|
+
console.timeEnd(`[SaveFiles] Total time`);
|
|
170
|
+
console.info(
|
|
171
|
+
`[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
|
|
172
|
+
);
|
|
126
173
|
}
|
|
127
174
|
return { result, errored };
|
|
128
175
|
});
|