importer-storage 1.0.14 → 1.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -61,11 +61,12 @@ var import_dotenv = require("dotenv");
61
61
  var import_axios = __toESM(require("axios"), 1);
62
62
  var import_jimp = require("jimp");
63
63
  var import_https_proxy_agent = require("https-proxy-agent");
64
+ var import_perf_hooks = require("perf_hooks");
64
65
  (0, import_dotenv.config)();
65
- if (!process.env.AWS_ACCESS_KEY_ID) {
66
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
66
67
  throw new Error("'AWS_ACCESS_KEY_ID' not set");
67
68
  }
68
- if (!process.env.AWS_ACCESS_KEY) {
69
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY) {
69
70
  throw new Error("'AWS_ACCESS_KEY' not set");
70
71
  }
71
72
  function extractFileInfo(filePath) {
@@ -75,13 +76,15 @@ function extractFileInfo(filePath) {
75
76
  const extension = extMatch ? extMatch[1].toLowerCase() : null;
76
77
  return { key, extension: extension || "jpg" };
77
78
  }
78
- var s3 = new import_client_s3.S3Client({
79
- region: process.env.AWS_REGION,
80
- credentials: {
81
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
82
- secretAccessKey: process.env.AWS_ACCESS_KEY
83
- }
84
- });
79
+ var s3 = new import_client_s3.S3Client(
80
+ !process.env.AWS_EXECUTION_ENV && process.env.AWS_ACCESS_KEY_ID && process.env.AWS_ACCESS_KEY && process.env.AWS_REGION ? {
81
+ region: process.env.AWS_REGION,
82
+ credentials: {
83
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
84
+ secretAccessKey: process.env.AWS_ACCESS_KEY
85
+ }
86
+ } : {}
87
+ );
85
88
  var BUCKET_NAME = process.env.AWS_BUCKET_NAME;
86
89
  if (!BUCKET_NAME) {
87
90
  throw new Error("'AWS_BUCKET_NAME' name not set");
@@ -95,14 +98,26 @@ var S3Storage = class {
95
98
  const result = [];
96
99
  const errored = [];
97
100
  const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
101
+ if (options.logging) {
102
+ console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
103
+ console.time(`[SaveFiles] Total time`);
104
+ }
98
105
  for (const { key, filePath } of keys) {
106
+ const start = import_perf_hooks.performance.now();
107
+ if (options.logging) {
108
+ console.info(`
109
+ [Processing Start] key=${key}, filePath=${filePath}`);
110
+ }
99
111
  try {
100
112
  const s3Key = `${options.savePath}/${key}`;
113
+ if (options.logging) {
114
+ console.info(`[S3 Key] ${s3Key}`);
115
+ }
101
116
  let stream = void 0;
102
117
  let buffer = void 0;
103
118
  if (isHttp(filePath)) {
104
119
  if (options.logging) {
105
- console.info(`[Start downloading] ${filePath}`);
120
+ console.info(`[Download Start] ${filePath}`);
106
121
  }
107
122
  const axiosResponse = yield import_axios.default.get(filePath, {
108
123
  responseType: "arraybuffer",
@@ -110,19 +125,33 @@ var S3Storage = class {
110
125
  httpsAgent: proxyAgent
111
126
  });
112
127
  if (options.logging) {
113
- console.info(`[Success downloading] ${filePath}`);
128
+ console.info(`[Download Success] ${filePath}`);
114
129
  }
115
130
  buffer = Buffer.from(axiosResponse.data);
116
131
  } else {
132
+ if (options.logging) {
133
+ console.info(`[Read from FS] ${filePath}`);
134
+ }
117
135
  stream = (0, import_fs.createReadStream)(filePath);
118
136
  }
119
137
  const tagString = options.tags.map(
120
138
  (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
121
139
  ).join("&");
122
140
  if (!stream && !buffer) {
141
+ if (options.logging) {
142
+ console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
143
+ }
123
144
  throw new Error("buffer or stream not set");
124
145
  }
146
+ if (options.logging) {
147
+ console.info(`[Image Processing Start] ${key}`);
148
+ }
125
149
  const image = buffer ? yield import_jimp.Jimp.fromBuffer(buffer) : yield import_jimp.Jimp.read(filePath);
150
+ if (options.logging) {
151
+ console.info(
152
+ `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
153
+ );
154
+ }
126
155
  const command = new import_client_s3.PutObjectCommand({
127
156
  Body: buffer || stream,
128
157
  Bucket: BUCKET_NAME,
@@ -140,16 +169,36 @@ var S3Storage = class {
140
169
  filePath,
141
170
  uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
142
171
  });
172
+ if (options.logging) {
173
+ console.info(`[Uploading] ${key} to S3...`);
174
+ }
143
175
  yield s3.send(command);
176
+ if (options.logging) {
177
+ console.info(`[Upload Success] ${key}`);
178
+ }
144
179
  } catch (e) {
145
180
  const error = e;
146
- console.warn(error.message);
181
+ if (options.logging) {
182
+ console.error(`[Upload Error] key=${key}, error=${error.message}`);
183
+ }
147
184
  errored.push({
148
185
  key,
149
186
  filePath,
150
187
  message: error.message
151
188
  });
152
189
  }
190
+ const end = import_perf_hooks.performance.now();
191
+ if (options.logging) {
192
+ console.info(
193
+ `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
194
+ );
195
+ }
196
+ }
197
+ if (options.logging) {
198
+ console.timeEnd(`[SaveFiles] Total time`);
199
+ console.info(
200
+ `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
201
+ );
153
202
  }
154
203
  return { result, errored };
155
204
  });
@@ -76710,11 +76710,12 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
76710
76710
 
76711
76711
  // src/logic/storage.ts
76712
76712
  var import_https_proxy_agent = __toESM(require_dist2(), 1);
76713
+ var import_perf_hooks = __require("perf_hooks");
76713
76714
  (0, import_dotenv.config)();
76714
- if (!process.env.AWS_ACCESS_KEY_ID) {
76715
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
76715
76716
  throw new Error("'AWS_ACCESS_KEY_ID' not set");
76716
76717
  }
76717
- if (!process.env.AWS_ACCESS_KEY) {
76718
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY) {
76718
76719
  throw new Error("'AWS_ACCESS_KEY' not set");
76719
76720
  }
76720
76721
  function extractFileInfo(filePath) {
@@ -76724,13 +76725,15 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
76724
76725
  const extension = extMatch ? extMatch[1].toLowerCase() : null;
76725
76726
  return { key, extension: extension || "jpg" };
76726
76727
  }
76727
- var s32 = new S3Client({
76728
- region: process.env.AWS_REGION,
76729
- credentials: {
76730
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
76731
- secretAccessKey: process.env.AWS_ACCESS_KEY
76732
- }
76733
- });
76728
+ var s32 = new S3Client(
76729
+ !process.env.AWS_EXECUTION_ENV && process.env.AWS_ACCESS_KEY_ID && process.env.AWS_ACCESS_KEY && process.env.AWS_REGION ? {
76730
+ region: process.env.AWS_REGION,
76731
+ credentials: {
76732
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
76733
+ secretAccessKey: process.env.AWS_ACCESS_KEY
76734
+ }
76735
+ } : {}
76736
+ );
76734
76737
  var BUCKET_NAME = process.env.AWS_BUCKET_NAME;
76735
76738
  if (!BUCKET_NAME) {
76736
76739
  throw new Error("'AWS_BUCKET_NAME' name not set");
@@ -76744,14 +76747,26 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
76744
76747
  const result = [];
76745
76748
  const errored = [];
76746
76749
  const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
76750
+ if (options.logging) {
76751
+ console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
76752
+ console.time(`[SaveFiles] Total time`);
76753
+ }
76747
76754
  for (const { key, filePath } of keys) {
76755
+ const start = import_perf_hooks.performance.now();
76756
+ if (options.logging) {
76757
+ console.info(`
76758
+ [Processing Start] key=${key}, filePath=${filePath}`);
76759
+ }
76748
76760
  try {
76749
76761
  const s3Key = `${options.savePath}/${key}`;
76762
+ if (options.logging) {
76763
+ console.info(`[S3 Key] ${s3Key}`);
76764
+ }
76750
76765
  let stream5 = void 0;
76751
76766
  let buffer = void 0;
76752
76767
  if (isHttp(filePath)) {
76753
76768
  if (options.logging) {
76754
- console.info(`[Start downloading] ${filePath}`);
76769
+ console.info(`[Download Start] ${filePath}`);
76755
76770
  }
76756
76771
  const axiosResponse = yield axios_default.get(filePath, {
76757
76772
  responseType: "arraybuffer",
@@ -76759,19 +76774,33 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
76759
76774
  httpsAgent: proxyAgent
76760
76775
  });
76761
76776
  if (options.logging) {
76762
- console.info(`[Success downloading] ${filePath}`);
76777
+ console.info(`[Download Success] ${filePath}`);
76763
76778
  }
76764
76779
  buffer = Buffer.from(axiosResponse.data);
76765
76780
  } else {
76781
+ if (options.logging) {
76782
+ console.info(`[Read from FS] ${filePath}`);
76783
+ }
76766
76784
  stream5 = (0, import_fs8.createReadStream)(filePath);
76767
76785
  }
76768
76786
  const tagString = options.tags.map(
76769
76787
  (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
76770
76788
  ).join("&");
76771
76789
  if (!stream5 && !buffer) {
76790
+ if (options.logging) {
76791
+ console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
76792
+ }
76772
76793
  throw new Error("buffer or stream not set");
76773
76794
  }
76795
+ if (options.logging) {
76796
+ console.info(`[Image Processing Start] ${key}`);
76797
+ }
76774
76798
  const image2 = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
76799
+ if (options.logging) {
76800
+ console.info(
76801
+ `[Image Loaded] key=${key}, width=${image2.width}, height=${image2.height}`
76802
+ );
76803
+ }
76775
76804
  const command = new PutObjectCommand({
76776
76805
  Body: buffer || stream5,
76777
76806
  Bucket: BUCKET_NAME,
@@ -76789,16 +76818,36 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
76789
76818
  filePath,
76790
76819
  uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
76791
76820
  });
76821
+ if (options.logging) {
76822
+ console.info(`[Uploading] ${key} to S3...`);
76823
+ }
76792
76824
  yield s32.send(command);
76825
+ if (options.logging) {
76826
+ console.info(`[Upload Success] ${key}`);
76827
+ }
76793
76828
  } catch (e5) {
76794
76829
  const error = e5;
76795
- console.warn(error.message);
76830
+ if (options.logging) {
76831
+ console.error(`[Upload Error] key=${key}, error=${error.message}`);
76832
+ }
76796
76833
  errored.push({
76797
76834
  key,
76798
76835
  filePath,
76799
76836
  message: error.message
76800
76837
  });
76801
76838
  }
76839
+ const end = import_perf_hooks.performance.now();
76840
+ if (options.logging) {
76841
+ console.info(
76842
+ `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
76843
+ );
76844
+ }
76845
+ }
76846
+ if (options.logging) {
76847
+ console.timeEnd(`[SaveFiles] Total time`);
76848
+ console.info(
76849
+ `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
76850
+ );
76802
76851
  }
76803
76852
  return { result, errored };
76804
76853
  });
package/dist/index.js CHANGED
@@ -32,11 +32,12 @@ import { config } from "dotenv";
32
32
  import axios from "axios";
33
33
  import { Jimp } from "jimp";
34
34
  import { HttpsProxyAgent } from "https-proxy-agent";
35
+ import { performance } from "perf_hooks";
35
36
  config();
36
- if (!process.env.AWS_ACCESS_KEY_ID) {
37
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
37
38
  throw new Error("'AWS_ACCESS_KEY_ID' not set");
38
39
  }
39
- if (!process.env.AWS_ACCESS_KEY) {
40
+ if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY) {
40
41
  throw new Error("'AWS_ACCESS_KEY' not set");
41
42
  }
42
43
  function extractFileInfo(filePath) {
@@ -46,13 +47,15 @@ function extractFileInfo(filePath) {
46
47
  const extension = extMatch ? extMatch[1].toLowerCase() : null;
47
48
  return { key, extension: extension || "jpg" };
48
49
  }
49
- var s3 = new S3Client({
50
- region: process.env.AWS_REGION,
51
- credentials: {
52
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
53
- secretAccessKey: process.env.AWS_ACCESS_KEY
54
- }
55
- });
50
+ var s3 = new S3Client(
51
+ !process.env.AWS_EXECUTION_ENV && process.env.AWS_ACCESS_KEY_ID && process.env.AWS_ACCESS_KEY && process.env.AWS_REGION ? {
52
+ region: process.env.AWS_REGION,
53
+ credentials: {
54
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
55
+ secretAccessKey: process.env.AWS_ACCESS_KEY
56
+ }
57
+ } : {}
58
+ );
56
59
  var BUCKET_NAME = process.env.AWS_BUCKET_NAME;
57
60
  if (!BUCKET_NAME) {
58
61
  throw new Error("'AWS_BUCKET_NAME' name not set");
@@ -66,14 +69,26 @@ var S3Storage = class {
66
69
  const result = [];
67
70
  const errored = [];
68
71
  const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
72
+ if (options.logging) {
73
+ console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
74
+ console.time(`[SaveFiles] Total time`);
75
+ }
69
76
  for (const { key, filePath } of keys) {
77
+ const start = performance.now();
78
+ if (options.logging) {
79
+ console.info(`
80
+ [Processing Start] key=${key}, filePath=${filePath}`);
81
+ }
70
82
  try {
71
83
  const s3Key = `${options.savePath}/${key}`;
84
+ if (options.logging) {
85
+ console.info(`[S3 Key] ${s3Key}`);
86
+ }
72
87
  let stream = void 0;
73
88
  let buffer = void 0;
74
89
  if (isHttp(filePath)) {
75
90
  if (options.logging) {
76
- console.info(`[Start downloading] ${filePath}`);
91
+ console.info(`[Download Start] ${filePath}`);
77
92
  }
78
93
  const axiosResponse = yield axios.get(filePath, {
79
94
  responseType: "arraybuffer",
@@ -81,19 +96,33 @@ var S3Storage = class {
81
96
  httpsAgent: proxyAgent
82
97
  });
83
98
  if (options.logging) {
84
- console.info(`[Success downloading] ${filePath}`);
99
+ console.info(`[Download Success] ${filePath}`);
85
100
  }
86
101
  buffer = Buffer.from(axiosResponse.data);
87
102
  } else {
103
+ if (options.logging) {
104
+ console.info(`[Read from FS] ${filePath}`);
105
+ }
88
106
  stream = createReadStream(filePath);
89
107
  }
90
108
  const tagString = options.tags.map(
91
109
  (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
92
110
  ).join("&");
93
111
  if (!stream && !buffer) {
112
+ if (options.logging) {
113
+ console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
114
+ }
94
115
  throw new Error("buffer or stream not set");
95
116
  }
117
+ if (options.logging) {
118
+ console.info(`[Image Processing Start] ${key}`);
119
+ }
96
120
  const image = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
121
+ if (options.logging) {
122
+ console.info(
123
+ `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
124
+ );
125
+ }
97
126
  const command = new PutObjectCommand({
98
127
  Body: buffer || stream,
99
128
  Bucket: BUCKET_NAME,
@@ -111,16 +140,36 @@ var S3Storage = class {
111
140
  filePath,
112
141
  uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
113
142
  });
143
+ if (options.logging) {
144
+ console.info(`[Uploading] ${key} to S3...`);
145
+ }
114
146
  yield s3.send(command);
147
+ if (options.logging) {
148
+ console.info(`[Upload Success] ${key}`);
149
+ }
115
150
  } catch (e) {
116
151
  const error = e;
117
- console.warn(error.message);
152
+ if (options.logging) {
153
+ console.error(`[Upload Error] key=${key}, error=${error.message}`);
154
+ }
118
155
  errored.push({
119
156
  key,
120
157
  filePath,
121
158
  message: error.message
122
159
  });
123
160
  }
161
+ const end = performance.now();
162
+ if (options.logging) {
163
+ console.info(
164
+ `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
165
+ );
166
+ }
167
+ }
168
+ if (options.logging) {
169
+ console.timeEnd(`[SaveFiles] Total time`);
170
+ console.info(
171
+ `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
172
+ );
124
173
  }
125
174
  return { result, errored };
126
175
  });
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "importer-storage",
3
3
  "private": false,
4
- "version": "1.0.14",
4
+ "version": "1.0.16",
5
5
  "main": "dist/index.js",
6
6
  "files": [
7
7
  "dist",