@arcblock/crawler 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -44,8 +44,18 @@ function formatSnapshot(snapshot, columns) {
|
|
|
44
44
|
}
|
|
45
45
|
// format html path to string
|
|
46
46
|
if (data.html) {
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
try {
|
|
48
|
+
const html = yield promises_1.default.readFile(node_path_1.default.join(config_1.config.dataDir, data.html));
|
|
49
|
+
data.html = html.toString();
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
config_1.logger.error('Failed to read html', {
|
|
53
|
+
err,
|
|
54
|
+
dataDir: config_1.config.dataDir,
|
|
55
|
+
snapshot,
|
|
56
|
+
});
|
|
57
|
+
data.html = '';
|
|
58
|
+
}
|
|
49
59
|
}
|
|
50
60
|
// remove sensitive options that should not be returned
|
|
51
61
|
if (data.options) {
|
|
@@ -104,10 +114,15 @@ function deleteSnapshots(where_1) {
|
|
|
104
114
|
});
|
|
105
115
|
const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
|
|
106
116
|
try {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
117
|
+
try {
|
|
118
|
+
yield Promise.all([
|
|
119
|
+
snapshot.html && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.html)),
|
|
120
|
+
snapshot.screenshot && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.screenshot)),
|
|
121
|
+
]);
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
config_1.logger.error('Failed to delete snapshot', { err, snapshot, dataDir: config_1.config.dataDir });
|
|
125
|
+
}
|
|
111
126
|
yield snapshot.destroy({ transaction: txn });
|
|
112
127
|
return snapshot.jobId;
|
|
113
128
|
}
|
|
@@ -34,8 +34,18 @@ export function formatSnapshot(snapshot, columns) {
|
|
|
34
34
|
}
|
|
35
35
|
// format html path to string
|
|
36
36
|
if (data.html) {
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
try {
|
|
38
|
+
const html = yield fs.readFile(path.join(config.dataDir, data.html));
|
|
39
|
+
data.html = html.toString();
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
logger.error('Failed to read html', {
|
|
43
|
+
err,
|
|
44
|
+
dataDir: config.dataDir,
|
|
45
|
+
snapshot,
|
|
46
|
+
});
|
|
47
|
+
data.html = '';
|
|
48
|
+
}
|
|
39
49
|
}
|
|
40
50
|
// remove sensitive options that should not be returned
|
|
41
51
|
if (data.options) {
|
|
@@ -94,10 +104,15 @@ export function deleteSnapshots(where_1) {
|
|
|
94
104
|
});
|
|
95
105
|
const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
|
|
96
106
|
try {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
107
|
+
try {
|
|
108
|
+
yield Promise.all([
|
|
109
|
+
snapshot.html && fs.unlink(path.join(config.dataDir, snapshot.html)),
|
|
110
|
+
snapshot.screenshot && fs.unlink(path.join(config.dataDir, snapshot.screenshot)),
|
|
111
|
+
]);
|
|
112
|
+
}
|
|
113
|
+
catch (err) {
|
|
114
|
+
logger.error('Failed to delete snapshot', { err, snapshot, dataDir: config.dataDir });
|
|
115
|
+
}
|
|
101
116
|
yield snapshot.destroy({ transaction: txn });
|
|
102
117
|
return snapshot.jobId;
|
|
103
118
|
}
|