@blocklet/crawler 2.4.37 → 2.4.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -104,11 +104,11 @@ const setUrlInfoToCache = async ({
|
|
|
104
104
|
if (!content || !url) {
|
|
105
105
|
return;
|
|
106
106
|
}
|
|
107
|
-
const lastmodValue = lastmod || (
|
|
107
|
+
const lastmodValue = lastmod || (/* @__PURE__ */new Date()).toISOString();
|
|
108
108
|
return await _utils.useCache.set((0, _utils.getRelativePath)(url), {
|
|
109
109
|
content,
|
|
110
110
|
lastmod: lastmodValue,
|
|
111
|
-
updatedAt: (
|
|
111
|
+
updatedAt: (/* @__PURE__ */new Date()).toISOString(),
|
|
112
112
|
nextDate: nextDate || getNextCrawlDate(lastmodValue)
|
|
113
113
|
});
|
|
114
114
|
};
|
|
@@ -141,7 +141,7 @@ const crawlUrl = async ({
|
|
|
141
141
|
formatPageContent
|
|
142
142
|
});
|
|
143
143
|
if (pageContent) {
|
|
144
|
-
const lastmodValue = lastmodMap?.get(url) || (
|
|
144
|
+
const lastmodValue = lastmodMap?.get(url) || (/* @__PURE__ */new Date()).toISOString();
|
|
145
145
|
await setUrlInfoToCache({
|
|
146
146
|
url,
|
|
147
147
|
content: pageContent,
|
|
@@ -242,7 +242,7 @@ const crawlBlocklet = async () => {
|
|
|
242
242
|
skipBlockletLocTotal++;
|
|
243
243
|
return false;
|
|
244
244
|
}
|
|
245
|
-
if (nextDate && new Date(nextDate).getTime() >= (
|
|
245
|
+
if (nextDate && new Date(nextDate).getTime() >= (/* @__PURE__ */new Date()).getTime()) {
|
|
246
246
|
skipBlockletLocTotal++;
|
|
247
247
|
return false;
|
|
248
248
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@blocklet/crawler",
|
|
3
|
-
"version": "2.4.
|
|
3
|
+
"version": "2.4.38",
|
|
4
4
|
"description": "blocklet crawler lib",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"author": "arcblock <blocklet@arcblock.io> https://github.com/blocklet",
|
|
48
48
|
"license": "ISC",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@abtnode/cron": "^1.16.
|
|
50
|
+
"@abtnode/cron": "^1.16.48",
|
|
51
51
|
"@blocklet/puppeteer": "^22.11.3",
|
|
52
|
-
"@blocklet/sdk": "^1.16.
|
|
52
|
+
"@blocklet/sdk": "^1.16.48",
|
|
53
53
|
"axios": "^1.7.7",
|
|
54
54
|
"fs-extra": "^11.2.0",
|
|
55
55
|
"generic-pool": "^3.9.0",
|