@cocreate/sitemap 1.4.2 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/automated.yml +25 -38
- package/CHANGELOG.md +29 -0
- package/package.json +5 -5
- package/release.config.js +12 -4
- package/src/index.js +528 -553
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
name: Automated Workflow
|
|
2
|
+
|
|
2
3
|
on:
|
|
3
4
|
push:
|
|
4
5
|
branches:
|
|
5
6
|
- master
|
|
7
|
+
|
|
6
8
|
jobs:
|
|
7
9
|
about:
|
|
8
10
|
runs-on: ubuntu-latest
|
|
@@ -18,52 +20,37 @@ jobs:
|
|
|
18
20
|
with:
|
|
19
21
|
direction: overwrite-github
|
|
20
22
|
githubToken: "${{ secrets.GITHUB }}"
|
|
23
|
+
|
|
21
24
|
release:
|
|
22
25
|
runs-on: ubuntu-latest
|
|
23
26
|
steps:
|
|
24
27
|
- name: Checkout
|
|
25
|
-
uses: actions/checkout@
|
|
26
|
-
- name: Setup Node.js
|
|
27
|
-
uses: actions/setup-node@v3
|
|
28
|
+
uses: actions/checkout@v4
|
|
28
29
|
with:
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
fetch-depth: 0 # Required so semantic-release can trace git tags/history
|
|
31
|
+
|
|
32
|
+
- name: Setup Node.js
|
|
33
|
+
uses: actions/setup-node@v4
|
|
33
34
|
with:
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
node-version: 22
|
|
36
|
+
|
|
37
|
+
- name: Install Semantic Release & Plugins
|
|
38
|
+
# Installs semantic-release and its plugins on the runner
|
|
39
|
+
run: |
|
|
40
|
+
npm install -g semantic-release \
|
|
41
|
+
@semantic-release/changelog \
|
|
42
|
+
@semantic-release/npm \
|
|
43
|
+
@semantic-release/github \
|
|
36
44
|
@semantic-release/git
|
|
37
|
-
|
|
45
|
+
|
|
46
|
+
- name: Run Semantic Release (Native)
|
|
47
|
+
id: semantic
|
|
48
|
+
# This will automatically pick up your export default config file in the repository root
|
|
49
|
+
run: npx semantic-release
|
|
38
50
|
env:
|
|
39
|
-
GITHUB_TOKEN: "${{ secrets.
|
|
51
|
+
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
|
40
52
|
NPM_TOKEN: "${{ secrets.NPM_TOKEN }}"
|
|
53
|
+
|
|
41
54
|
outputs:
|
|
42
55
|
new_release_published: "${{ steps.semantic.outputs.new_release_published }}"
|
|
43
|
-
new_release_version: "${{ steps.semantic.outputs.new_release_version }}"
|
|
44
|
-
upload:
|
|
45
|
-
runs-on: ubuntu-latest
|
|
46
|
-
needs: release
|
|
47
|
-
if: needs.release.outputs.new_release_published == 'true'
|
|
48
|
-
env:
|
|
49
|
-
VERSION: "${{ needs.release.outputs.new_release_version }}"
|
|
50
|
-
steps:
|
|
51
|
-
- name: Checkout
|
|
52
|
-
uses: actions/checkout@v3
|
|
53
|
-
- name: Setup Node.js
|
|
54
|
-
uses: actions/setup-node@v3
|
|
55
|
-
with:
|
|
56
|
-
node-version: 16
|
|
57
|
-
- name: Set npm registry auth
|
|
58
|
-
run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc
|
|
59
|
-
- name: Install dependencies
|
|
60
|
-
run: yarn install
|
|
61
|
-
|
|
62
|
-
- name: Set Environment Variables
|
|
63
|
-
run: |
|
|
64
|
-
echo "organization_id=${{ secrets.COCREATE_ORGANIZATION_ID }}" >> $GITHUB_ENV
|
|
65
|
-
echo "key=${{ secrets.COCREATE_KEY }}" >> $GITHUB_ENV
|
|
66
|
-
echo "host=${{ secrets.COCREATE_HOST }}" >> $GITHUB_ENV
|
|
67
|
-
- name: CoCreate Upload
|
|
68
|
-
run: coc upload
|
|
69
|
-
|
|
56
|
+
new_release_version: "${{ steps.semantic.outputs.new_release_version }}"
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,32 @@
|
|
|
1
|
+
## [1.6.1](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.6.0...v1.6.1) (2026-07-17)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* bump dependencies ([898562c](https://github.com/CoCreate-app/CoCreate-sitemap/commit/898562ce06850982fc169c485a2d89c3f2ac2c4d))
|
|
7
|
+
|
|
8
|
+
# [1.6.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.5.0...v1.6.0) (2026-07-17)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* update automated workflow and release configuration for improved semantic release handling ([fd3183f](https://github.com/CoCreate-app/CoCreate-sitemap/commit/fd3183f36440cdc6ef003611161bcec483c094f7))
|
|
14
|
+
|
|
15
|
+
# [1.5.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.4.2...v1.5.0) (2026-07-14)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Bug Fixes
|
|
19
|
+
|
|
20
|
+
* removed post install ([65d5ea3](https://github.com/CoCreate-app/CoCreate-sitemap/commit/65d5ea331baf439992fab9c1dce5a95d7210e32e))
|
|
21
|
+
* semantic version handling. Reorganize .gitignore for improved clarity and structure ([63fa105](https://github.com/CoCreate-app/CoCreate-sitemap/commit/63fa105fe480103454c88f23d210585053190179))
|
|
22
|
+
* update .gitignore to include package-lock.json and pnpm-lock.yaml ([9bab5d7](https://github.com/CoCreate-app/CoCreate-sitemap/commit/9bab5d740c465515755a5daac62a55c697375b3d))
|
|
23
|
+
* update module export to ES6 syntax in release.config.js ([4096874](https://github.com/CoCreate-app/CoCreate-sitemap/commit/409687476de524c2bc1c88882031a8c43547c0e3))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
### Features
|
|
27
|
+
|
|
28
|
+
* update package.json and refactor index.js to use ES modules ([a951277](https://github.com/CoCreate-app/CoCreate-sitemap/commit/a9512776ddd5f74bf9f9aab0ce947b80b210f6b7))
|
|
29
|
+
|
|
1
30
|
## [1.4.2](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.4.1...v1.4.2) (2025-09-01)
|
|
2
31
|
|
|
3
32
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cocreate/sitemap",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.1",
|
|
4
4
|
"description": "A simple sitemap component in vanilla javascript. Easily configured using HTML5 data-attributes and/or JavaScript API.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sitemap",
|
|
@@ -16,10 +16,9 @@
|
|
|
16
16
|
"access": "public"
|
|
17
17
|
},
|
|
18
18
|
"scripts": {
|
|
19
|
-
"start": "
|
|
20
|
-
"build": "
|
|
21
|
-
"dev": "
|
|
22
|
-
"postinstall": "node -e \"const { execSync } = require('child_process'); try { execSync('coc --version', { stdio: 'ignore' }); } catch (error) { try { execSync('npm install -g @cocreate/cli', { stdio: 'inherit' }); console.log('Installed \"@cocreate/cli\" globally.'); } catch (error) { console.error('Failed to install \"@cocreate/cli\" globally:', error); } }\""
|
|
19
|
+
"start": "webpack --config webpack.config.js",
|
|
20
|
+
"build": "webpack --mode=production --config webpack.config.js",
|
|
21
|
+
"dev": "webpack --config webpack.config.js --watch"
|
|
23
22
|
},
|
|
24
23
|
"repository": {
|
|
25
24
|
"type": "git",
|
|
@@ -35,6 +34,7 @@
|
|
|
35
34
|
"type": "GitHub Sponsors ❤",
|
|
36
35
|
"url": "https://github.com/sponsors/CoCreate-app"
|
|
37
36
|
},
|
|
37
|
+
"type": "module",
|
|
38
38
|
"main": "./src/index.js",
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"node-html-parser": "^6.1.13"
|
package/release.config.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
export default {
|
|
2
2
|
dryRun: false,
|
|
3
3
|
branches: ["master"],
|
|
4
4
|
plugins: [
|
|
@@ -10,12 +10,20 @@ module.exports = {
|
|
|
10
10
|
changelogFile: "CHANGELOG.md",
|
|
11
11
|
},
|
|
12
12
|
],
|
|
13
|
-
"@semantic-release/npm",
|
|
14
|
-
"@semantic-release/github",
|
|
13
|
+
"@semantic-release/npm",
|
|
15
14
|
[
|
|
16
|
-
"@semantic-release/
|
|
15
|
+
"@semantic-release/github",
|
|
17
16
|
{
|
|
17
|
+
successComment: false,
|
|
18
|
+
failTitle: false,
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
[
|
|
22
|
+
"@semantic-release/git",
|
|
23
|
+
{
|
|
24
|
+
// Only stage and commit the changelog and package.json
|
|
18
25
|
assets: ["CHANGELOG.md", "package.json"],
|
|
26
|
+
message: "chore(release): ${nextRelease.version} [skip ci]",
|
|
19
27
|
},
|
|
20
28
|
],
|
|
21
29
|
],
|
package/src/index.js
CHANGED
|
@@ -1,555 +1,530 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
// Check if the file is HTML and contains a noindex meta or title tag
|
|
44
|
-
if (file["content-type"] === "text/html") {
|
|
45
|
-
if (
|
|
46
|
-
/<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(
|
|
47
|
-
file.src
|
|
48
|
-
)
|
|
49
|
-
)
|
|
50
|
-
return;
|
|
51
|
-
if (!/<title[^>]*>[\s\S]*?<\/title>/i.test(file.src)) return;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// Compare the lastmod date in the sitemap with the modified.on date
|
|
55
|
-
if (file.sitemap && file.sitemap.lastmod && file.modified.on) {
|
|
56
|
-
if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// Logic to update the sitemap
|
|
61
|
-
this.updateSitemap(file, host);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// ToDo: check for <link rel="alternate" hreflang="x" /> and create entries
|
|
65
|
-
async updateSitemap(file, host) {
|
|
66
|
-
try {
|
|
67
|
-
if (!file.sitemap) file.sitemap = {};
|
|
68
|
-
|
|
69
|
-
const entry = this.createEntry(file, host);
|
|
70
|
-
|
|
71
|
-
let { mainSitemap, sitemap } = await this.getSitemap(file, host);
|
|
72
|
-
|
|
73
|
-
if (file.pathname) {
|
|
74
|
-
// Perform regex search starting at the pathname
|
|
75
|
-
const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(
|
|
76
|
-
/[.*+?^${}()|[\]\\]/g,
|
|
77
|
-
"\\$&"
|
|
78
|
-
)}.*?</loc>[\\s\\S]*?</url>`;
|
|
79
|
-
const match = sitemap.src.match(new RegExp(regexPattern));
|
|
80
|
-
|
|
81
|
-
if (match) {
|
|
82
|
-
const position = match.index; // Start position of the <url> block
|
|
83
|
-
const endPosition = match.index + match[0].length; // End position of the <url> block
|
|
84
|
-
|
|
85
|
-
// Replace the original <url> block with the modified one
|
|
86
|
-
sitemap.src =
|
|
87
|
-
sitemap.src.slice(0, position) +
|
|
88
|
-
entry +
|
|
89
|
-
sitemap.src.slice(endPosition);
|
|
90
|
-
} else {
|
|
91
|
-
sitemap.src = sitemap.src.replace(
|
|
92
|
-
"</urlset>",
|
|
93
|
-
`${entry}</urlset>`
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
} else {
|
|
97
|
-
file.sitemap.pathname = sitemap.pathname;
|
|
98
|
-
sitemap.src = sitemap.src.replace(
|
|
99
|
-
"</urlset>",
|
|
100
|
-
`${entry}</urlset>`
|
|
101
|
-
);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
this.saveSitemap(mainSitemap, host);
|
|
105
|
-
this.saveSitemap(sitemap, host);
|
|
106
|
-
this.saveSitemap(file, host);
|
|
107
|
-
|
|
108
|
-
// console.log('Sitemap updated successfully.');
|
|
109
|
-
} catch (err) {
|
|
110
|
-
console.error("Error updating sitemap:", err);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
createEntry(file) {
|
|
115
|
-
file.sitemap.loc = file.pathname;
|
|
116
|
-
file.sitemap.lastmod = file.modified.on;
|
|
117
|
-
|
|
118
|
-
if (file["content-type"] === "text/html") {
|
|
119
|
-
this.parseHtml(file);
|
|
120
|
-
|
|
121
|
-
if (
|
|
122
|
-
file.sitemap.type !== "news" &&
|
|
123
|
-
file.sitemap.type !== "image" &&
|
|
124
|
-
file.sitemap.type !== "video"
|
|
125
|
-
) {
|
|
126
|
-
if (!file.sitemap.changefreq)
|
|
127
|
-
file.sitemap.changefreq = "monthly";
|
|
128
|
-
|
|
129
|
-
if (!file.sitemap.priority) {
|
|
130
|
-
const depth = (file.pathname.match(/\//g) || []).length;
|
|
131
|
-
file.sitemap.priority = Math.max(
|
|
132
|
-
0.1,
|
|
133
|
-
1.0 - (depth - 1) * 0.1
|
|
134
|
-
).toFixed(1);
|
|
135
|
-
}
|
|
136
|
-
} else {
|
|
137
|
-
delete file.sitemap.changefreq;
|
|
138
|
-
delete file.sitemap.priority;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
let entry = `\t<url>\n`;
|
|
143
|
-
|
|
144
|
-
for (const key of Object.keys(file.sitemap)) {
|
|
145
|
-
if (key === "pathname" || key === "type") continue;
|
|
146
|
-
|
|
147
|
-
let value = file.sitemap[key];
|
|
148
|
-
|
|
149
|
-
if (
|
|
150
|
-
typeof value === "object" &&
|
|
151
|
-
value !== null &&
|
|
152
|
-
!(value instanceof Date)
|
|
153
|
-
) {
|
|
154
|
-
if (!Array.isArray(value)) value = [value];
|
|
155
|
-
|
|
156
|
-
for (let i = 0; i < value.length; i++) {
|
|
157
|
-
entry += `\t\t<${key}:${key}>\n`;
|
|
158
|
-
|
|
159
|
-
for (const nestedKey of Object.keys(value[i])) {
|
|
160
|
-
let nestedValue = value[i][nestedKey];
|
|
161
|
-
// Handle nested objects
|
|
162
|
-
if (
|
|
163
|
-
typeof nestedValue === "object" &&
|
|
164
|
-
nestedValue !== null &&
|
|
165
|
-
!(nestedValue instanceof Date)
|
|
166
|
-
) {
|
|
167
|
-
entry += `\t\t\t<${key}:${nestedKey}>\n`;
|
|
168
|
-
for (const subKey of Object.keys(nestedValue)) {
|
|
169
|
-
const subValue = nestedValue[subKey];
|
|
170
|
-
entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
|
|
171
|
-
}
|
|
172
|
-
entry += `\t\t\t</${key}:${nestedKey}>\n`;
|
|
173
|
-
} else {
|
|
174
|
-
if (nestedKey === "loc") {
|
|
175
|
-
if (
|
|
176
|
-
!nestedValue.startsWith("https://") &&
|
|
177
|
-
!nestedValue.startsWith("http://") &&
|
|
178
|
-
!nestedValue.startsWith("{{$host}}")
|
|
179
|
-
) {
|
|
180
|
-
nestedValue = `{{$host}}${nestedValue}`;
|
|
181
|
-
}
|
|
182
|
-
} else if (nestedKey === "publication_date") {
|
|
183
|
-
nestedValue =
|
|
184
|
-
new Date(nestedValue)
|
|
185
|
-
.toISOString()
|
|
186
|
-
.split(".")[0] + "Z";
|
|
187
|
-
} else {
|
|
188
|
-
nestedValue = this.encodeXML(nestedValue);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
entry += `\t\t</${key}:${key}>\n`;
|
|
196
|
-
}
|
|
197
|
-
} else {
|
|
198
|
-
if (key === "loc") {
|
|
199
|
-
if (
|
|
200
|
-
!value.startsWith("https://") &&
|
|
201
|
-
!value.startsWith("http://")
|
|
202
|
-
) {
|
|
203
|
-
value = `{{$host}}${value}`;
|
|
204
|
-
}
|
|
205
|
-
} else if (key === "lastmod") {
|
|
206
|
-
value =
|
|
207
|
-
new Date(file.modified.on).toISOString().split(".")[0] +
|
|
208
|
-
"Z";
|
|
209
|
-
} else {
|
|
210
|
-
value = this.encodeXML(value);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
entry += `\t\t<${key}>${value}</${key}>\n`;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
entry += `\t</url>\n`;
|
|
218
|
-
|
|
219
|
-
return entry;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
async getSitemap(file, host) {
|
|
223
|
-
let mainSitemap = {
|
|
224
|
-
host: file.host,
|
|
225
|
-
name: "sitemap.xml",
|
|
226
|
-
path: "/",
|
|
227
|
-
pathname: "/sitemap.xml",
|
|
228
|
-
directory: "/",
|
|
229
|
-
"content-type": "application/xml",
|
|
230
|
-
public: true,
|
|
231
|
-
organization_id: file.organization_id
|
|
232
|
-
};
|
|
233
|
-
|
|
234
|
-
mainSitemap = await this.readSitemap(mainSitemap, host);
|
|
235
|
-
if (!mainSitemap.src) mainSitemap.src = this.createSitemap("main");
|
|
236
|
-
|
|
237
|
-
let sitemap = {
|
|
238
|
-
host: file.host,
|
|
239
|
-
path: "/",
|
|
240
|
-
pathname: file.sitemap.pathname,
|
|
241
|
-
directory: "/",
|
|
242
|
-
"content-type": "application/xml",
|
|
243
|
-
public: true,
|
|
244
|
-
organization_id: file.organization_id
|
|
245
|
-
};
|
|
246
|
-
|
|
247
|
-
// Update loc using pathname
|
|
248
|
-
file.sitemap.loc = `${file.pathname}`;
|
|
249
|
-
|
|
250
|
-
// Query the database for the correct sitemap based on the loc and type
|
|
251
|
-
if (file.sitemap.pathname) {
|
|
252
|
-
sitemap = await this.readSitemap(sitemap, host);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
if (!sitemap.src) {
|
|
256
|
-
let type = "sitemap";
|
|
257
|
-
|
|
258
|
-
// Identify content type to determine sitemap type
|
|
259
|
-
if (file["content-type"].startsWith("image/")) {
|
|
260
|
-
type = "image";
|
|
261
|
-
} else if (file["content-type"].startsWith("video/")) {
|
|
262
|
-
type = "video";
|
|
263
|
-
} else if (file.sitemap.type === "news") {
|
|
264
|
-
type = "news";
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
let name = `sitemap`;
|
|
268
|
-
if (type === "image" || type === "video" || type === "news")
|
|
269
|
-
name = `sitemap-${type}`;
|
|
270
|
-
|
|
271
|
-
// If no existing sitemap found check last index sitemap
|
|
272
|
-
let index = await this.getLastSitemapIndex(mainSitemap, name);
|
|
273
|
-
if (index) {
|
|
274
|
-
sitemap.pathname = `/${name}${index}.xml`;
|
|
275
|
-
sitemap = await this.readSitemap(sitemap, host);
|
|
276
|
-
} else {
|
|
277
|
-
index = 1;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
// Check if there's room in the last index sitemap
|
|
281
|
-
if (!this.checkSitemap(sitemap.src)) {
|
|
282
|
-
if (sitemap.src) index += 1;
|
|
283
|
-
else sitemap.src = this.createSitemap(type);
|
|
284
|
-
|
|
285
|
-
sitemap.name = `${name}${index}.xml`;
|
|
286
|
-
sitemap.pathname = `/${name}${index}.xml`;
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
// Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
|
|
291
|
-
const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
|
|
292
|
-
|
|
293
|
-
// Execute the regex match against the sitemap index source
|
|
294
|
-
const match = mainSitemap.src.match(new RegExp(regexPattern));
|
|
295
|
-
|
|
296
|
-
// Check if a match is found
|
|
297
|
-
if (!match) {
|
|
298
|
-
//TODO: if sitemap found but not in index should we add to sitemap pathname to index or should we check the sitmap for the next index available see if room add or create new index.
|
|
299
|
-
const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
|
|
300
|
-
mainSitemap.src = mainSitemap.src.replace(
|
|
301
|
-
"</sitemapindex>",
|
|
302
|
-
`${indexEntry}\n</sitemapindex>`
|
|
303
|
-
);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
return { mainSitemap, sitemap };
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
async readSitemap(file, host) {
|
|
310
|
-
let data = {
|
|
311
|
-
method: "object.read",
|
|
312
|
-
host: host,
|
|
313
|
-
array: "files",
|
|
314
|
-
$filter: {
|
|
315
|
-
query: {
|
|
316
|
-
host: { $in: [host, "*"] },
|
|
317
|
-
pathname: file.pathname
|
|
318
|
-
},
|
|
319
|
-
limit: 1
|
|
320
|
-
},
|
|
321
|
-
organization_id: file.organization_id
|
|
322
|
-
};
|
|
323
|
-
data = await this.crud.send(data);
|
|
324
|
-
if (data.object && data.object.length) return data.object[0];
|
|
325
|
-
else return file;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
createSitemap(type) {
|
|
329
|
-
const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
|
|
330
|
-
const sitemapNamespace =
|
|
331
|
-
'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
|
|
332
|
-
|
|
333
|
-
if (type === "main") {
|
|
334
|
-
return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
|
|
335
|
-
} else {
|
|
336
|
-
const imageNamespace =
|
|
337
|
-
'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
|
|
338
|
-
const videoNamespace =
|
|
339
|
-
'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
|
|
340
|
-
const newsNamespace =
|
|
341
|
-
'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
|
|
342
|
-
|
|
343
|
-
if (type === "image") {
|
|
344
|
-
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
|
|
345
|
-
} else if (type === "video") {
|
|
346
|
-
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
|
|
347
|
-
} else {
|
|
348
|
-
// For 'news' type or any other types
|
|
349
|
-
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
async saveSitemap(file, host) {
|
|
355
|
-
let data = {
|
|
356
|
-
method: "object.update",
|
|
357
|
-
host: host,
|
|
358
|
-
array: "files",
|
|
359
|
-
object: file,
|
|
360
|
-
upsert: true,
|
|
361
|
-
organization_id: file.organization_id
|
|
362
|
-
};
|
|
363
|
-
if (!file._id)
|
|
364
|
-
data.$filter = {
|
|
365
|
-
query: {
|
|
366
|
-
host: { $in: [host, "*"] },
|
|
367
|
-
pathname: file.pathname
|
|
368
|
-
},
|
|
369
|
-
limit: 1
|
|
370
|
-
};
|
|
371
|
-
|
|
372
|
-
data = await this.crud.send(data);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
async getLastSitemapIndex(mainSitemap, filename) {
|
|
376
|
-
try {
|
|
377
|
-
// Use regex to match all sitemap entries for the given type
|
|
378
|
-
const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, "g");
|
|
379
|
-
const matches = mainSitemap.src.match(regex);
|
|
380
|
-
|
|
381
|
-
return matches ? matches.length : null;
|
|
382
|
-
} catch (err) {
|
|
383
|
-
console.error(
|
|
384
|
-
`Error determining next sitemap index for ${filename}:`,
|
|
385
|
-
err
|
|
386
|
-
);
|
|
387
|
-
return null; // Or some default value or throw an error
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
checkSitemap(sitemap) {
|
|
392
|
-
try {
|
|
393
|
-
if (!sitemap) return false;
|
|
394
|
-
|
|
395
|
-
// Count the number of <url> entries
|
|
396
|
-
const urlCount = (sitemap.match(/<url>/g) || []).length;
|
|
397
|
-
if (urlCount >= 50000) return false;
|
|
398
|
-
|
|
399
|
-
// Get the size of the sitemap string in bytes
|
|
400
|
-
const fileSizeInBytes = Buffer.byteLength(sitemap, "utf8");
|
|
401
|
-
const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
|
|
402
|
-
|
|
403
|
-
// console.log(`Sitemap has ${urlCount} entries and is ${fileSizeInMB.toFixed(2)} MB.`);
|
|
404
|
-
|
|
405
|
-
// Check if the file size exceeds either the 50MB limit or the MongoDB 16MB limit
|
|
406
|
-
if (fileSizeInMB >= 50 || fileSizeInMB >= 15) return false;
|
|
407
|
-
|
|
408
|
-
return true;
|
|
409
|
-
} catch (err) {
|
|
410
|
-
console.error("Error checking sitemap file:", err);
|
|
411
|
-
return false;
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
parseHtml(file) {
|
|
416
|
-
const dom = parse(file.src);
|
|
417
|
-
const entries = dom.querySelectorAll('[sitemap="true"]');
|
|
418
|
-
|
|
419
|
-
let types = ["image", "video", "news"];
|
|
420
|
-
|
|
421
|
-
const previousEntries = {};
|
|
422
|
-
for (let i = 0; i < types.length; i++) {
|
|
423
|
-
if (!file.sitemap[types[i]]) continue;
|
|
424
|
-
if (Array.isArray(file.sitemap[types[i]]))
|
|
425
|
-
previousEntries[types[i]] = file.sitemap[types[i]];
|
|
426
|
-
else previousEntries[types[i]] = [file.sitemap[types[i]]];
|
|
427
|
-
|
|
428
|
-
delete file.sitemap[types[i]];
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
for (let i = 0; i < entries.length; i++) {
|
|
432
|
-
let type = "",
|
|
433
|
-
query = "";
|
|
434
|
-
let existingObject;
|
|
435
|
-
let entryObject = {};
|
|
436
|
-
|
|
437
|
-
if (entries[i].tagName === "IMG") {
|
|
438
|
-
// Corrected to 'IMG' for images
|
|
439
|
-
type = "image";
|
|
440
|
-
query = "loc";
|
|
441
|
-
entryObject.loc = entries[i].getAttribute("src");
|
|
442
|
-
entryObject.title =
|
|
443
|
-
entries[i].getAttribute("sitemap-title") ||
|
|
444
|
-
entries[i].getAttribute("title") ||
|
|
445
|
-
entries[i].getAttribute("alt");
|
|
446
|
-
entryObject.caption =
|
|
447
|
-
entries[i].getAttribute("sitemap-caption") ||
|
|
448
|
-
entries[i].getAttribute("alt") ||
|
|
449
|
-
entryObject.title;
|
|
450
|
-
entryObject.geo_location = entries[i].getAttribute(
|
|
451
|
-
"sitemap-geo-location"
|
|
452
|
-
);
|
|
453
|
-
} else if (entries[i].tagName === "VIDEO") {
|
|
454
|
-
type = "video";
|
|
455
|
-
query = "content_loc";
|
|
456
|
-
entryObject.content_loc = entries[i].src;
|
|
457
|
-
entryObject.title =
|
|
458
|
-
entries[i].getAttribute("sitemap-title") ||
|
|
459
|
-
entries[i].getAttribute("title");
|
|
460
|
-
entryObject.description =
|
|
461
|
-
entries[i].getAttribute("description"); // 'description' if available
|
|
462
|
-
entryObject.thumbnail_loc =
|
|
463
|
-
entries[i].getAttribute("sitemap-thumbnail") ||
|
|
464
|
-
entries[i].getAttribute("poster");
|
|
465
|
-
entryObject.duration =
|
|
466
|
-
entries[i].getAttribute("sitemap-duration");
|
|
467
|
-
} else {
|
|
468
|
-
type = "news";
|
|
469
|
-
file.sitemap.type = "news";
|
|
470
|
-
query = "title";
|
|
471
|
-
entryObject.title = entries[i].getAttribute("sitemap-title");
|
|
472
|
-
if (!entryObject.title) {
|
|
473
|
-
const title = dom.querySelector("title");
|
|
474
|
-
entryObject.title = title ? title.text : "";
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
entryObject.publication = {
|
|
478
|
-
name: entries[i].getAttribute("sitemap-publication-name"), // Use proper attribute
|
|
479
|
-
language: entries[i].getAttribute(
|
|
480
|
-
"sitemap-publication-language"
|
|
481
|
-
) // Use proper attribute
|
|
482
|
-
};
|
|
483
|
-
|
|
484
|
-
if (!entryObject.publication.language) {
|
|
485
|
-
// Fallback to HTML lang attribute
|
|
486
|
-
const htmlElement = dom.querySelector("html");
|
|
487
|
-
entryObject.publication.language = htmlElement
|
|
488
|
-
? htmlElement.getAttribute("lang")
|
|
489
|
-
: null;
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
entryObject.publication_date =
|
|
493
|
-
entries[i].getAttribute("sitemap-publication-date") ||
|
|
494
|
-
file.modified.on;
|
|
495
|
-
|
|
496
|
-
entryObject.keywords =
|
|
497
|
-
entries[i].getAttribute("sitemap-keywords");
|
|
498
|
-
if (!entryObject.keywords) {
|
|
499
|
-
const keywords = dom.querySelector('meta[name="keywords"]');
|
|
500
|
-
entryObject.keywords = keywords
|
|
501
|
-
? keywords.getAttribute("content")
|
|
502
|
-
: "";
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
entryObject.genres = entries[i].getAttribute("sitemap-genres");
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
if (previousEntries[type]) {
|
|
509
|
-
existingObject = previousEntries[type].find(
|
|
510
|
-
(item) => item[query] === entryObject[query]
|
|
511
|
-
);
|
|
512
|
-
entryObject = { ...existingObject, ...entryObject };
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
Object.keys(entryObject).forEach((key) => {
|
|
516
|
-
if (!entryObject[key]) delete entryObject[key];
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
if (!file.sitemap[type]) file.sitemap[type] = [];
|
|
520
|
-
|
|
521
|
-
file.sitemap[type].push(entryObject);
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
if (
|
|
525
|
-
file.sitemap.type !== "news" &&
|
|
526
|
-
file.sitemap.type !== "image" &&
|
|
527
|
-
file.sitemap.type !== "video"
|
|
528
|
-
) {
|
|
529
|
-
const priorityMeta = dom.querySelector(
|
|
530
|
-
'meta[name="sitemap-priority"]'
|
|
531
|
-
);
|
|
532
|
-
const changefreqMeta = dom.querySelector(
|
|
533
|
-
'meta[name="sitemap-changefreq"]'
|
|
534
|
-
);
|
|
535
|
-
file.sitemap.priority = priorityMeta
|
|
536
|
-
? priorityMeta.getAttribute("content")
|
|
537
|
-
: file.sitemap.priority; // Default priority if not specified
|
|
538
|
-
file.sitemap.changefreq = changefreqMeta
|
|
539
|
-
? changefreqMeta.getAttribute("content")
|
|
540
|
-
: file.sitemap.changefreq || "monthly"; // Default changefreq if not specified
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
encodeXML(str) {
|
|
545
|
-
if (str)
|
|
546
|
-
return str
|
|
547
|
-
.replace(/&/g, "&")
|
|
548
|
-
.replace(/</g, "<")
|
|
549
|
-
.replace(/>/g, ">")
|
|
550
|
-
.replace(/"/g, """)
|
|
551
|
-
.replace(/'/g, "'");
|
|
552
|
-
}
|
|
1
|
+
import { parse } from "node-html-parser";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Clean, stateless functional entry point to check and verify a file against the sitemap.
|
|
5
|
+
* All dependencies are passed explicitly via the parameters object.
|
|
6
|
+
* * @param {Object} options
|
|
7
|
+
* @param {Object} options.file - The file object being verified
|
|
8
|
+
* @param {string} options.host - The active website hostname
|
|
9
|
+
* @param {Object} options.crud - Crud database instance
|
|
10
|
+
*/
|
|
11
|
+
export async function check({ file, host, crud }) {
|
|
12
|
+
// Ensure the file is HTML and does not have a sitemap object yet
|
|
13
|
+
if (!file.sitemap && file["content-type"] !== "text/html") return;
|
|
14
|
+
|
|
15
|
+
// Ensure the file is public and is not sitemap false
|
|
16
|
+
if (
|
|
17
|
+
file.sitemap === false ||
|
|
18
|
+
file.sitemap === "false" ||
|
|
19
|
+
!file.public ||
|
|
20
|
+
file.public === "false"
|
|
21
|
+
)
|
|
22
|
+
return;
|
|
23
|
+
|
|
24
|
+
// Check if the file is HTML and contains a noindex meta or title tag
|
|
25
|
+
if (file["content-type"] === "text/html") {
|
|
26
|
+
if (
|
|
27
|
+
/<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(
|
|
28
|
+
file.src
|
|
29
|
+
)
|
|
30
|
+
)
|
|
31
|
+
return;
|
|
32
|
+
if (!/<title[^>]*>[\s\S]*?<\/title>/i.test(file.src)) return;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Compare the lastmod date in the sitemap with the modified.on date
|
|
36
|
+
if (file.sitemap && file.sitemap.lastmod && file.modified?.on) {
|
|
37
|
+
if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Logic to update the sitemap asynchronously
|
|
42
|
+
await updateSitemap({ file, host, crud });
|
|
553
43
|
}
|
|
554
44
|
|
|
555
|
-
|
|
45
|
+
// ==========================================
|
|
46
|
+
// Stateless Private Helper Functions
|
|
47
|
+
// ==========================================
|
|
48
|
+
|
|
49
|
+
async function updateSitemap({ file, host, crud }) {
|
|
50
|
+
try {
|
|
51
|
+
if (!file.sitemap) file.sitemap = {};
|
|
52
|
+
|
|
53
|
+
const entry = createEntry(file);
|
|
54
|
+
|
|
55
|
+
let { mainSitemap, sitemap } = await getSitemap({ file, host, crud });
|
|
56
|
+
|
|
57
|
+
if (file.pathname) {
|
|
58
|
+
// Perform regex search starting at the pathname
|
|
59
|
+
const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(
|
|
60
|
+
/[.*+?^${}()|[\]\\]/g,
|
|
61
|
+
"\\$&"
|
|
62
|
+
)}.*?</loc>[\\s\\S]*?</url>`;
|
|
63
|
+
const match = sitemap.src.match(new RegExp(regexPattern));
|
|
64
|
+
|
|
65
|
+
if (match) {
|
|
66
|
+
const position = match.index; // Start position of the <url> block
|
|
67
|
+
const endPosition = match.index + match[0].length; // End position of the <url> block
|
|
68
|
+
|
|
69
|
+
// Replace the original <url> block with the modified one
|
|
70
|
+
sitemap.src =
|
|
71
|
+
sitemap.src.slice(0, position) +
|
|
72
|
+
entry +
|
|
73
|
+
sitemap.src.slice(endPosition);
|
|
74
|
+
} else {
|
|
75
|
+
sitemap.src = sitemap.src.replace(
|
|
76
|
+
"</urlset>",
|
|
77
|
+
`${entry}</urlset>`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
} else {
|
|
81
|
+
file.sitemap.pathname = sitemap.pathname;
|
|
82
|
+
sitemap.src = sitemap.src.replace(
|
|
83
|
+
"</urlset>",
|
|
84
|
+
`${entry}</urlset>`
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
await saveSitemap({ file: mainSitemap, host, crud });
|
|
89
|
+
await saveSitemap({ file: sitemap, host, crud });
|
|
90
|
+
await saveSitemap({ file, host, crud });
|
|
91
|
+
|
|
92
|
+
} catch (err) {
|
|
93
|
+
console.error("Error updating sitemap:", err);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function createEntry(file) {
|
|
98
|
+
file.sitemap.loc = file.pathname;
|
|
99
|
+
file.sitemap.lastmod = file.modified?.on;
|
|
100
|
+
|
|
101
|
+
if (file["content-type"] === "text/html") {
|
|
102
|
+
parseHtml(file);
|
|
103
|
+
|
|
104
|
+
if (
|
|
105
|
+
file.sitemap.type !== "news" &&
|
|
106
|
+
file.sitemap.type !== "image" &&
|
|
107
|
+
file.sitemap.type !== "video"
|
|
108
|
+
) {
|
|
109
|
+
if (!file.sitemap.changefreq)
|
|
110
|
+
file.sitemap.changefreq = "monthly";
|
|
111
|
+
|
|
112
|
+
if (!file.sitemap.priority) {
|
|
113
|
+
const depth = (file.pathname.match(/\//g) || []).length;
|
|
114
|
+
file.sitemap.priority = Math.max(
|
|
115
|
+
0.1,
|
|
116
|
+
1.0 - (depth - 1) * 0.1
|
|
117
|
+
).toFixed(1);
|
|
118
|
+
}
|
|
119
|
+
} else {
|
|
120
|
+
delete file.sitemap.changefreq;
|
|
121
|
+
delete file.sitemap.priority;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let entry = `\t<url>\n`;
|
|
126
|
+
|
|
127
|
+
for (const key of Object.keys(file.sitemap)) {
|
|
128
|
+
if (key === "pathname" || key === "type") continue;
|
|
129
|
+
|
|
130
|
+
let value = file.sitemap[key];
|
|
131
|
+
|
|
132
|
+
if (
|
|
133
|
+
typeof value === "object" &&
|
|
134
|
+
value !== null &&
|
|
135
|
+
!(value instanceof Date)
|
|
136
|
+
) {
|
|
137
|
+
if (!Array.isArray(value)) value = [value];
|
|
138
|
+
|
|
139
|
+
for (let i = 0; i < value.length; i++) {
|
|
140
|
+
entry += `\t\t<${key}:${key}>\n`;
|
|
141
|
+
|
|
142
|
+
for (const nestedKey of Object.keys(value[i])) {
|
|
143
|
+
let nestedValue = value[i][nestedKey];
|
|
144
|
+
// Handle nested objects
|
|
145
|
+
if (
|
|
146
|
+
typeof nestedValue === "object" &&
|
|
147
|
+
nestedValue !== null &&
|
|
148
|
+
!(nestedValue instanceof Date)
|
|
149
|
+
) {
|
|
150
|
+
entry += `\t\t\t<${key}:${nestedKey}>\n`;
|
|
151
|
+
for (const subKey of Object.keys(nestedValue)) {
|
|
152
|
+
const subValue = nestedValue[subKey];
|
|
153
|
+
entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
|
|
154
|
+
}
|
|
155
|
+
entry += `\t\t\t</${key}:${nestedKey}>\n`;
|
|
156
|
+
} else {
|
|
157
|
+
if (nestedKey === "loc") {
|
|
158
|
+
if (
|
|
159
|
+
!nestedValue.startsWith("https://") &&
|
|
160
|
+
!nestedValue.startsWith("http://") &&
|
|
161
|
+
!nestedValue.startsWith("{{$host}}")
|
|
162
|
+
) {
|
|
163
|
+
nestedValue = `{{$host}}${nestedValue}`;
|
|
164
|
+
}
|
|
165
|
+
} else if (nestedKey === "publication_date") {
|
|
166
|
+
nestedValue =
|
|
167
|
+
new Date(nestedValue)
|
|
168
|
+
.toISOString()
|
|
169
|
+
.split(".")[0] + "Z";
|
|
170
|
+
} else {
|
|
171
|
+
nestedValue = encodeXML(nestedValue);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
entry += `\t\t</${key}:${key}>\n`;
|
|
179
|
+
}
|
|
180
|
+
} else {
|
|
181
|
+
if (key === "loc") {
|
|
182
|
+
if (
|
|
183
|
+
!value.startsWith("https://") &&
|
|
184
|
+
!value.startsWith("http://")
|
|
185
|
+
) {
|
|
186
|
+
value = `{{$host}}${value}`;
|
|
187
|
+
}
|
|
188
|
+
} else if (key === "lastmod") {
|
|
189
|
+
value =
|
|
190
|
+
new Date(file.modified?.on).toISOString().split(".")[0] +
|
|
191
|
+
"Z";
|
|
192
|
+
} else {
|
|
193
|
+
value = encodeXML(value);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
entry += `\t\t<${key}>${value}</${key}>\n`;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
entry += `\t</url>\n`;
|
|
201
|
+
|
|
202
|
+
return entry;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async function getSitemap({ file, host, crud }) {
|
|
206
|
+
let mainSitemap = {
|
|
207
|
+
host: file.host,
|
|
208
|
+
name: "sitemap.xml",
|
|
209
|
+
path: "/",
|
|
210
|
+
pathname: "/sitemap.xml",
|
|
211
|
+
directory: "/",
|
|
212
|
+
"content-type": "application/xml",
|
|
213
|
+
public: true,
|
|
214
|
+
organization_id: file.organization_id
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
mainSitemap = await readSitemap({ file: mainSitemap, host, crud });
|
|
218
|
+
if (!mainSitemap.src) mainSitemap.src = generateSitemapXml("main");
|
|
219
|
+
|
|
220
|
+
let sitemap = {
|
|
221
|
+
host: file.host,
|
|
222
|
+
path: "/",
|
|
223
|
+
pathname: file.sitemap.pathname,
|
|
224
|
+
directory: "/",
|
|
225
|
+
"content-type": "application/xml",
|
|
226
|
+
public: true,
|
|
227
|
+
organization_id: file.organization_id
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
// Update loc using pathname
|
|
231
|
+
file.sitemap.loc = `${file.pathname}`;
|
|
232
|
+
|
|
233
|
+
// Query the database for the correct sitemap based on the loc and type
|
|
234
|
+
if (file.sitemap.pathname) {
|
|
235
|
+
sitemap = await readSitemap({ file: sitemap, host, crud });
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (!sitemap.src) {
|
|
239
|
+
let type = "sitemap";
|
|
240
|
+
|
|
241
|
+
// Identify content type to determine sitemap type
|
|
242
|
+
if (file["content-type"].startsWith("image/")) {
|
|
243
|
+
type = "image";
|
|
244
|
+
} else if (file["content-type"].startsWith("video/")) {
|
|
245
|
+
type = "video";
|
|
246
|
+
} else if (file.sitemap.type === "news") {
|
|
247
|
+
type = "news";
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
let name = `sitemap`;
|
|
251
|
+
if (type === "image" || type === "video" || type === "news")
|
|
252
|
+
name = `sitemap-${type}`;
|
|
253
|
+
|
|
254
|
+
// If no existing sitemap found check last index sitemap
|
|
255
|
+
let index = await getLastSitemapIndex(mainSitemap, name);
|
|
256
|
+
if (index) {
|
|
257
|
+
sitemap.pathname = `/${name}${index}.xml`;
|
|
258
|
+
sitemap = await readSitemap({ file: sitemap, host, crud });
|
|
259
|
+
} else {
|
|
260
|
+
index = 1;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Check if there's room in the last index sitemap
|
|
264
|
+
if (!checkSitemapXml(sitemap.src)) {
|
|
265
|
+
if (sitemap.src) index += 1;
|
|
266
|
+
else sitemap.src = generateSitemapXml(type);
|
|
267
|
+
|
|
268
|
+
sitemap.name = `${name}${index}.xml`;
|
|
269
|
+
sitemap.pathname = `/${name}${index}.xml`;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
|
|
274
|
+
const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
|
|
275
|
+
|
|
276
|
+
// Execute the regex match against the sitemap index source
|
|
277
|
+
const match = mainSitemap.src.match(new RegExp(regexPattern));
|
|
278
|
+
|
|
279
|
+
// Check if a match is found
|
|
280
|
+
if (!match) {
|
|
281
|
+
const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
|
|
282
|
+
mainSitemap.src = mainSitemap.src.replace(
|
|
283
|
+
"</sitemapindex>",
|
|
284
|
+
`${indexEntry}\n</sitemapindex>`
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return { mainSitemap, sitemap };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function readSitemap({ file, host, crud }) {
|
|
292
|
+
let data = {
|
|
293
|
+
method: "object.read",
|
|
294
|
+
host: host,
|
|
295
|
+
array: "files",
|
|
296
|
+
$filter: {
|
|
297
|
+
query: {
|
|
298
|
+
host: { $in: [host, "*"] },
|
|
299
|
+
pathname: file.pathname
|
|
300
|
+
},
|
|
301
|
+
limit: 1
|
|
302
|
+
},
|
|
303
|
+
organization_id: file.organization_id
|
|
304
|
+
};
|
|
305
|
+
data = await crud.send(data);
|
|
306
|
+
if (data.object && data.object.length) return data.object[0];
|
|
307
|
+
else return file;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function generateSitemapXml(type) {
|
|
311
|
+
const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
|
|
312
|
+
const sitemapNamespace =
|
|
313
|
+
'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
|
|
314
|
+
|
|
315
|
+
if (type === "main") {
|
|
316
|
+
return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
|
|
317
|
+
} else {
|
|
318
|
+
const imageNamespace =
|
|
319
|
+
'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
|
|
320
|
+
const videoNamespace =
|
|
321
|
+
'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
|
|
322
|
+
const newsNamespace =
|
|
323
|
+
'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
|
|
324
|
+
|
|
325
|
+
if (type === "image") {
|
|
326
|
+
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
|
|
327
|
+
} else if (type === "video") {
|
|
328
|
+
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
|
|
329
|
+
} else {
|
|
330
|
+
// For 'news' type or any other types
|
|
331
|
+
return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
async function saveSitemap({ file, host, crud }) {
|
|
337
|
+
let data = {
|
|
338
|
+
method: "object.update",
|
|
339
|
+
host: host,
|
|
340
|
+
array: "files",
|
|
341
|
+
object: file,
|
|
342
|
+
upsert: true,
|
|
343
|
+
organization_id: file.organization_id
|
|
344
|
+
};
|
|
345
|
+
if (!file._id)
|
|
346
|
+
data.$filter = {
|
|
347
|
+
query: {
|
|
348
|
+
host: { $in: [host, "*"] },
|
|
349
|
+
pathname: file.pathname
|
|
350
|
+
},
|
|
351
|
+
limit: 1
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
await crud.send(data);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async function getLastSitemapIndex(mainSitemap, filename) {
|
|
358
|
+
try {
|
|
359
|
+
// Use regex to match all sitemap entries for the given type
|
|
360
|
+
const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, "g");
|
|
361
|
+
const matches = mainSitemap.src.match(regex);
|
|
362
|
+
|
|
363
|
+
return matches ? matches.length : null;
|
|
364
|
+
} catch (err) {
|
|
365
|
+
console.error(
|
|
366
|
+
`Error determining next sitemap index for ${filename}:`,
|
|
367
|
+
err
|
|
368
|
+
);
|
|
369
|
+
return null;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function checkSitemapXml(sitemap) {
|
|
374
|
+
try {
|
|
375
|
+
if (!sitemap) return false;
|
|
376
|
+
|
|
377
|
+
// Count the number of <url> entries
|
|
378
|
+
const urlCount = (sitemap.match(/<url>/g) || []).length;
|
|
379
|
+
if (urlCount >= 50000) return false;
|
|
380
|
+
|
|
381
|
+
// Get the size of the sitemap string in bytes
|
|
382
|
+
const fileSizeInBytes = Buffer.byteLength(sitemap, "utf8");
|
|
383
|
+
const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
|
|
384
|
+
|
|
385
|
+
// Check if the file size exceeds either the 50MB limit or the MongoDB 16MB limit
|
|
386
|
+
if (fileSizeInMB >= 50 || fileSizeInMB >= 15) return false;
|
|
387
|
+
|
|
388
|
+
return true;
|
|
389
|
+
} catch (err) {
|
|
390
|
+
console.error("Error checking sitemap file:", err);
|
|
391
|
+
return false;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function parseHtml(file) {
|
|
396
|
+
const dom = parse(file.src);
|
|
397
|
+
const entries = dom.querySelectorAll('[sitemap="true"]');
|
|
398
|
+
|
|
399
|
+
let types = ["image", "video", "news"];
|
|
400
|
+
|
|
401
|
+
const previousEntries = {};
|
|
402
|
+
for (let i = 0; i < types.length; i++) {
|
|
403
|
+
if (!file.sitemap[types[i]]) continue;
|
|
404
|
+
if (Array.isArray(file.sitemap[types[i]]))
|
|
405
|
+
previousEntries[types[i]] = file.sitemap[types[i]];
|
|
406
|
+
else previousEntries[types[i]] = [file.sitemap[types[i]]];
|
|
407
|
+
|
|
408
|
+
delete file.sitemap[types[i]];
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
for (let i = 0; i < entries.length; i++) {
|
|
412
|
+
let type = "",
|
|
413
|
+
query = "";
|
|
414
|
+
let existingObject;
|
|
415
|
+
let entryObject = {};
|
|
416
|
+
|
|
417
|
+
if (entries[i].tagName === "IMG") {
|
|
418
|
+
type = "image";
|
|
419
|
+
query = "loc";
|
|
420
|
+
entryObject.loc = entries[i].getAttribute("src");
|
|
421
|
+
entryObject.title =
|
|
422
|
+
entries[i].getAttribute("sitemap-title") ||
|
|
423
|
+
entries[i].getAttribute("title") ||
|
|
424
|
+
entries[i].getAttribute("alt");
|
|
425
|
+
entryObject.caption =
|
|
426
|
+
entries[i].getAttribute("sitemap-caption") ||
|
|
427
|
+
entries[i].getAttribute("alt") ||
|
|
428
|
+
entryObject.title;
|
|
429
|
+
entryObject.geo_location = entries[i].getAttribute(
|
|
430
|
+
"sitemap-geo-location"
|
|
431
|
+
);
|
|
432
|
+
} else if (entries[i].tagName === "VIDEO") {
|
|
433
|
+
type = "video";
|
|
434
|
+
query = "content_loc";
|
|
435
|
+
entryObject.content_loc = entries[i].src;
|
|
436
|
+
entryObject.title =
|
|
437
|
+
entries[i].getAttribute("sitemap-title") ||
|
|
438
|
+
entries[i].getAttribute("title");
|
|
439
|
+
entryObject.description =
|
|
440
|
+
entries[i].getAttribute("description");
|
|
441
|
+
entryObject.thumbnail_loc =
|
|
442
|
+
entries[i].getAttribute("sitemap-thumbnail") ||
|
|
443
|
+
entries[i].getAttribute("poster");
|
|
444
|
+
entryObject.duration =
|
|
445
|
+
entries[i].getAttribute("sitemap-duration");
|
|
446
|
+
} else {
|
|
447
|
+
type = "news";
|
|
448
|
+
file.sitemap.type = "news";
|
|
449
|
+
query = "title";
|
|
450
|
+
entryObject.title = entries[i].getAttribute("sitemap-title");
|
|
451
|
+
if (!entryObject.title) {
|
|
452
|
+
const title = dom.querySelector("title");
|
|
453
|
+
entryObject.title = title ? title.text : "";
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
entryObject.publication = {
|
|
457
|
+
name: entries[i].getAttribute("sitemap-publication-name"),
|
|
458
|
+
language: entries[i].getAttribute(
|
|
459
|
+
"sitemap-publication-language"
|
|
460
|
+
)
|
|
461
|
+
};
|
|
462
|
+
|
|
463
|
+
if (!entryObject.publication.language) {
|
|
464
|
+
const htmlElement = dom.querySelector("html");
|
|
465
|
+
entryObject.publication.language = htmlElement
|
|
466
|
+
? htmlElement.getAttribute("lang")
|
|
467
|
+
: null;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
entryObject.publication_date =
|
|
471
|
+
entries[i].getAttribute("sitemap-publication-date") ||
|
|
472
|
+
file.modified?.on;
|
|
473
|
+
|
|
474
|
+
entryObject.keywords =
|
|
475
|
+
entries[i].getAttribute("sitemap-keywords");
|
|
476
|
+
if (!entryObject.keywords) {
|
|
477
|
+
const keywords = dom.querySelector('meta[name="keywords"]');
|
|
478
|
+
entryObject.keywords = keywords
|
|
479
|
+
? keywords.getAttribute("content")
|
|
480
|
+
: "";
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
entryObject.genres = entries[i].getAttribute("sitemap-genres");
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
if (previousEntries[type]) {
|
|
487
|
+
existingObject = previousEntries[type].find(
|
|
488
|
+
(item) => item[query] === entryObject[query]
|
|
489
|
+
);
|
|
490
|
+
entryObject = { ...existingObject, ...entryObject };
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
Object.keys(entryObject).forEach((key) => {
|
|
494
|
+
if (!entryObject[key]) delete entryObject[key];
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
if (!file.sitemap[type]) file.sitemap[type] = [];
|
|
498
|
+
|
|
499
|
+
file.sitemap[type].push(entryObject);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
if (
|
|
503
|
+
file.sitemap.type !== "news" &&
|
|
504
|
+
file.sitemap.type !== "image" &&
|
|
505
|
+
file.sitemap.type !== "video"
|
|
506
|
+
) {
|
|
507
|
+
const priorityMeta = dom.querySelector(
|
|
508
|
+
'meta[name="sitemap-priority"]'
|
|
509
|
+
);
|
|
510
|
+
const changefreqMeta = dom.querySelector(
|
|
511
|
+
'meta[name="sitemap-changefreq"]'
|
|
512
|
+
);
|
|
513
|
+
file.sitemap.priority = priorityMeta
|
|
514
|
+
? priorityMeta.getAttribute("content")
|
|
515
|
+
: file.sitemap.priority;
|
|
516
|
+
file.sitemap.changefreq = changefreqMeta
|
|
517
|
+
? changefreqMeta.getAttribute("content")
|
|
518
|
+
: file.sitemap.changefreq || "monthly";
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function encodeXML(str) {
|
|
523
|
+
if (str)
|
|
524
|
+
return str
|
|
525
|
+
.replace(/&/g, "&")
|
|
526
|
+
.replace(/</g, "<")
|
|
527
|
+
.replace(/>/g, ">")
|
|
528
|
+
.replace(/"/g, """)
|
|
529
|
+
.replace(/'/g, "'");
|
|
530
|
+
}
|