@cocreate/sitemap 1.4.0 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/docs/index.html +216 -216
  3. package/package.json +1 -9
  4. package/src/index.js +527 -451
package/src/index.js CHANGED
@@ -23,457 +23,533 @@
23
23
  const { parse } = require("node-html-parser");
24
24
 
25
25
  class CoCreateSitemap {
26
- constructor(crud) {
27
- this.crud = crud;
28
- }
29
-
30
- async check(file, host) {
31
- // Ensure the file is HTML and does not have a sitemap object yet
32
- if (!file.sitemap && file['content-type'] !== 'text/html')
33
- return;
34
-
35
- // Ensure the file is public and is not sitemap false
36
- if (file.sitemap === false || file.sitemap === 'false' || !file.public || file.public === "false")
37
- return;
38
-
39
- // Check if the file is HTML and contains a noindex meta or title tag
40
- if (file['content-type'] === 'text/html') {
41
- if (/<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(file.src))
42
- return;
43
- if (!(/<title[^>]*>[\s\S]*?<\/title>/i.test(file.src)))
44
- return;
45
- }
46
-
47
- // Compare the lastmod date in the sitemap with the modified.on date
48
- if (file.sitemap && file.sitemap.lastmod && file.modified.on) {
49
- if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
50
- return;
51
- }
52
-
53
- // Logic to update the sitemap
54
- this.updateSitemap(file, host);
55
- }
56
-
57
- async updateSitemap(file, host) {
58
- try {
59
- if (!file.sitemap)
60
- file.sitemap = {}
61
-
62
- const entry = this.createEntry(file, host);
63
-
64
- let { mainSitemap, sitemap } = await this.getSitemap(file, host);
65
-
66
- if (file.pathname) {
67
- // Perform regex search starting at the pathname
68
- const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*?</loc>[\\s\\S]*?</url>`;
69
- const match = sitemap.src.match(new RegExp(regexPattern));
70
-
71
- if (match) {
72
- const position = match.index; // Start position of the <url> block
73
- const endPosition = match.index + match[0].length; // End position of the <url> block
74
-
75
- // Replace the original <url> block with the modified one
76
- sitemap.src = sitemap.src.slice(0, position) + entry + sitemap.src.slice(endPosition);
77
- } else {
78
- sitemap.src = sitemap.src.replace('</urlset>', `${entry}</urlset>`);
79
- }
80
- } else {
81
- file.sitemap.pathname = sitemap.pathname
82
- sitemap.src = sitemap.src.replace('</urlset>', `${entry}</urlset>`);
83
- }
84
-
85
- this.saveSitemap(mainSitemap, host);
86
- this.saveSitemap(sitemap, host);
87
- this.saveSitemap(file, host);
88
-
89
- // console.log('Sitemap updated successfully.');
90
- } catch (err) {
91
- console.error('Error updating sitemap:', err);
92
- }
93
- }
94
-
95
- createEntry(file) {
96
- file.sitemap.loc = file.pathname;
97
- file.sitemap.lastmod = file.modified.on;
98
-
99
- if (file['content-type'] === 'text/html') {
100
- this.parseHtml(file)
101
-
102
- if (file.sitemap.type !== 'news' && file.sitemap.type !== 'image' && file.sitemap.type !== 'video') {
103
- if (!file.sitemap.changefreq)
104
- file.sitemap.changefreq = 'monthly';
105
-
106
- if (!file.sitemap.priority) {
107
- const depth = (file.pathname.match(/\//g) || []).length;
108
- file.sitemap.priority = Math.max(0.1, 1.0 - (depth - 1) * 0.1).toFixed(1);
109
- }
110
- } else {
111
- delete file.sitemap.changefreq
112
- delete file.sitemap.priority
113
- }
114
- }
115
-
116
- let entry = `\t<url>\n`;
117
-
118
- for (const key of Object.keys(file.sitemap)) {
119
- if (key === 'pathname' || key === 'type')
120
- continue
121
-
122
- let value = file.sitemap[key];
123
-
124
- if (typeof value === 'object' && value !== null && !(value instanceof Date)) {
125
- if (!Array.isArray(value))
126
- value = [value]
127
-
128
- for (let i = 0; i < value.length; i++) {
129
- entry += `\t\t<${key}:${key}>\n`;
130
-
131
- for (const nestedKey of Object.keys(value[i])) {
132
- let nestedValue = value[i][nestedKey];
133
- // Handle nested objects
134
- if (typeof nestedValue === 'object' && nestedValue !== null && !(nestedValue instanceof Date)) {
135
- entry += `\t\t\t<${key}:${nestedKey}>\n`;
136
- for (const subKey of Object.keys(nestedValue)) {
137
- const subValue = nestedValue[subKey];
138
- entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
139
- }
140
- entry += `\t\t\t</${key}:${nestedKey}>\n`;
141
- } else {
142
- if (nestedKey === 'loc') {
143
- if (!nestedValue.startsWith('https://') && !nestedValue.startsWith('http://') && !nestedValue.startsWith('{{$host}}')) {
144
- nestedValue = `{{$host}}${nestedValue}`;
145
- }
146
- } else if (nestedKey === 'publication_date') {
147
- nestedValue = new Date(nestedValue).toISOString().split('.')[0] + "Z";
148
- } else {
149
- nestedValue = this.encodeXML(nestedValue)
150
- }
151
-
152
- entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
153
- }
154
- }
155
-
156
- entry += `\t\t</${key}:${key}>\n`;
157
- }
158
- } else {
159
- if (key === 'loc') {
160
- if (!value.startsWith('https://') && !value.startsWith('http://')) {
161
- value = `{{$host}}${value}`;
162
- }
163
- } else if (key === 'lastmod') {
164
- value = new Date(file.modified.on).toISOString().split('.')[0] + "Z";
165
- } else {
166
- value = this.encodeXML(value)
167
- }
168
-
169
- entry += `\t\t<${key}>${value}</${key}>\n`;
170
- }
171
- }
172
-
173
- entry += `\t</url>\n`;
174
-
175
- return entry;
176
- }
177
-
178
- async getSitemap(file, host) {
179
- let mainSitemap = {
180
- host: file.host,
181
- name: 'sitemap.xml',
182
- path: '/',
183
- pathname: '/sitemap.xml',
184
- directory: '/',
185
- 'content-type': 'application/xml',
186
- public: true,
187
- organization_id: file.organization_id
188
- }
189
-
190
- mainSitemap = await this.readSitemap(mainSitemap, host);
191
- if (!mainSitemap.src)
192
- mainSitemap.src = this.createSitemap('main')
193
-
194
- let sitemap = {
195
- host: file.host,
196
- path: '/',
197
- pathname: file.sitemap.pathname,
198
- directory: '/',
199
- 'content-type': 'application/xml',
200
- public: true,
201
- organization_id: file.organization_id
202
- }
203
-
204
- // Update loc using pathname
205
- file.sitemap.loc = `${file.pathname}`
206
-
207
- // Query the database for the correct sitemap based on the loc and type
208
- if (file.sitemap.pathname) {
209
- sitemap = await this.readSitemap(sitemap, host);
210
- }
211
-
212
- if (!sitemap.src) {
213
- let type = 'sitemap';
214
-
215
- // Identify content type to determine sitemap type
216
- if (file['content-type'].startsWith('image/')) {
217
- type = 'image';
218
- } else if (file['content-type'].startsWith('video/')) {
219
- type = 'video';
220
- } else if (file.sitemap.type === 'news') {
221
- type = 'news';
222
- }
223
-
224
- let name = `sitemap`
225
- if (type === 'image' || type === 'video' || type === 'news')
226
- name = `sitemap-${type}`
227
-
228
- // If no existing sitemap found check last index sitemap
229
- let index = await this.getLastSitemapIndex(mainSitemap, name);
230
- if (index) {
231
- sitemap.pathname = `/${name}${index}.xml`
232
- sitemap = await this.readSitemap(sitemap, host);
233
- } else {
234
- index = 1
235
- }
236
-
237
- // Check if there's room in the last index sitemap
238
- if (!this.checkSitemap(sitemap.src)) {
239
- if (sitemap.src)
240
- index += 1
241
- else
242
- sitemap.src = this.createSitemap(type);
243
-
244
- sitemap.name = `${name}${index}.xml`
245
- sitemap.pathname = `/${name}${index}.xml`
246
-
247
- }
248
-
249
- }
250
-
251
- // Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
252
- const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
253
-
254
- // Execute the regex match against the sitemap index source
255
- const match = mainSitemap.src.match(new RegExp(regexPattern));
256
-
257
- // Check if a match is found
258
- if (!match) {
259
- //TODO: if sitemap found but not in index should we add to sitemap pathname to index or should we check the sitmap for the next index available see if room add or create new index.
260
- const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
261
- mainSitemap.src = mainSitemap.src.replace('</sitemapindex>', `${indexEntry}\n</sitemapindex>`);
262
- }
263
-
264
- return { mainSitemap, sitemap }
265
- }
266
-
267
- async readSitemap(file, host) {
268
- let data = {
269
- method: 'object.read',
270
- host: host,
271
- array: 'files',
272
- $filter: {
273
- query: {
274
- host: { $in: [host, '*'] },
275
- pathname: file.pathname
276
- },
277
- limit: 1
278
- },
279
- organization_id: file.organization_id
280
- }
281
- data = await this.crud.send(data)
282
- if (data.object && data.object.length)
283
- return data.object[0]
284
- else
285
- return file
286
- }
287
-
288
- createSitemap(type) {
289
- const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
290
- const sitemapNamespace = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
291
-
292
- if (type === 'main') {
293
- return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
294
- } else {
295
- const imageNamespace = 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
296
- const videoNamespace = 'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
297
- const newsNamespace = 'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
298
-
299
- if (type === 'image') {
300
- return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
301
- } else if (type === 'video') {
302
- return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
303
- } else { // For 'news' type or any other types
304
- return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
305
- }
306
- }
307
- }
308
-
309
- async saveSitemap(file, host) {
310
- let data = {
311
- method: 'object.update',
312
- host: host,
313
- array: 'files',
314
- object: file,
315
- upsert: true,
316
- organization_id: file.organization_id
317
- }
318
- if (!file._id)
319
- data.$filter = {
320
- query: {
321
- host: { $in: [host, '*'] },
322
- pathname: file.pathname
323
- },
324
- limit: 1
325
- }
326
-
327
- data = await this.crud.send(data)
328
-
329
- }
330
-
331
- async getLastSitemapIndex(mainSitemap, filename) {
332
- try {
333
- // Use regex to match all sitemap entries for the given type
334
- const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, 'g');
335
- const matches = mainSitemap.src.match(regex);
336
-
337
- return matches ? matches.length : null;
338
- } catch (err) {
339
- console.error(`Error determining next sitemap index for ${filename}:`, err);
340
- return null; // Or some default value or throw an error
341
- }
342
- }
343
-
344
- checkSitemap(sitemap) {
345
- try {
346
- if (!sitemap)
347
- return false
348
-
349
- // Count the number of <url> entries
350
- const urlCount = (sitemap.match(/<url>/g) || []).length;
351
- if (urlCount >= 50000)
352
- return false;
353
-
354
- // Get the size of the sitemap string in bytes
355
- const fileSizeInBytes = Buffer.byteLength(sitemap, 'utf8');
356
- const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
357
-
358
- // console.log(`Sitemap has ${urlCount} entries and is ${fileSizeInMB.toFixed(2)} MB.`);
359
-
360
- // Check if the file size exceeds either the 50MB limit or the MongoDB 16MB limit
361
- if (fileSizeInMB >= 50 || fileSizeInMB >= 15)
362
- return false;
363
-
364
- return true;
365
- } catch (err) {
366
- console.error('Error checking sitemap file:', err);
367
- return false;
368
- }
369
- }
370
-
371
- parseHtml(file) {
372
- const dom = parse(file.src);
373
- const entries = dom.querySelectorAll('[sitemap="true"]');
374
-
375
- let types = ['image', 'video', 'news']
376
-
377
- const previousEntries = {}
378
- for (let i = 0; i < types.length; i++) {
379
- if (!file.sitemap[types[i]])
380
- continue
381
- if (Array.isArray(file.sitemap[types[i]]))
382
- previousEntries[types[i]] = file.sitemap[types[i]]
383
- else
384
- previousEntries[types[i]] = [file.sitemap[types[i]]]
385
-
386
- delete file.sitemap[types[i]]
387
- }
388
-
389
- for (let i = 0; i < entries.length; i++) {
390
- let type = '', query = '';
391
- let existingObject
392
- let entryObject = {};
393
-
394
- if (entries[i].tagName === 'IMG') { // Corrected to 'IMG' for images
395
- type = 'image';
396
- query = 'loc'
397
- entryObject.loc = entries[i].getAttribute('src');
398
- entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title') || entries[i].getAttribute('alt');
399
- entryObject.caption = entries[i].getAttribute('sitemap-caption') || entries[i].getAttribute('alt') || entryObject.title;
400
- entryObject.geo_location = entries[i].getAttribute('sitemap-geo-location');
401
- } else if (entries[i].tagName === 'VIDEO') {
402
- type = 'video';
403
- query = 'content_loc'
404
- entryObject.content_loc = entries[i].src;
405
- entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title');
406
- entryObject.description = entries[i].getAttribute('description'); // 'description' if available
407
- entryObject.thumbnail_loc = entries[i].getAttribute('sitemap-thumbnail') || entries[i].getAttribute('poster');
408
- entryObject.duration = entries[i].getAttribute('sitemap-duration');
409
- } else {
410
- type = 'news';
411
- file.sitemap.type = 'news';
412
- query = 'title'
413
- entryObject.title = entries[i].getAttribute('sitemap-title');
414
- if (!entryObject.title) {
415
- const title = dom.querySelector('title');
416
- entryObject.title = title ? title.text : '';
417
- }
418
-
419
- entryObject.publication = {
420
- name: entries[i].getAttribute('sitemap-publication-name'), // Use proper attribute
421
- language: entries[i].getAttribute('sitemap-publication-language') // Use proper attribute
422
- };
423
-
424
- if (!entryObject.publication.language) {
425
- // Fallback to HTML lang attribute
426
- const htmlElement = dom.querySelector('html');
427
- entryObject.publication.language = htmlElement ? htmlElement.getAttribute('lang') : null;
428
- }
429
-
430
- entryObject.publication_date = entries[i].getAttribute('sitemap-publication-date') || file.modified.on;
431
-
432
- entryObject.keywords = entries[i].getAttribute('sitemap-keywords');
433
- if (!entryObject.keywords) {
434
- const keywords = dom.querySelector('meta[name="keywords"]');
435
- entryObject.keywords = keywords ? keywords.getAttribute('content') : '';
436
- }
437
-
438
- entryObject.genres = entries[i].getAttribute('sitemap-genres');
439
- }
440
-
441
- if (previousEntries[type]) {
442
- existingObject = previousEntries[type].find(item => item[query] === entryObject[query]);
443
- entryObject = { ...existingObject, ...entryObject }
444
- }
445
-
446
- Object.keys(entryObject).forEach(key => {
447
- if (!entryObject[key])
448
- delete entryObject[key]
449
- });
450
-
451
- if (!file.sitemap[type])
452
- file.sitemap[type] = []
453
-
454
- file.sitemap[type].push(entryObject)
455
-
456
- }
457
-
458
- if (file.sitemap.type !== 'news' && file.sitemap.type !== 'image' && file.sitemap.type !== 'video') {
459
- const priorityMeta = dom.querySelector('meta[name="sitemap-priority"]');
460
- const changefreqMeta = dom.querySelector('meta[name="sitemap-changefreq"]');
461
- file.sitemap.priority = priorityMeta ? priorityMeta.getAttribute('content') : file.sitemap.priority; // Default priority if not specified
462
- file.sitemap.changefreq = changefreqMeta ? changefreqMeta.getAttribute('content') : file.sitemap.changefreq || 'monthly'; // Default changefreq if not specified
463
- }
464
-
465
- }
466
-
467
- encodeXML(str) {
468
- if (str)
469
- return str
470
- .replace(/&/g, '&amp;')
471
- .replace(/</g, '&lt;')
472
- .replace(/>/g, '&gt;')
473
- .replace(/"/g, '&quot;')
474
- .replace(/'/g, '&apos;');
475
- }
476
-
26
+ constructor(crud) {
27
+ this.crud = crud;
28
+ }
29
+
30
+ async check(file, host) {
31
+ // Ensure the file is HTML and does not have a sitemap object yet
32
+ if (!file.sitemap && file["content-type"] !== "text/html") return;
33
+
34
+ // Ensure the file is public and is not sitemap false
35
+ if (
36
+ file.sitemap === false ||
37
+ file.sitemap === "false" ||
38
+ !file.public ||
39
+ file.public === "false"
40
+ )
41
+ return;
42
+
43
+ // Check if the file is HTML and contains a noindex meta or title tag
44
+ if (file["content-type"] === "text/html") {
45
+ if (
46
+ /<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(
47
+ file.src
48
+ )
49
+ )
50
+ return;
51
+ if (!/<title[^>]*>[\s\S]*?<\/title>/i.test(file.src)) return;
52
+ }
53
+
54
+ // Compare the lastmod date in the sitemap with the modified.on date
55
+ if (file.sitemap && file.sitemap.lastmod && file.modified.on) {
56
+ if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
57
+ return;
58
+ }
59
+
60
+ // Logic to update the sitemap
61
+ this.updateSitemap(file, host);
62
+ }
63
+
64
+ // ToDo: check for <link rel="alternate" hreflang="x" /> and create entries
65
+ async updateSitemap(file, host) {
66
+ try {
67
+ if (!file.sitemap) file.sitemap = {};
68
+
69
+ const entry = this.createEntry(file, host);
70
+
71
+ let { mainSitemap, sitemap } = await this.getSitemap(file, host);
72
+
73
+ if (file.pathname) {
74
+ // Perform regex search starting at the pathname
75
+ const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(
76
+ /[.*+?^${}()|[\]\\]/g,
77
+ "\\$&"
78
+ )}.*?</loc>[\\s\\S]*?</url>`;
79
+ const match = sitemap.src.match(new RegExp(regexPattern));
80
+
81
+ if (match) {
82
+ const position = match.index; // Start position of the <url> block
83
+ const endPosition = match.index + match[0].length; // End position of the <url> block
84
+
85
+ // Replace the original <url> block with the modified one
86
+ sitemap.src =
87
+ sitemap.src.slice(0, position) +
88
+ entry +
89
+ sitemap.src.slice(endPosition);
90
+ } else {
91
+ sitemap.src = sitemap.src.replace(
92
+ "</urlset>",
93
+ `${entry}</urlset>`
94
+ );
95
+ }
96
+ } else {
97
+ file.sitemap.pathname = sitemap.pathname;
98
+ sitemap.src = sitemap.src.replace(
99
+ "</urlset>",
100
+ `${entry}</urlset>`
101
+ );
102
+ }
103
+
104
+ this.saveSitemap(mainSitemap, host);
105
+ this.saveSitemap(sitemap, host);
106
+ this.saveSitemap(file, host);
107
+
108
+ // console.log('Sitemap updated successfully.');
109
+ } catch (err) {
110
+ console.error("Error updating sitemap:", err);
111
+ }
112
+ }
113
+
114
+ createEntry(file) {
115
+ file.sitemap.loc = file.pathname;
116
+ file.sitemap.lastmod = file.modified.on;
117
+
118
+ if (file["content-type"] === "text/html") {
119
+ this.parseHtml(file);
120
+
121
+ if (
122
+ file.sitemap.type !== "news" &&
123
+ file.sitemap.type !== "image" &&
124
+ file.sitemap.type !== "video"
125
+ ) {
126
+ if (!file.sitemap.changefreq)
127
+ file.sitemap.changefreq = "monthly";
128
+
129
+ if (!file.sitemap.priority) {
130
+ const depth = (file.pathname.match(/\//g) || []).length;
131
+ file.sitemap.priority = Math.max(
132
+ 0.1,
133
+ 1.0 - (depth - 1) * 0.1
134
+ ).toFixed(1);
135
+ }
136
+ } else {
137
+ delete file.sitemap.changefreq;
138
+ delete file.sitemap.priority;
139
+ }
140
+ }
141
+
142
+ let entry = `\t<url>\n`;
143
+
144
+ for (const key of Object.keys(file.sitemap)) {
145
+ if (key === "pathname" || key === "type") continue;
146
+
147
+ let value = file.sitemap[key];
148
+
149
+ if (
150
+ typeof value === "object" &&
151
+ value !== null &&
152
+ !(value instanceof Date)
153
+ ) {
154
+ if (!Array.isArray(value)) value = [value];
155
+
156
+ for (let i = 0; i < value.length; i++) {
157
+ entry += `\t\t<${key}:${key}>\n`;
158
+
159
+ for (const nestedKey of Object.keys(value[i])) {
160
+ let nestedValue = value[i][nestedKey];
161
+ // Handle nested objects
162
+ if (
163
+ typeof nestedValue === "object" &&
164
+ nestedValue !== null &&
165
+ !(nestedValue instanceof Date)
166
+ ) {
167
+ entry += `\t\t\t<${key}:${nestedKey}>\n`;
168
+ for (const subKey of Object.keys(nestedValue)) {
169
+ const subValue = nestedValue[subKey];
170
+ entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
171
+ }
172
+ entry += `\t\t\t</${key}:${nestedKey}>\n`;
173
+ } else {
174
+ if (nestedKey === "loc") {
175
+ if (
176
+ !nestedValue.startsWith("https://") &&
177
+ !nestedValue.startsWith("http://") &&
178
+ !nestedValue.startsWith("{{$host}}")
179
+ ) {
180
+ nestedValue = `{{$host}}${nestedValue}`;
181
+ }
182
+ } else if (nestedKey === "publication_date") {
183
+ nestedValue =
184
+ new Date(nestedValue)
185
+ .toISOString()
186
+ .split(".")[0] + "Z";
187
+ } else {
188
+ nestedValue = this.encodeXML(nestedValue);
189
+ }
190
+
191
+ entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
192
+ }
193
+ }
194
+
195
+ entry += `\t\t</${key}:${key}>\n`;
196
+ }
197
+ } else {
198
+ if (key === "loc") {
199
+ if (
200
+ !value.startsWith("https://") &&
201
+ !value.startsWith("http://")
202
+ ) {
203
+ value = `{{$host}}${value}`;
204
+ }
205
+ } else if (key === "lastmod") {
206
+ value =
207
+ new Date(file.modified.on).toISOString().split(".")[0] +
208
+ "Z";
209
+ } else {
210
+ value = this.encodeXML(value);
211
+ }
212
+
213
+ entry += `\t\t<${key}>${value}</${key}>\n`;
214
+ }
215
+ }
216
+
217
+ entry += `\t</url>\n`;
218
+
219
+ return entry;
220
+ }
221
+
222
+ async getSitemap(file, host) {
223
+ let mainSitemap = {
224
+ host: file.host,
225
+ name: "sitemap.xml",
226
+ path: "/",
227
+ pathname: "/sitemap.xml",
228
+ directory: "/",
229
+ "content-type": "application/xml",
230
+ public: true,
231
+ organization_id: file.organization_id
232
+ };
233
+
234
+ mainSitemap = await this.readSitemap(mainSitemap, host);
235
+ if (!mainSitemap.src) mainSitemap.src = this.createSitemap("main");
236
+
237
+ let sitemap = {
238
+ host: file.host,
239
+ path: "/",
240
+ pathname: file.sitemap.pathname,
241
+ directory: "/",
242
+ "content-type": "application/xml",
243
+ public: true,
244
+ organization_id: file.organization_id
245
+ };
246
+
247
+ // Update loc using pathname
248
+ file.sitemap.loc = `${file.pathname}`;
249
+
250
+ // Query the database for the correct sitemap based on the loc and type
251
+ if (file.sitemap.pathname) {
252
+ sitemap = await this.readSitemap(sitemap, host);
253
+ }
254
+
255
+ if (!sitemap.src) {
256
+ let type = "sitemap";
257
+
258
+ // Identify content type to determine sitemap type
259
+ if (file["content-type"].startsWith("image/")) {
260
+ type = "image";
261
+ } else if (file["content-type"].startsWith("video/")) {
262
+ type = "video";
263
+ } else if (file.sitemap.type === "news") {
264
+ type = "news";
265
+ }
266
+
267
+ let name = `sitemap`;
268
+ if (type === "image" || type === "video" || type === "news")
269
+ name = `sitemap-${type}`;
270
+
271
+ // If no existing sitemap found check last index sitemap
272
+ let index = await this.getLastSitemapIndex(mainSitemap, name);
273
+ if (index) {
274
+ sitemap.pathname = `/${name}${index}.xml`;
275
+ sitemap = await this.readSitemap(sitemap, host);
276
+ } else {
277
+ index = 1;
278
+ }
279
+
280
+ // Check if there's room in the last index sitemap
281
+ if (!this.checkSitemap(sitemap.src)) {
282
+ if (sitemap.src) index += 1;
283
+ else sitemap.src = this.createSitemap(type);
284
+
285
+ sitemap.name = `${name}${index}.xml`;
286
+ sitemap.pathname = `/${name}${index}.xml`;
287
+ }
288
+ }
289
+
290
+ // Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
291
+ const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
292
+
293
+ // Execute the regex match against the sitemap index source
294
+ const match = mainSitemap.src.match(new RegExp(regexPattern));
295
+
296
+ // Check if a match is found
297
+ if (!match) {
298
+ //TODO: if sitemap found but not in index should we add to sitemap pathname to index or should we check the sitmap for the next index available see if room add or create new index.
299
+ const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
300
+ mainSitemap.src = mainSitemap.src.replace(
301
+ "</sitemapindex>",
302
+ `${indexEntry}\n</sitemapindex>`
303
+ );
304
+ }
305
+
306
+ return { mainSitemap, sitemap };
307
+ }
308
+
309
+ async readSitemap(file, host) {
310
+ let data = {
311
+ method: "object.read",
312
+ host: host,
313
+ array: "files",
314
+ $filter: {
315
+ query: {
316
+ host: { $in: [host, "*"] },
317
+ pathname: file.pathname
318
+ },
319
+ limit: 1
320
+ },
321
+ organization_id: file.organization_id
322
+ };
323
+ data = await this.crud.send(data);
324
+ if (data.object && data.object.length) return data.object[0];
325
+ else return file;
326
+ }
327
+
328
+ createSitemap(type) {
329
+ const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
330
+ const sitemapNamespace =
331
+ 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
332
+
333
+ if (type === "main") {
334
+ return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
335
+ } else {
336
+ const imageNamespace =
337
+ 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
338
+ const videoNamespace =
339
+ 'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
340
+ const newsNamespace =
341
+ 'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
342
+
343
+ if (type === "image") {
344
+ return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
345
+ } else if (type === "video") {
346
+ return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
347
+ } else {
348
+ // For 'news' type or any other types
349
+ return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
350
+ }
351
+ }
352
+ }
353
+
354
+ async saveSitemap(file, host) {
355
+ let data = {
356
+ method: "object.update",
357
+ host: host,
358
+ array: "files",
359
+ object: file,
360
+ upsert: true,
361
+ organization_id: file.organization_id
362
+ };
363
+ if (!file._id)
364
+ data.$filter = {
365
+ query: {
366
+ host: { $in: [host, "*"] },
367
+ pathname: file.pathname
368
+ },
369
+ limit: 1
370
+ };
371
+
372
+ data = await this.crud.send(data);
373
+ }
374
+
375
+ async getLastSitemapIndex(mainSitemap, filename) {
376
+ try {
377
+ // Use regex to match all sitemap entries for the given type
378
+ const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, "g");
379
+ const matches = mainSitemap.src.match(regex);
380
+
381
+ return matches ? matches.length : null;
382
+ } catch (err) {
383
+ console.error(
384
+ `Error determining next sitemap index for ${filename}:`,
385
+ err
386
+ );
387
+ return null; // Or some default value or throw an error
388
+ }
389
+ }
390
+
391
+ checkSitemap(sitemap) {
392
+ try {
393
+ if (!sitemap) return false;
394
+
395
+ // Count the number of <url> entries
396
+ const urlCount = (sitemap.match(/<url>/g) || []).length;
397
+ if (urlCount >= 50000) return false;
398
+
399
+ // Get the size of the sitemap string in bytes
400
+ const fileSizeInBytes = Buffer.byteLength(sitemap, "utf8");
401
+ const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
402
+
403
+ // console.log(`Sitemap has ${urlCount} entries and is ${fileSizeInMB.toFixed(2)} MB.`);
404
+
405
+ // Check if the file size exceeds either the 50MB limit or the MongoDB 16MB limit
406
+ if (fileSizeInMB >= 50 || fileSizeInMB >= 15) return false;
407
+
408
+ return true;
409
+ } catch (err) {
410
+ console.error("Error checking sitemap file:", err);
411
+ return false;
412
+ }
413
+ }
414
+
415
+ parseHtml(file) {
416
+ const dom = parse(file.src);
417
+ const entries = dom.querySelectorAll('[sitemap="true"]');
418
+
419
+ let types = ["image", "video", "news"];
420
+
421
+ const previousEntries = {};
422
+ for (let i = 0; i < types.length; i++) {
423
+ if (!file.sitemap[types[i]]) continue;
424
+ if (Array.isArray(file.sitemap[types[i]]))
425
+ previousEntries[types[i]] = file.sitemap[types[i]];
426
+ else previousEntries[types[i]] = [file.sitemap[types[i]]];
427
+
428
+ delete file.sitemap[types[i]];
429
+ }
430
+
431
+ for (let i = 0; i < entries.length; i++) {
432
+ let type = "",
433
+ query = "";
434
+ let existingObject;
435
+ let entryObject = {};
436
+
437
+ if (entries[i].tagName === "IMG") {
438
+ // Corrected to 'IMG' for images
439
+ type = "image";
440
+ query = "loc";
441
+ entryObject.loc = entries[i].getAttribute("src");
442
+ entryObject.title =
443
+ entries[i].getAttribute("sitemap-title") ||
444
+ entries[i].getAttribute("title") ||
445
+ entries[i].getAttribute("alt");
446
+ entryObject.caption =
447
+ entries[i].getAttribute("sitemap-caption") ||
448
+ entries[i].getAttribute("alt") ||
449
+ entryObject.title;
450
+ entryObject.geo_location = entries[i].getAttribute(
451
+ "sitemap-geo-location"
452
+ );
453
+ } else if (entries[i].tagName === "VIDEO") {
454
+ type = "video";
455
+ query = "content_loc";
456
+ entryObject.content_loc = entries[i].src;
457
+ entryObject.title =
458
+ entries[i].getAttribute("sitemap-title") ||
459
+ entries[i].getAttribute("title");
460
+ entryObject.description =
461
+ entries[i].getAttribute("description"); // 'description' if available
462
+ entryObject.thumbnail_loc =
463
+ entries[i].getAttribute("sitemap-thumbnail") ||
464
+ entries[i].getAttribute("poster");
465
+ entryObject.duration =
466
+ entries[i].getAttribute("sitemap-duration");
467
+ } else {
468
+ type = "news";
469
+ file.sitemap.type = "news";
470
+ query = "title";
471
+ entryObject.title = entries[i].getAttribute("sitemap-title");
472
+ if (!entryObject.title) {
473
+ const title = dom.querySelector("title");
474
+ entryObject.title = title ? title.text : "";
475
+ }
476
+
477
+ entryObject.publication = {
478
+ name: entries[i].getAttribute("sitemap-publication-name"), // Use proper attribute
479
+ language: entries[i].getAttribute(
480
+ "sitemap-publication-language"
481
+ ) // Use proper attribute
482
+ };
483
+
484
+ if (!entryObject.publication.language) {
485
+ // Fallback to HTML lang attribute
486
+ const htmlElement = dom.querySelector("html");
487
+ entryObject.publication.language = htmlElement
488
+ ? htmlElement.getAttribute("lang")
489
+ : null;
490
+ }
491
+
492
+ entryObject.publication_date =
493
+ entries[i].getAttribute("sitemap-publication-date") ||
494
+ file.modified.on;
495
+
496
+ entryObject.keywords =
497
+ entries[i].getAttribute("sitemap-keywords");
498
+ if (!entryObject.keywords) {
499
+ const keywords = dom.querySelector('meta[name="keywords"]');
500
+ entryObject.keywords = keywords
501
+ ? keywords.getAttribute("content")
502
+ : "";
503
+ }
504
+
505
+ entryObject.genres = entries[i].getAttribute("sitemap-genres");
506
+ }
507
+
508
+ if (previousEntries[type]) {
509
+ existingObject = previousEntries[type].find(
510
+ (item) => item[query] === entryObject[query]
511
+ );
512
+ entryObject = { ...existingObject, ...entryObject };
513
+ }
514
+
515
+ Object.keys(entryObject).forEach((key) => {
516
+ if (!entryObject[key]) delete entryObject[key];
517
+ });
518
+
519
+ if (!file.sitemap[type]) file.sitemap[type] = [];
520
+
521
+ file.sitemap[type].push(entryObject);
522
+ }
523
+
524
+ if (
525
+ file.sitemap.type !== "news" &&
526
+ file.sitemap.type !== "image" &&
527
+ file.sitemap.type !== "video"
528
+ ) {
529
+ const priorityMeta = dom.querySelector(
530
+ 'meta[name="sitemap-priority"]'
531
+ );
532
+ const changefreqMeta = dom.querySelector(
533
+ 'meta[name="sitemap-changefreq"]'
534
+ );
535
+ file.sitemap.priority = priorityMeta
536
+ ? priorityMeta.getAttribute("content")
537
+ : file.sitemap.priority; // Default priority if not specified
538
+ file.sitemap.changefreq = changefreqMeta
539
+ ? changefreqMeta.getAttribute("content")
540
+ : file.sitemap.changefreq || "monthly"; // Default changefreq if not specified
541
+ }
542
+ }
543
+
544
+ encodeXML(str) {
545
+ if (str)
546
+ return str
547
+ .replace(/&/g, "&amp;")
548
+ .replace(/</g, "&lt;")
549
+ .replace(/>/g, "&gt;")
550
+ .replace(/"/g, "&quot;")
551
+ .replace(/'/g, "&apos;");
552
+ }
477
553
  }
478
554
 
479
555
  module.exports = CoCreateSitemap;