@cocreate/sitemap 1.6.3 → 1.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -52
- package/package.json +12 -10
- package/src/index.js +19 -20
package/README.md
CHANGED
|
@@ -1,91 +1,149 @@
|
|
|
1
1
|
# CoCreate-sitemap
|
|
2
2
|
|
|
3
|
-
A
|
|
3
|
+
A high-performance, stateless, and functional JavaScript utility for automated XML sitemap generation and management. Engineered to run seamlessly across clustered environments, it dynamically builds, updates, and shards compliant sitemaps—including standard URLs, Images, Videos, and Google News metadata—by evaluating data modifications and parsing live HTML content.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
---
|
|
11
6
|
|
|
12
|
-
|
|
7
|
+
## Table of Contents
|
|
13
8
|
|
|
14
|
-
|
|
9
|
+
* [Features](#features)
|
|
10
|
+
* [Installation](#installation)
|
|
11
|
+
* [Usage](#usage)
|
|
12
|
+
* [How it Works](#how-it-works)
|
|
13
|
+
* [HTML Schema Overrides](#html-schema-overrides)
|
|
14
|
+
* [Announcements](#announcements)
|
|
15
|
+
* [Roadmap](#roadmap)
|
|
16
|
+
* [How to Contribute](#how-to-contribute)
|
|
17
|
+
* [About](#about)
|
|
18
|
+
* [License](#license)
|
|
15
19
|
|
|
16
|
-
|
|
20
|
+
---
|
|
17
21
|
|
|
18
|
-
##
|
|
22
|
+
## Features
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
24
|
+
* **Dynamic XML Automation:** Intercepts real-time document modifications to generate, update, or expand sitemaps instantly.
|
|
25
|
+
* **Algorithmic Guardrails:** Automatically enforces strict SEO quality filters, bypassing draft states, private URLs, `noindex` directions, and views missing a valid `<title>`.
|
|
26
|
+
* **Delta Freshness Verifications:** Prevents redundant database write cycles by skipping processing if the active map's `lastmod` timestamp is already up to date with the resource's latest modification.
|
|
27
|
+
* **Hierarchical SEO Weighting:** Automatically derives page crawling priority values based on the layout depth of the URL path, eliminating manual weighting.
|
|
28
|
+
* **Rich Media Graphing:** Scans internal HTML nodes decorated with `sitemap="true"` to extract context and populate independent tracking sets for Images, Videos, and Google News.
|
|
29
|
+
* **Automated Index Sharding:** Monitors file buffers natively to cleanly split indexing paths when passing Google's 50,000 URL limit or MongoDB's 15MB document safety limit.
|
|
23
30
|
|
|
24
|
-
|
|
25
|
-
<script src="https://cdn.cocreate.app/sitemap/latest/CoCreate-sitemap.min.css"></script>
|
|
26
|
-
```
|
|
31
|
+
---
|
|
27
32
|
|
|
28
|
-
##
|
|
33
|
+
## Installation
|
|
29
34
|
|
|
30
|
-
```
|
|
31
|
-
|
|
35
|
+
```bash
|
|
36
|
+
npm install @cocreate/sitemap
|
|
32
37
|
```
|
|
33
38
|
|
|
34
|
-
|
|
39
|
+
> [!NOTE]
|
|
40
|
+
> All core underlying dependencies (including parsing utilities like `node-html-parser`) are managed natively and installed automatically.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
### JavaScript API
|
|
47
|
+
|
|
48
|
+
Inject your runtime context (`file`, `host`, and your decoupled `crud` database engine) into the stateless `check` entry point:
|
|
49
|
+
|
|
50
|
+
```javascript
|
|
51
|
+
import { check } from '@cocreate/sitemap';
|
|
52
|
+
|
|
53
|
+
const fileContext = {
|
|
54
|
+
"content-type": "text/html",
|
|
55
|
+
"public": true,
|
|
56
|
+
"pathname": "/blog/dynamic-sitemaps",
|
|
57
|
+
"src": "<html><head><title>My Blog Post</title></head><body>...</body></html>",
|
|
58
|
+
"modified": { on: "2026-07-19T08:30:00Z" }
|
|
59
|
+
};
|
|
35
60
|
|
|
36
|
-
|
|
37
|
-
|
|
61
|
+
await check({
|
|
62
|
+
file: fileContext,
|
|
63
|
+
host: "https://example.com",
|
|
64
|
+
crud: yourCrudDatabaseInstance
|
|
65
|
+
});
|
|
38
66
|
```
|
|
39
67
|
|
|
40
|
-
|
|
68
|
+
---
|
|
41
69
|
|
|
42
|
-
|
|
43
|
-
- [Announcements](#announcements)
|
|
44
|
-
- [Roadmap](#roadmap)
|
|
45
|
-
- [How to Contribute](#how-to-contribute)
|
|
46
|
-
- [About](#about)
|
|
47
|
-
- [License](#license)
|
|
70
|
+
## How it Works
|
|
48
71
|
|
|
49
|
-
|
|
72
|
+
1. **Eligibility Filter:** The module checks structural requirements (such as content type, public access flags, absence of `noindex` directives, and valid `<title>` nodes).
|
|
73
|
+
2. **Freshness Assessment:** The target's change vector is cross-referenced against saved records. If no updates are required, execution exits early to reduce unnecessary I/O.
|
|
74
|
+
3. **DOM Content Extraction:** For valid HTML documents, the layout engine parses targeted media assets marked for sitemap ingestion.
|
|
75
|
+
4. **Sharding Verification:** The engine reads the root index (`sitemap.xml`) to locate corresponding media buckets. If a sitemap approaches Google's 50,000 URL limit or MongoDB's 15MB document limit, a new sitemap shard is automatically created.
|
|
76
|
+
5. **XML Serialization:** Encodes structural characters and writes updated sitemap data back through your configured `crud` transport.
|
|
50
77
|
|
|
51
|
-
|
|
78
|
+
---
|
|
52
79
|
|
|
53
|
-
|
|
80
|
+
## HTML Schema Overrides
|
|
54
81
|
|
|
55
|
-
|
|
82
|
+
You can explicitly override calculated priorities, change frequencies, or extract media nodes directly from your frontend markup.
|
|
56
83
|
|
|
57
|
-
|
|
84
|
+
### Meta Specifications
|
|
58
85
|
|
|
59
|
-
|
|
86
|
+
* **Exclude Page:** `<meta name="robots" content="noindex">`
|
|
87
|
+
* **Custom Crawling Priority:** `<meta name="sitemap-priority" content="0.9">`
|
|
88
|
+
* **Custom Change Frequency:** `<meta name="sitemap-changefreq" content="weekly">`
|
|
60
89
|
|
|
61
|
-
|
|
90
|
+
### Extended Media Node Extractors
|
|
62
91
|
|
|
63
|
-
|
|
92
|
+
Simply mark target elements with `sitemap="true"` to automatically include them in media sitemaps:
|
|
64
93
|
|
|
65
|
-
|
|
94
|
+
```html
|
|
95
|
+
<!-- Automated Image Mapping -->
|
|
96
|
+
<img src="/assets/hero.jpg"
|
|
97
|
+
sitemap="true"
|
|
98
|
+
title="Hero View"
|
|
99
|
+
alt="Main Dashboard Profile"
|
|
100
|
+
sitemap-geo-location="New York, NY" />
|
|
101
|
+
|
|
102
|
+
<!-- Automated Video Mapping -->
|
|
103
|
+
<video src="/media/explainer.mp4"
|
|
104
|
+
sitemap="true"
|
|
105
|
+
title="Product Walkthrough"
|
|
106
|
+
poster="/thumbs/explainer.jpg"
|
|
107
|
+
description="A complete system overview"
|
|
108
|
+
sitemap-duration="180" />
|
|
109
|
+
```
|
|
66
110
|
|
|
67
|
-
|
|
111
|
+
---
|
|
68
112
|
|
|
69
|
-
|
|
113
|
+
## Announcements
|
|
70
114
|
|
|
71
|
-
|
|
115
|
+
All updates, protocol revisions, and performance improvements are documented in our `CHANGELOG.md` and on the [CoCreate Sitemap GitHub Releases](https://github.com/CoCreate-app/CoCreate-sitemap/releases) page.
|
|
72
116
|
|
|
73
|
-
|
|
117
|
+
---
|
|
74
118
|
|
|
75
|
-
|
|
119
|
+
## Roadmap
|
|
76
120
|
|
|
77
|
-
|
|
121
|
+
If you are interested in the future direction of this project, please take a look at our open issues and pull requests on the repository. We welcome community input on scaling indexing strategies and improving sitemap generation performance.
|
|
78
122
|
|
|
79
|
-
|
|
123
|
+
---
|
|
80
124
|
|
|
81
|
-
|
|
125
|
+
## How to Contribute
|
|
82
126
|
|
|
83
|
-
|
|
127
|
+
We encourage contribution to our libraries (you might even score some nifty swag), please see our [CONTRIBUTING.md](https://github.com/CoCreate-app/CoCreate-sitemap/blob/master/CONTRIBUTING.md) guide for details. If you encounter any bugs or wish to make feature requests, please submit an issue on our [GitHub Issues](https://github.com/CoCreate-app/CoCreate-sitemap/issues) tracker. We want this library to be community-driven, and CoCreate-led. We need your help to realize this goal.
|
|
84
128
|
|
|
85
|
-
|
|
129
|
+
For broader system configurations and API guides, please visit our [CoCreate Sitemap Documentation](https://cocreatejs.com/docs/sitemap).
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## About
|
|
86
134
|
|
|
87
|
-
|
|
135
|
+
`@cocreate/sitemap` is designed, built, and supported by the CoCreate Developer Experience Team.
|
|
88
136
|
|
|
89
|
-
|
|
137
|
+
> [!NOTE]
|
|
138
|
+
> Please contact the Developer Experience Team via [GitHub Discussions](https://github.com/CoCreate-app/CoCreate-sitemap/discussions) or join our Discord community for questions about sitemap generation, indexing, or deployment.
|
|
139
|
+
|
|
140
|
+
`@cocreate/sitemap` is maintained and funded by CoCreate. The names and logos for CoCreate are trademarks of CoCreate, LLC.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
This software is dual-licensed under the GNU Affero General Public License version 3 (AGPLv3) and a commercial license.
|
|
90
147
|
|
|
91
|
-
|
|
148
|
+
* **Open Source Use:** For open-source projects and non-commercial use, this software is available under the AGPLv3. For the full license text, see the LICENSE file.
|
|
149
|
+
* **Commercial Use:** For-profit companies and individuals intending to use this software for commercial purposes must obtain a commercial license. The commercial license is available when you sign up for an API key on our website.
|
package/package.json
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cocreate/sitemap",
|
|
3
|
-
"version": "1.6.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.6.4",
|
|
4
|
+
"description": "Automated stateless XML sitemap engine with multi-format media extraction (Images, Videos, Google News) and dynamic data-sharding for scalable document architectures.",
|
|
5
5
|
"keywords": [
|
|
6
|
-
"sitemap",
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"
|
|
6
|
+
"sitemap-generator",
|
|
7
|
+
"xml-sitemap",
|
|
8
|
+
"seo-automation",
|
|
9
|
+
"sitemap-sharding",
|
|
10
|
+
"google-news-sitemap",
|
|
11
|
+
"video-sitemap",
|
|
12
|
+
"image-sitemap",
|
|
13
|
+
"html-parsing",
|
|
14
|
+
"stateless-utility",
|
|
15
|
+
"cocreate-module"
|
|
14
16
|
],
|
|
15
17
|
"publishConfig": {
|
|
16
18
|
"access": "public"
|
package/src/index.js
CHANGED
|
@@ -56,11 +56,11 @@ async function updateSitemap({ file, host, crud }) {
|
|
|
56
56
|
let { mainSitemap, sitemap } = await getSitemap({ file, host, crud });
|
|
57
57
|
|
|
58
58
|
if (file.pathname) {
|
|
59
|
-
//
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
)}
|
|
59
|
+
// Escapes regular expression special characters safely
|
|
60
|
+
const escapedPathname = file.pathname.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
61
|
+
|
|
62
|
+
// Matches path blocks whether they start with templates, protocols, or raw paths
|
|
63
|
+
const regexPattern = `<url>\\s*<loc>\\s*(?:https?:)?\\/?\\/?(?:\\{\\{\\$host\\}\\})?.*?${escapedPathname}\\s*<\\/loc>[\\s\\S]*?<\\/url>`;
|
|
64
64
|
const match = sitemap.src.match(new RegExp(regexPattern));
|
|
65
65
|
|
|
66
66
|
if (match) {
|
|
@@ -142,7 +142,7 @@ function createEntry(file) {
|
|
|
142
142
|
|
|
143
143
|
for (const nestedKey of Object.keys(value[i])) {
|
|
144
144
|
let nestedValue = value[i][nestedKey];
|
|
145
|
-
// Handle nested objects
|
|
145
|
+
// Handle nested objects safely
|
|
146
146
|
if (
|
|
147
147
|
typeof nestedValue === "object" &&
|
|
148
148
|
nestedValue !== null &&
|
|
@@ -151,7 +151,7 @@ function createEntry(file) {
|
|
|
151
151
|
entry += `\t\t\t<${key}:${nestedKey}>\n`;
|
|
152
152
|
for (const subKey of Object.keys(nestedValue)) {
|
|
153
153
|
const subValue = nestedValue[subKey];
|
|
154
|
-
entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
|
|
154
|
+
entry += `\t\t\t\t<${key}:${subKey}>${encodeXML(subValue)}</${key}:${subKey}>\n`;
|
|
155
155
|
}
|
|
156
156
|
entry += `\t\t\t</${key}:${nestedKey}>\n`;
|
|
157
157
|
} else {
|
|
@@ -279,7 +279,7 @@ async function getSitemap({ file, host, crud }) {
|
|
|
279
279
|
|
|
280
280
|
// Check if a match is found
|
|
281
281
|
if (!match) {
|
|
282
|
-
const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
|
|
282
|
+
const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n\t</sitemap>`;
|
|
283
283
|
mainSitemap.src = mainSitemap.src.replace(
|
|
284
284
|
"</sitemapindex>",
|
|
285
285
|
`${indexEntry}\n</sitemapindex>`
|
|
@@ -365,7 +365,7 @@ async function getLastSitemapIndex(mainSitemap, filename) {
|
|
|
365
365
|
} catch (err) {
|
|
366
366
|
console.error(
|
|
367
367
|
`Error determining next sitemap index for ${filename}:`,
|
|
368
|
-
|
|
368
|
+
err
|
|
369
369
|
);
|
|
370
370
|
return null;
|
|
371
371
|
}
|
|
@@ -383,8 +383,8 @@ function checkSitemapXml(sitemap) {
|
|
|
383
383
|
const fileSizeInBytes = Buffer.byteLength(sitemap, "utf8");
|
|
384
384
|
const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
|
|
385
385
|
|
|
386
|
-
// Check if the file size exceeds
|
|
387
|
-
if (fileSizeInMB >=
|
|
386
|
+
// Check if the file size exceeds MongoDB 16MB document boundary limit safely
|
|
387
|
+
if (fileSizeInMB >= 15) return false;
|
|
388
388
|
|
|
389
389
|
return true;
|
|
390
390
|
} catch (err) {
|
|
@@ -435,7 +435,6 @@ function parseHtml(file) {
|
|
|
435
435
|
} else if (entries[i].tagName === "VIDEO") {
|
|
436
436
|
type = "video";
|
|
437
437
|
query = "content_loc";
|
|
438
|
-
// FIXED: Using standard DOM query selector `.getAttribute("src")` instead of property `.src`
|
|
439
438
|
entryObject.content_loc = entries[i].getAttribute("src");
|
|
440
439
|
entryObject.title =
|
|
441
440
|
entries[i].getAttribute("sitemap-title") ||
|
|
@@ -524,11 +523,11 @@ function parseHtml(file) {
|
|
|
524
523
|
}
|
|
525
524
|
|
|
526
525
|
function encodeXML(str) {
|
|
527
|
-
if (str)
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
526
|
+
if (str === null || str === undefined) return "";
|
|
527
|
+
return String(str)
|
|
528
|
+
.replace(/&/g, "&")
|
|
529
|
+
.replace(/</g, "<")
|
|
530
|
+
.replace(/>/g, ">")
|
|
531
|
+
.replace(/"/g, """)
|
|
532
|
+
.replace(/'/g, "'");
|
|
533
|
+
}
|