@adobe/spacecat-shared-scrape-client 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +2 -2
- package/src/clients/scrape-job-supervisor.js +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@adobe/spacecat-shared-scrape-client-v1.0.5](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.4...@adobe/spacecat-shared-scrape-client-v1.0.5) (2025-07-27)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* **deps:** update external fixes ([#878](https://github.com/adobe/spacecat-shared/issues/878)) ([b049828](https://github.com/adobe/spacecat-shared/commit/b04982839c0ff5e4de4ab0e37508c5eb5272a679))
|
|
7
|
+
|
|
8
|
+
# [@adobe/spacecat-shared-scrape-client-v1.0.4](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.3...@adobe/spacecat-shared-scrape-client-v1.0.4) (2025-07-24)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* (scrape-job-supervisor): add offset for URL numbering in batches ([#868](https://github.com/adobe/spacecat-shared/issues/868)) ([12789c0](https://github.com/adobe/spacecat-shared/commit/12789c0cabe33ad5e526793d645bfef421a851af))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-scrape-client-v1.0.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.2...@adobe/spacecat-shared-scrape-client-v1.0.3) (2025-07-21)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adobe/spacecat-shared-scrape-client",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.5",
|
|
4
4
|
"description": "Shared modules of the Spacecat Services - Scrape Client",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"chai": "5.2.1",
|
|
44
44
|
"chai-as-promised": "8.0.1",
|
|
45
|
-
"nock": "14.0.
|
|
45
|
+
"nock": "14.0.7",
|
|
46
46
|
"sinon": "20.0.0",
|
|
47
47
|
"sinon-chai": "4.0.0",
|
|
48
48
|
"typescript": "5.8.3"
|
|
@@ -173,11 +173,14 @@ function ScrapeJobSupervisor(services, config) {
|
|
|
173
173
|
urlBatches = [urls]; // Wrap in an array to maintain consistent structure
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
for (const batch of urlBatches) {
|
|
176
|
+
for (const [index, batch] of urlBatches.entries()) {
|
|
177
|
+
// Calculate the offset for numbering the URLs in the batch
|
|
178
|
+
const offset = index * maxUrlsPerMessage;
|
|
177
179
|
const message = {
|
|
178
180
|
processingType,
|
|
179
181
|
jobId: scrapeJob.getId(),
|
|
180
182
|
batch,
|
|
183
|
+
batchOffset: offset,
|
|
181
184
|
customHeaders,
|
|
182
185
|
options,
|
|
183
186
|
};
|