@goflag/cli 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1144,6 +1144,7 @@ async function crawl(options) {
|
|
|
1144
1144
|
const pages = [];
|
|
1145
1145
|
const errors = [];
|
|
1146
1146
|
let truncated = false;
|
|
1147
|
+
const seenFinal = /* @__PURE__ */ new Set();
|
|
1147
1148
|
while (queue.length > 0) {
|
|
1148
1149
|
if (options.inspectOptions?.signal?.aborted) break;
|
|
1149
1150
|
const wave = queue.splice(0, concurrency);
|
|
@@ -1165,6 +1166,9 @@ async function crawl(options) {
|
|
|
1165
1166
|
errors.push({ url: result.item.url, message: result.error });
|
|
1166
1167
|
continue;
|
|
1167
1168
|
}
|
|
1169
|
+
const finalUrl = canonicaliseUrl(result.page.fetch.finalUrl) ?? result.page.fetch.finalUrl;
|
|
1170
|
+
if (seenFinal.has(finalUrl)) continue;
|
|
1171
|
+
seenFinal.add(finalUrl);
|
|
1168
1172
|
pages.push(result.page);
|
|
1169
1173
|
if (options.onPage) {
|
|
1170
1174
|
options.onPage(result.page, { visited: pages.length, queued: queue.length });
|
|
@@ -4410,4 +4414,4 @@ export {
|
|
|
4410
4414
|
summarize,
|
|
4411
4415
|
Logger
|
|
4412
4416
|
};
|
|
4413
|
-
//# sourceMappingURL=chunk-
|
|
4417
|
+
//# sourceMappingURL=chunk-5FO2Q3II.js.map
|