@metreeca/pipe-url 0.10.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,42 @@
+# @metreeca/pipe-url
+
+[![npm](https://img.shields.io/npm/v/@metreeca/pipe-url)](https://www.npmjs.com/package/@metreeca/pipe-url)
+
+URL processing tasks for [@metreeca/pipe](https://github.com/metreeca/pipe).
+
+# Installation
+
+```shell
+npm install @metreeca/gear      # the job executor
+npm install @metreeca/pipe-url  # this package
+```
+
+> [!IMPORTANT]
+>
+> Node.js 22 or later is required.
+
+> [!WARNING]
+>
+> TypeScript consumers must use `"moduleResolution": "nodenext"/"node16"/"bundler"` in `tsconfig.json`.
+> The legacy `"node"` resolver is not supported.
+
+# Usage
+
+| Task               | Description                    |
+|--------------------|--------------------------------|
+| [`fetch()`][fetch] | Resource fetcher               |
+| [`crawl()`][crawl] | URL graph walker and harvester |
+
+[fetch]: https://metreeca.github.io/pipe/functions/_metreeca_pipe-url.fetch.html
+
+[crawl]: https://metreeca.github.io/pipe/functions/_metreeca_pipe-url.crawl.html
+
+# Support
+
+- open an [issue](https://github.com/metreeca/pipe/issues) to report a problem or to suggest a new feature
+- start a [discussion](https://github.com/metreeca/pipe/discussions) to ask a how-to question or to share an idea
+
+# License
+
+This project is licensed under the Apache 2.0 License –
+see [LICENSE](https://github.com/metreeca/pipe?tab=Apache-2.0-1-ov-file) file for details.

package/dist/crawl.d.ts

@@ -0,0 +1,140 @@
+import type { Optional, URLLike } from "@metreeca/core";
+import type { Awaitable, Awaitables } from "@metreeca/core/async";
+import type { Task } from "@metreeca/flow";
+/**
+ * Possibly asynchronous, possibly absent value.
+ *
+ * Absence and asynchrony are taken uniformly, so that a provider hands over whatever it already holds, a value, a
+ * promise or nothing at all.
+ *
+ * @typeParam T The type of the supplied value
+ */
+export type Source<T> = Awaitable<Optional<T>>;
+/**
+ * Creates a URL graph walker.
+ *
+ * The generated task converts a feed of seed URLs into a feed of the URLs reachable from them, so that a consumer
+ * works on a whole graph of URLs while stating no more than the step from one URL to the next. URLs are emitted
+ * breadth-first in level order, every seed first, then every URL one step away from a seed, and so on, so that the
+ * first arrival at a URL is also its shallowest one.
+ *
+ * The walker navigates a graph of URLs without retrieving the resources they identify. Any retrieval needed to find
+ * the links of a URL is up to `walker`, and deriving results from the crawled URLs is up to the tasks downstream; the
+ * harvester form takes retrieval and derivation over when a crawl must read each resource anyway. Seeds and links are
+ * stated as {@link URLLike} values, strings and {@link !URL URL} objects alike, but reach `walker` and the feed as
+ * parsed objects. Each object is a fresh copy, safe to be altered.
+ *
+ * > [!NOTE]
+ * >
+ * > - **Incremental**: seeds are emitted as they are pulled, and reachable URLs level by level, so the feed produced
+ * >   runs dry when the source and `walker` do. No URL reachable from a seed is emitted until the source runs dry,
+ * >   so the feed never completes on an endless source.
+ * > - **Materialising**: every crawled URL is retained for the whole lifetime of the feed, as are the seeds and the
+ * >   level being crawled, so an unbounded or widely branching graph may exhaust memory.
+ * > - **Stateful**: the URLs already crawled decide the ones that follow, so a task invoked per nested feed or per
+ * >   run crawls each independently, reaching a URL once per invocation rather than once for the feed as a whole.
+ *
+ * > [!IMPORTANT]
+ * >
+ * > URLs are crawled at most once across the whole feed, whatever seed they are reached from, so cyclic and
+ * > converging graphs are crawled without duplicates and without looping. URLs are matched in canonical form: an
+ * > omitted path or an uppercase host doesn't make a URL distinct, while a trailing slash or a fragment does.
+ *
+ * @param walker The function stating the URLs linked from a URL, none if it is a leaf
+ *
+ * @returns A task converting a feed of seed URLs into a feed of the seeds and the URLs reachable from them, each as a
+ *          parsed object
+ *
+ * @throws {@link !Error Error} While the feed is consumed, whatever the source reports while producing seeds, or
+ *                              whatever `walker` reports while stating the URLs linked from a URL
+ *
+ * @throws {@link !TypeError TypeError} While the feed is consumed, if a seed or a link cannot be parsed on its
+ *                                      own, a relative reference among them
+ *
+ * @example
+ *
+ * ```typescript
+ * const pages: Record<string, string[]> = { "/a": ["/b", "/c"], "/b": ["/d"], "/c": ["/d"], "/d": [] };
+ *
+ * await pipe(
+ *   (items(["https://example.com/a"]))
+ *   (crawl(url => pages[url.pathname]?.map(path => new URL(path, url))))
+ *   (toArray())
+ * );  // the URLs of /a, /b, /c and /d, in that order
+ * ```
+ *
+ * @group Factories
+ */
+export declare function crawl(walker: (url: URL) => Source<Awaitables<URLLike>>): Task<URLLike, URL>;
+/**
+ * Creates a URL graph harvester.
+ *
+ * The generated task converts a feed of seed URLs into a feed of results derived from the resources the crawled URLs
+ * identify, so that a consumer harvests a whole graph of URLs while stating the retrieval of a URL as a task of its
+ * own. Each crawled URL is retrieved once, however many links converge on it. The value retrieved for it serves both
+ * to find its links and to derive its results, so no resource is read twice. Results are emitted in level order: the
+ * results of every seed first, then those of every URL one step away from a seed, and so on.
+ *
+ * Retrieval is stated as a task over a whole level rather than as a step per URL, so the consumer controls how many
+ * URLs are retrieved at a time with the tasks already at hand: a forked `feeder` retrieves several at once, an
+ * unforked one retrieves them in turn. A URL is left out of the harvest by emitting nothing for it. Seeds and links
+ * are stated as {@link URLLike} values, strings and {@link !URL URL} objects alike, but reach `feeder` as parsed
+ * objects. Each object is a fresh copy, safe to be altered.
+ *
+ * > [!NOTE]
+ * >
+ * > - **Incremental**: the results of the seeds are emitted as `feeder` draws them, and those of reachable URLs level
+ * >   by level, so the feed produced runs dry when the source, `feeder`, `walker` and `mapper` do. No URL reachable
+ * >   from a seed is fed until the source runs dry, so the feed never completes on an endless source.
+ * > - **Materialising**: every crawled URL is retained for the whole lifetime of the feed, as for the walker form.
+ * >   The value retrieved for a URL is released as soon as it is walked and mapped, so it is never retained across
+ * >   levels.
+ * > - **Stateful**: the URLs already crawled decide the ones that follow, as for the walker form.
+ *
+ * > [!IMPORTANT]
+ * >
+ * > URLs are crawled at most once across the whole feed, whatever seed they are reached from, and matched in
+ * > canonical form, as for the walker form.
+ *
+ * > [!IMPORTANT]
+ * >
+ * > `feeder` is invoked once per level, so any state it initialises on invocation lasts for that level only. State
+ * > spanning the whole crawl belongs to the enclosing closure. Levels never mix, but order within a level is up to
+ * > `feeder`: a feeder retrieving several URLs at a time harvests a level in completion order.
+ *
+ * @typeParam V The type of the value retrieved for a crawled URL
+ * @typeParam R The type of the results derived from a crawled URL
+ *
+ * @param feeder The task retrieving a value for each URL of a level, emitting nothing for a URL to be crawled no
+ *               further and to contribute no result
+ * @param walker The function stating the URLs linked from the value retrieved for a URL, none if it is a leaf
+ * @param mapper The function stating the results derived from the value retrieved for a URL, either a single result
+ *               or a sequence of them, none if it contributes no result
+ *
+ * @returns A task converting a feed of seed URLs into a feed of the results derived from every crawled URL
+ *
+ * @throws {@link !Error Error} While the feed is consumed, whatever the source reports while producing seeds, or
+ *                              whatever `feeder`, `walker` and `mapper` report while retrieving, walking and mapping
+ *                              a URL
+ *
+ * @throws {@link !TypeError TypeError} While the feed is consumed, if a seed or a link cannot be parsed on its
+ *                                      own, a relative reference among them
+ *
+ * @example
+ *
+ * ```typescript
+ * await pipe(
+ *   (items(["https://example.com/products/"]))
+ *   (crawl(
+ *     fork(4, urls => urls(fetch())(html())(xpath())), // the index pages, four retrievals at a time
+ *     page => page("//nav//a/@href").map(link), // the index pages it paginates to
+ *     page => page("//article//a/@href").map(link) // the item links it lists
+ *   ))
+ *   (toArray())
+ * );  // the item links of every index page
+ * ```
+ *
+ * @group Factories
+ */
+export declare function crawl<V, R>(feeder: Task<URL, V>, walker: (data: V) => Source<Awaitables<URLLike>>, mapper: (data: V) => Source<R | Awaitables<R>>): Task<URLLike, R>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAI3C;;;;;;;GAOG;AACH,MAAM,MAAM,MAAM,CAAC,CAAC,IACnB,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAMxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,wBAAgB,KAAK,CACpB,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAC/C,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AACH,wBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC,EACzB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EACpB,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAChD,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,GAC5C,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC"}

package/dist/crawl.js

@@ -0,0 +1,85 @@
+/*
+ * Copyright © 2026 Metreeca srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { items } from "@metreeca/flow/feeds";
+/**
+ * Creates a URL graph walker or harvester.
+ */
+export function crawl(...steps) {
+    return steps.length === 1
+        ? roam(steps[0])
+        : reap(steps[0], steps[1], steps[2]);
+    function roam(walker) {
+        return source => items((async function* () {
+            const admitted = admitting();
+            // the seed level, drained before descending so that the first arrival at a URL is its shallowest;
+            // seeds are emitted as they are pulled, so a slow source doesn't withhold the ones already in
+            const seeds = [];
+            for await (const url of admitted(source)) {
+                seeds.push(url);
+                yield url;
+            }
+            yield* descending(seeds, reach);
+            async function* reach(frontier, reached) {
+                for (const url of frontier) {
+                    for await (const next of admitted(await walker(url))) {
+                        reached.push(next);
+                        yield next;
+                    }
+                }
+            }
+        })());
+    }
+    function reap(feeder, walker, mapper) {
+        return source => items((async function* () {
+            const admitted = admitting();
+            // the URLs linked from the seed level, buffered until the source runs dry so that the first arrival at a
+            // URL is its shallowest; seeds are fed as `feeder` draws them, so a slow source doesn't withhold the
+            // ones already in
+            const linked = [];
+            yield* reach(admitted(source), linked);
+            yield* descending(linked, reach);
+            async function* reach(frontier, reached) {
+                for await (const data of feeder(items(frontier))) {
+                    yield* items(await mapper(data) ?? []);
+                    for await (const next of admitted(await walker(data))) {
+                        reached.push(next);
+                    }
+                }
+            }
+        })());
+    }
+    function admitting() {
+        const crawled = new Set();
+        return async function* (links) {
+            for await (const link of items(links ?? [])) {
+                const url = new URL(link);
+                if (!crawled.has(url.href)) {
+                    crawled.add(url.href);
+                    yield url;
+                }
+            }
+        };
+    }
+    async function* descending(seeds, reach) {
+        let frontier = seeds;
+        while (frontier.length > 0) {
+            const reached = [];
+            yield* reach(frontier, reached);
+            frontier = reached;
+        }
+    }
+}
+//# sourceMappingURL=crawl.js.map

package/dist/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAyJ7C;;GAEG;AACH,MAAM,UAAU,KAAK,CAAO,GAAG,KAQ7B;IAGD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;QACxB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAGtC,SAAS,IAAI,CAAC,MAAiD;QAE9D,OAAO,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC;YAEtC,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC;YAE7B,kGAAkG;YAClG,8FAA8F;YAE9F,MAAM,KAAK,GAAU,EAAE,CAAC;YAExB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAE1C,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEhB,MAAM,GAAG,CAAC;YAEX,CAAC;YAED,KAAK,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAGhC,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,QAAwB,EAAE,OAAc;gBAE7D,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAE5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;wBAEtD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAEnB,MAAM,IAAI,CAAC;oBAEZ,CAAC;gBAEF,CAAC;YAEF,CAAC;QAEF,CAAC,CAAC,EAAE,CAAC,CAAC;IAEP,CAAC;IAED,SAAS,IAAI,CACZ,MAAoB,EACpB,MAAgD,EAChD,MAA8C;QAG9C,OAAO,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC;YAEtC,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC;YAE7B,yGAAyG;YACzG,qGAAqG;YACrG,kBAAkB;YAElB,MAAM,MAAM,GAAU,EAAE,CAAC;YAEzB,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;YAEvC,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAGjC,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,QAAyB,EAAE,OAAc;gBAE9D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;oBAElD,KAAK,CAAC,CAAC,KAAK,CAAI,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBAE1C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;wBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACpB,CAAC;gBAEF,CAAC;YAEF,CAAC;QAEF,CAAC,CAAC,EAAE,CAAC,CAAC;IAEP,CAAC;IAGD,SAAS,SAAS;QAEjB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAElC,OAAO,KAAK,SAAS,CAAC,EAAE,KAAK;YAE5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;gBAE7C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAE1B,IAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAG,CAAC;oBAE9B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAEtB,MAAM,GAAG,CAAC;gBAEX,CAAC;YAEF,CAAC;QAEF,CAAC,CAAC;IAEH,CAAC;IAED,KAAK,SAAS,CAAC,CAAC,UAAU,CACzB,KAAqB,EACrB,KAAqE;QAGrE,IAAI,QAAQ,GAAmB,KAAK,CAAC;QAErC,OAAQ,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAG,CAAC;YAE9B,MAAM,OAAO,GAAU,EAAE,CAAC;YAE1B,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEhC,QAAQ,GAAG,OAAO,CAAC;QAEpB,CAAC;IAEF,CAAC;AAEF,CAAC"}

package/dist/fetch.d.ts

@@ -0,0 +1,60 @@
+import type { Task } from "@metreeca/flow";
+import { type Middleware } from "@metreeca/http";
+/**
+ * Creates a resource fetcher.
+ *
+ * The generated task converts a feed of requests into a feed of
+ * {@link https://developer.mozilla.org/docs/Web/API/Response Response} objects, so that a consumer draws on remote and
+ * local resources through the client the execution supplies rather than through one of its own.
+ *
+ * A request is given as the standard `fetch()` function accepts it, that is as a URL string, a
+ * {@link https://developer.mozilla.org/docs/Web/API/URL URL} object or a
+ * {@link https://developer.mozilla.org/docs/Web/API/Request Request} object. A response is emitted as soon as its head
+ * is received, with the body left unread for the consumer to draw or discard; a response carrying no body is emitted
+ * all the same.
+ *
+ * Exchanges are routed through the fetch client of the {@link @metreeca/gear!index.executor execution} enclosing the
+ * call to this factory, so the transport is chosen by the job rather than by the task. Binding {@link createFetch}
+ * routes every exchange through a throttled, cached or stubbed client without altering the task. An execution with no
+ * binding falls back on the standard `fetch` function.
+ *
+ * Exchanges are sent with desktop browser `User-Agent` and `Accept` fields. Sites that serve unattended clients
+ * differently, or refuse them altogether, are thus scraped as a browser would be. A request that already states one of
+ * these fields keeps its own value, as do the values stated by `middlewares`.
+ *
+ * Content coding is negotiated by the fetch client, which decodes response bodies before they reach the consumer. The
+ * task states no `Accept-Encoding` field of its own.
+ *
+ * Every exchange is logged as it is performed, so a run leaves a trace of the resources it drew on and of the ones it
+ * was denied. Requests are logged as stated, before `middlewares` can alter them.
+ *
+ * > [!NOTE]
+ * >
+ * > - **Incremental**: each response is emitted as soon as its head is received, so the feed produced runs dry when
+ * >   the source does, and an endless source is read for as long as the feed is consumed.
+ * > - **Streaming**: responses are drawn one at a time and handed over with the body unread, none retained, so
+ * >   resources of any size are handled without holding them in memory, as long as each body is read or cancelled
+ * >   before the next response is drawn.
+ * > - **Stateless**: the task keeps no state across requests, so the outcome is unaffected by how the feed is split
+ * >   across nested feeds or runs; any state carried across exchanges belongs to `middlewares` and the fetch client.
+ *
+ * > [!WARNING]
+ * >
+ * > A request stating a URL that is not absolute, or malformed in any other way, is dropped before it is sent, as is
+ * > a response stating an unsuccessful status; both are reported to the log, leaving the feed to run to completion.
+ *
+ * @param middlewares The middlewares to be layered over the fetch client, in request processing order
+ *
+ * @returns A task converting a feed of requests into a feed of responses
+ *
+ * @throws {@link !Error Error} If no execution is running, as the fetch client is resolved when the task is created
+ *
+ * @throws {@link !Error Error} While the feed is consumed, whatever the source reports while producing requests, or
+ *                              whatever the exchange reports while connecting to a resource or receiving its response
+ *
+ * @see {@link https://developer.mozilla.org/docs/Web/API/Window/fetch `fetch()`}
+ *
+ * @group Factories
+ */
+export declare function fetch(...middlewares: readonly Middleware[]): Task<string | URL | Request, Response>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAG3C,OAAO,EAAe,KAAK,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAyC9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,wBAAgB,KAAK,CAAC,GAAG,WAAW,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,QAAQ,CAAC,CA4BnG"}

package/dist/fetch.js

@@ -0,0 +1,120 @@
+/*
+ * Copyright © 2026 Metreeca srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { items } from "@metreeca/flow/feeds";
+import { service } from "@metreeca/gear";
+import { createFetch } from "@metreeca/http";
+import { headers } from "@metreeca/http/headers";
+import { monitor } from "@metreeca/http/monitor";
+import { transport } from "@metreeca/http/transport";
+import { log } from "@metreeca/tape";
+/**
+ * The `User-Agent` field stated on every exchange.
+ *
+ * States a current desktop Chrome, so that sites serving unattended clients differently, or refusing them altogether,
+ * are scraped as a browser would be. The platform token and the `AppleWebKit` and `Safari` tokens are frozen by the
+ * reduced user agent Chrome sends, leaving the Chrome version as the only one to be kept in step.
+ *
+ * @see {@link https://www.rfc-editor.org/rfc/rfc9110#section-10.1.5 RFC 9110 § 10.1.5 - User-Agent}
+ * @see {@link https://developer.chrome.com/docs/privacy-security/user-agent-client-hints User-Agent Client Hints}
+ */
+const UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
+    + " AppleWebKit/537.36 (KHTML, like Gecko)"
+    + " Chrome/151.0.0.0"
+    + " Safari/537.36";
+/**
+ * The `Accept` field stated on every exchange.
+ *
+ * Ranks media types as Chrome ranks them, down to the `q=0.8` weight carried by the catch-all entry, so that a site
+ * reading the field against the browser claimed by {@link UserAgent} is not handed a mismatched pair to fingerprint.
+ *
+ * @see {@link https://www.rfc-editor.org/rfc/rfc9110#section-12.5.1 RFC 9110 § 12.5.1 - Accept}
+ */
+const Accept = "text/html,"
+    + "application/xhtml+xml,"
+    + "application/xml;q=0.9,"
+    + "*/*;q=0.8";
+const logger = log(import.meta.url);
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/**
+ * Creates a resource fetcher.
+ *
+ * The generated task converts a feed of requests into a feed of
+ * {@link https://developer.mozilla.org/docs/Web/API/Response Response} objects, so that a consumer draws on remote and
+ * local resources through the client the execution supplies rather than through one of its own.
+ *
+ * A request is given as the standard `fetch()` function accepts it, that is as a URL string, a
+ * {@link https://developer.mozilla.org/docs/Web/API/URL URL} object or a
+ * {@link https://developer.mozilla.org/docs/Web/API/Request Request} object. A response is emitted as soon as its head
+ * is received, with the body left unread for the consumer to draw or discard; a response carrying no body is emitted
+ * all the same.
+ *
+ * Exchanges are routed through the fetch client of the {@link @metreeca/gear!index.executor execution} enclosing the
+ * call to this factory, so the transport is chosen by the job rather than by the task. Binding {@link createFetch}
+ * routes every exchange through a throttled, cached or stubbed client without altering the task. An execution with no
+ * binding falls back on the standard `fetch` function.
+ *
+ * Exchanges are sent with desktop browser `User-Agent` and `Accept` fields. Sites that serve unattended clients
+ * differently, or refuse them altogether, are thus scraped as a browser would be. A request that already states one of
+ * these fields keeps its own value, as do the values stated by `middlewares`.
+ *
+ * Content coding is negotiated by the fetch client, which decodes response bodies before they reach the consumer. The
+ * task states no `Accept-Encoding` field of its own.
+ *
+ * Every exchange is logged as it is performed, so a run leaves a trace of the resources it drew on and of the ones it
+ * was denied. Requests are logged as stated, before `middlewares` can alter them.
+ *
+ * > [!NOTE]
+ * >
+ * > - **Incremental**: each response is emitted as soon as its head is received, so the feed produced runs dry when
+ * >   the source does, and an endless source is read for as long as the feed is consumed.
+ * > - **Streaming**: responses are drawn one at a time and handed over with the body unread, none retained, so
+ * >   resources of any size are handled without holding them in memory, as long as each body is read or cancelled
+ * >   before the next response is drawn.
+ * > - **Stateless**: the task keeps no state across requests, so the outcome is unaffected by how the feed is split
+ * >   across nested feeds or runs; any state carried across exchanges belongs to `middlewares` and the fetch client.
+ *
+ * > [!WARNING]
+ * >
+ * > A request stating a URL that is not absolute, or malformed in any other way, is dropped before it is sent, as is
+ * > a response stating an unsuccessful status; both are reported to the log, leaving the feed to run to completion.
+ *
+ * @param middlewares The middlewares to be layered over the fetch client, in request processing order
+ *
+ * @returns A task converting a feed of requests into a feed of responses
+ *
+ * @throws {@link !Error Error} If no execution is running, as the fetch client is resolved when the task is created
+ *
+ * @throws {@link !Error Error} While the feed is consumed, whatever the source reports while producing requests, or
+ *                              whatever the exchange reports while connecting to a resource or receiving its response
+ *
+ * @see {@link https://developer.mozilla.org/docs/Web/API/Window/fetch `fetch()`}
+ *
+ * @group Factories
+ */
+export function fetch(...middlewares) {
+    const fetch = service(createFetch);
+    return requests => items((async function* () {
+        const send = createFetch(monitor(logger), ...middlewares, headers({
+            "Accept": Accept,
+            "User-Agent": UserAgent
+        }), transport(fetch));
+        for await (const request of requests) {
+            const response = await send(request);
+            yield* response.ok ? [response] : []; // unsuccessful responses are reported by the monitor and skipped
+        }
+    })());
+}
+//# sourceMappingURL=fetch.js.map

package/dist/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,WAAW,EAAmB,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAGrC;;;;;;;;;GASG;AACH,MAAM,SAAS,GAAG,iDAAiD;MACjE,yCAAyC;MACzC,mBAAmB;MACnB,gBAAgB,CAAC;AAEnB;;;;;;;GAOG;AACH,MAAM,MAAM,GAAG,YAAY;MACzB,wBAAwB;MACxB,wBAAwB;MACxB,WAAW,CAAC;AAGd,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAGpC,wHAAwH;AAExH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAM,UAAU,KAAK,CAAC,GAAG,WAAkC;IAE1D,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEnC,OAAO,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC;QAExC,MAAM,IAAI,GAAG,WAAW,CACvB,OAAO,CAAC,MAAM,CAAC,EACf,GAAG,WAAW,EACd,OAAO,CAAC;YAEP,QAAQ,EAAE,MAAM;YAChB,YAAY,EAAE,SAAS;SAEvB,CAAC,EACF,SAAS,CAAC,KAAK,CAAC,CAChB,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAEtC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,iEAAiE;QAExG,CAAC;IAEF,CAAC,CAAC,EAAE,CAAC,CAAC;AAEP,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * URL processing tasks.
+ *
+ * Retrieves the resources that URLs identify and crawls the graphs their links form, so a job can draw on remote and
+ * local content as a feed. Exchanges run through the fetch client the enclosing execution supplies, so a job can swap
+ * in a throttled, cached or stubbed client without altering its tasks.
+ *
+ * @module index
+ *
+ * @see {@link https://www.rfc-editor.org/rfc/rfc3986 RFC 3986 Uniform Resource Identifier (URI): Generic Syntax}
+ */
+export * from "./fetch.js";
+export * from "./crawl.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAgBA;;;;;;;;;;GAUG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,29 @@
+/*
+ * Copyright © 2026 Metreeca srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * URL processing tasks.
+ *
+ * Retrieves the resources that URLs identify and crawls the graphs their links form, so a job can draw on remote and
+ * local content as a feed. Exchanges run through the fetch client the enclosing execution supplies, so a job can swap
+ * in a throttled, cached or stubbed client without altering its tasks.
+ *
+ * @module index
+ *
+ * @see {@link https://www.rfc-editor.org/rfc/rfc3986 RFC 3986 Uniform Resource Identifier (URI): Generic Syntax}
+ */
+export * from "./fetch.js";
+export * from "./crawl.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH;;;;;;;;;;GAUG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC"}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@metreeca/pipe-url",
+  "version": "0.10.0",
+  "description": "URL processing tasks for @metreeca/pipe.",
+  "keywords": [
+    "typescript",
+    "pipeline",
+    "etl",
+    "data-integration",
+    "url"
+  ],
+  "homepage": "https://metreeca.github.io/pipe/",
+  "bugs": {
+    "url": "https://github.com/metreeca/pipe/issues",
+    "email": "info@metreeca.com"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Metreeca",
+    "email": "info@metreeca.com",
+    "url": "https://www.metreeca.com/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/metreeca/pipe.git",
+    "directory": "packages/pipe-url"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "@metreeca/source": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "README.md",
+    "LICENSE",
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.22.0"
+  },
+  "scripts": {
+    "clean": "rm -rf dist docs node_modules",
+    "build": "rm -rf dist && tsc -p tsconfig.build.json",
+    "check": "npx vitest --typecheck run"
+  },
+  "dependencies": {
+    "@metreeca/core": "^0.12.0",
+    "@metreeca/flow": "^0.11.0",
+    "@metreeca/gear": "^0.3.0",
+    "@metreeca/http": "^0.4.0",
+    "@metreeca/tape": "^0.11.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.9.3",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.13"
+  }
+}