@bluefields/primitives 0.2.0

package/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/dist/canonicalize.d.ts

@@ -0,0 +1,37 @@
+/**
+ * URL canonicalization v1 (Review #2).
+ *
+ * Turns a customer-submitted URL into a stable identifier used everywhere
+ * we key on URL: snapshots, extractions cache, subscriptions index, cost
+ * records. Get it wrong and we either fail to dedup (paying twice for the
+ * same content) or over-dedup (treating different pages as the same).
+ *
+ * The schema records `canon_version` on every row that carries a
+ * canonicalized URL, so we can roll a v2 ruleset later without rewriting
+ * history. NEVER edit this file in place — if rules change, copy to
+ * canonicalize.v2.ts, bump CANON_VERSION, and route via a version
+ * dispatcher.
+ *
+ * Inputs: a URL string as submitted by a customer. Must parse under the
+ * WHATWG URL Standard.
+ *
+ * Outputs three values:
+ *   - url_canonical : canonical form for display + indexing
+ *   - url_surt      : sort-friendly reverse-domain key for host clustering
+ *   - canon_version : integer ruleset version (1 for this file)
+ */
+export declare const CANON_VERSION: 1;
+export declare class InvalidUrlError extends Error {
+    constructor(message: string);
+}
+export interface CanonicalUrl {
+    url_canonical: string;
+    url_surt: string;
+    canon_version: number;
+}
+/**
+ * Canonicalize a URL. Throws InvalidUrlError on parse failure or
+ * unsupported scheme. The returned object is safe to persist as
+ * (url_canonical, url_surt, canon_version) into the schema.
+ */
+export declare function canonicalizeUrl(input: string): CanonicalUrl;

package/dist/canonicalize.js

@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * URL canonicalization v1 (Review #2).
+ *
+ * Turns a customer-submitted URL into a stable identifier used everywhere
+ * we key on URL: snapshots, extractions cache, subscriptions index, cost
+ * records. Get it wrong and we either fail to dedup (paying twice for the
+ * same content) or over-dedup (treating different pages as the same).
+ *
+ * The schema records `canon_version` on every row that carries a
+ * canonicalized URL, so we can roll a v2 ruleset later without rewriting
+ * history. NEVER edit this file in place — if rules change, copy to
+ * canonicalize.v2.ts, bump CANON_VERSION, and route via a version
+ * dispatcher.
+ *
+ * Inputs: a URL string as submitted by a customer. Must parse under the
+ * WHATWG URL Standard.
+ *
+ * Outputs three values:
+ *   - url_canonical : canonical form for display + indexing
+ *   - url_surt      : sort-friendly reverse-domain key for host clustering
+ *   - canon_version : integer ruleset version (1 for this file)
+ */
+import { isIPv4, isIPv6 } from 'node:net';
+export const CANON_VERSION = 1;
+export class InvalidUrlError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'InvalidUrlError';
+    }
+}
+// ── Tracking-parameter strip lists ──────────────────────────────────────
+const STRIP_PARAMS_GLOBAL = new Set([
+    // Google Analytics / Ads
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    'utm_id',
+    '_ga',
+    '_gl',
+    'gclid',
+    'gclsrc',
+    'dclid',
+    // Facebook
+    'fbclid',
+    'fb_action_ids',
+    'fb_action_types',
+    'fb_source',
+    // Microsoft
+    'msclkid',
+    // Mailchimp
+    'mc_eid',
+    'mc_cid',
+    // Instagram
+    'igshid',
+    // YouTube — `t` is a timestamp deep-link, kept; `si` is tracking, stripped
+    'feature',
+    'si',
+    // Generic tracking
+    'ref',
+    'ref_src',
+    'ref_url',
+    'referrer',
+    'share',
+    'share_id',
+    '_branch_match_id',
+]);
+// Host-conditional strip rules. `s` and `t` are tracking on Twitter/X but
+// meaningful (e.g. timestamps) elsewhere; only strip them on these hosts.
+const TWITTER_HOSTS = new Set([
+    'twitter.com',
+    'www.twitter.com',
+    'x.com',
+    'www.x.com',
+    't.co',
+]);
+const STRIP_PARAMS_TWITTER = new Set(['s', 't']);
+// ── Public API ──────────────────────────────────────────────────────────
+/**
+ * Canonicalize a URL. Throws InvalidUrlError on parse failure or
+ * unsupported scheme. The returned object is safe to persist as
+ * (url_canonical, url_surt, canon_version) into the schema.
+ */
+export function canonicalizeUrl(input) {
+    let parsed;
+    try {
+        parsed = new URL(input);
+    }
+    catch {
+        throw new InvalidUrlError(`could not parse: ${input}`);
+    }
+    // Rule 2: scheme allowlist (http/https only, lowercased).
+    const scheme = parsed.protocol.slice(0, -1).toLowerCase();
+    if (scheme !== 'http' && scheme !== 'https') {
+        throw new InvalidUrlError(`unsupported scheme: ${scheme}`);
+    }
+    // Rule 3: WHATWG URL already lowercases the host and converts IDN to
+    // Punycode. WHATWG also returns IPv6 hostnames in `[...]` form; strip
+    // brackets here so SURT computation gets the bare host.
+    const rawHost = parsed.hostname.toLowerCase();
+    const hostNoBrackets = rawHost.startsWith('[') && rawHost.endsWith(']') ? rawHost.slice(1, -1) : rawHost;
+    // Rule 4: drop default ports.
+    const isDefaultPort = (scheme === 'http' && (parsed.port === '' || parsed.port === '80')) ||
+        (scheme === 'https' && (parsed.port === '' || parsed.port === '443'));
+    const portPart = isDefaultPort ? '' : `:${parsed.port}`;
+    // Rules 5-7: preserve path case + trailing slash; decode unreserved %xx;
+    // uppercase remaining %xx hex.
+    const path = normalizePathPercentEncoding(parsed.pathname);
+    // Rule 9: query handling — strip tracking params, sort the rest.
+    const queryString = canonicalizeQuery(parsed.searchParams, hostNoBrackets);
+    // Rule 10: reassemble. Drop fragment (Rule 8) by simply not appending it.
+    const url_canonical = `${scheme}://${rawHost}${portPart}${path}${queryString}`;
+    // SURT for host-level clustering and ops. See computeSurt() docstring.
+    const url_surt = computeSurt(hostNoBrackets, portPart, path, queryString);
+    return { url_canonical, url_surt, canon_version: CANON_VERSION };
+}
+// ── Internals ──────────────────────────────────────────────────────────
+/**
+ * Decode percent-encoded unreserved characters per RFC 3986 §6.2.2.2,
+ * then uppercase any remaining percent-encoded hex. WHATWG's URL parser
+ * normalizes some of this but is inconsistent across runtimes; we
+ * normalize explicitly so the output is deterministic.
+ *
+ * Unreserved per RFC 3986: A-Z a-z 0-9 - . _ ~
+ */
+function normalizePathPercentEncoding(path) {
+    return path.replace(/%([0-9A-Fa-f]{2})/g, (_match, hex) => {
+        const code = Number.parseInt(hex, 16);
+        if (
+        // 0-9
+        (code >= 0x30 && code <= 0x39) ||
+            // A-Z
+            (code >= 0x41 && code <= 0x5a) ||
+            // a-z
+            (code >= 0x61 && code <= 0x7a) ||
+            // - . _ ~
+            code === 0x2d ||
+            code === 0x2e ||
+            code === 0x5f ||
+            code === 0x7e) {
+            return String.fromCharCode(code);
+        }
+        return `%${hex.toUpperCase()}`;
+    });
+}
+function canonicalizeQuery(params, host) {
+    const isTwitterHost = TWITTER_HOSTS.has(host);
+    // Collect kept (key, value) pairs, drop strip-listed.
+    const kept = [];
+    for (const [key, value] of params) {
+        if (STRIP_PARAMS_GLOBAL.has(key))
+            continue;
+        if (isTwitterHost && STRIP_PARAMS_TWITTER.has(key))
+            continue;
+        kept.push([key, value]);
+    }
+    if (kept.length === 0)
+        return '';
+    // Stable sort: primary by key, secondary by original order (preserved
+    // because Array.prototype.sort is stable in Node 12+).
+    kept.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    const sp = new URLSearchParams();
+    for (const [k, v] of kept)
+        sp.append(k, v);
+    return `?${sp.toString()}`;
+}
+/**
+ * SURT (Sort-friendly URI Reordering Transform) — IA's canonical key for
+ * host-level clustering. Reverses the dotted hostname into a sortable
+ * comma-separated key. Example:
+ *
+ *   https://www.example.com/path?q=1
+ *   →  com,example,www,/path?q=1
+ *
+ * For IP-literal hosts (IPv4 or IPv6) there's no hierarchical structure
+ * to reverse, so we use the literal host as-is. IPv6 is wrapped in
+ * brackets for unambiguous parsing.
+ */
+function computeSurt(host, portPart, path, queryString) {
+    let surtHost;
+    if (isIPv4(host)) {
+        surtHost = host;
+    }
+    else if (isIPv6(host)) {
+        surtHost = `[${host}]`;
+    }
+    else {
+        // Reverse the dotted segments. "www.example.com" -> "com,example,www"
+        const segments = host.split('.').filter((s) => s.length > 0);
+        surtHost = segments.reverse().join(',');
+    }
+    return `${surtHost}${portPart},${path}${queryString}`;
+}
+//# sourceMappingURL=canonicalize.js.map

package/dist/canonicalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"canonicalize.js","sourceRoot":"","sources":["../src/canonicalize.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAE1C,MAAM,CAAC,MAAM,aAAa,GAAG,CAAU,CAAC;AAExC,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAQD,2EAA2E;AAE3E,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS;IAC1C,yBAAyB;IACzB,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,UAAU;IACV,aAAa;IACb,QAAQ;IACR,KAAK;IACL,KAAK;IACL,OAAO;IACP,QAAQ;IACR,OAAO;IACP,WAAW;IACX,QAAQ;IACR,eAAe;IACf,iBAAiB;IACjB,WAAW;IACX,YAAY;IACZ,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,2EAA2E;IAC3E,SAAS;IACT,IAAI;IACJ,mBAAmB;IACnB,KAAK;IACL,SAAS;IACT,SAAS;IACT,UAAU;IACV,OAAO;IACP,UAAU;IACV,kBAAkB;CACnB,CAAC,CAAC;AAEH,0EAA0E;AAC1E,0EAA0E;AAC1E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS;IACpC,aAAa;IACb,iBAAiB;IACjB,OAAO;IACP,WAAW;IACX,MAAM;CACP,CAAC,CAAC;AACH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,2EAA2E;AAE3E;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,eAAe,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,0DAA0D;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QAC5C,MAAM,IAAI,eAAe,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,qEAAqE;IACrE,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,cAAc,GAClB,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEpF,8BAA8B;IAC9B,MAAM,aAAa,GACjB,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACnE,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;IAExD,yEAAyE;IACzE,+BAA+B;IAC/B,MAAM,IAAI,GAAG,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3D,iEAAiE;IACjE,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IAE3E,0EAA0E;IAC1E,MAAM,aAAa,GAAG,GAAG,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;IAE/E,uEAAuE;IACvE,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IAE1E,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;AACnE,CAAC;AAED,0EAA0E;AAE1E;;;;;;;GAOG;AACH,SAAS,4BAA4B,CAAC,IAAY;IAChD,OAAO,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,GAAW,EAAE,EAAE;QAChE,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACtC;QACE,MAAM;QACN,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC;YAC9B,MAAM;YACN,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC;YAC9B,MAAM;YACN,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC;YAC9B,UAAU;YACV,IAAI,KAAK,IAAI;YACb,IAAI,KAAK,IAAI;YACb,IAAI,KAAK,IAAI;YACb,IAAI,KAAK,IAAI,EACb,CAAC;YACD,OAAO,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAuB,EAAE,IAAY;IAC9D,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9C,sDAAsD;IACtD,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC3C,IAAI,aAAa,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7D,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IAC1B,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,sEAAsE;IACtE,uDAAuD;IACvD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI;QAAE,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,OAAO,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB,EAAE,IAAY,EAAE,WAAmB;IACpF,IAAI,QAAgB,CAAC;IACrB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,QAAQ,GAAG,IAAI,IAAI,GAAG,CAAC;IACzB,CAAC;SAAM,CAAC;QACN,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7D,QAAQ,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,QAAQ,GAAG,QAAQ,IAAI,IAAI,GAAG,WAAW,EAAE,CAAC;AACxD,CAAC"}

package/dist/env.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Read a positive-integer operational knob from the environment, falling back
+ * to the compiled default. Lets an operator tune timeouts / concurrency /
+ * thresholds per-deploy without a code change; a missing, non-numeric, or
+ * non-positive value uses the fallback (these knobs are never legitimately 0).
+ */
+export declare function envInt(name: string, fallback: number): number;

package/dist/env.js

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Read a positive-integer operational knob from the environment, falling back
+ * to the compiled default. Lets an operator tune timeouts / concurrency /
+ * thresholds per-deploy without a code change; a missing, non-numeric, or
+ * non-positive value uses the fallback (these knobs are never legitimately 0).
+ */
+export function envInt(name, fallback) {
+    const raw = process.env[name];
+    if (raw === undefined || raw === '')
+        return fallback;
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+//# sourceMappingURL=env.js.map

package/dist/env.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"env.js","sourceRoot":"","sources":["../src/env.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC;;;;;GAKG;AACH,MAAM,UAAU,MAAM,CAAC,IAAY,EAAE,QAAgB;IACnD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IACrD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChE,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @bluefields/primitives — pure, secret-free infrastructure primitives shared
+ * by the open engine (fetcher/extractor) and the private control plane.
+ *
+ * Carved out of @bluefields/control so the engine can be open-sourced without
+ * dragging in billing/customers/quotas. Everything here is deterministic
+ * library code with no database, no secrets, and no business/policy: URL
+ * canonicalization, env parsing, structured logging, SSRF-safe fetching, and
+ * robots.txt parsing/evaluation. (Plan-based robots POLICY stays in control —
+ * that is business logic, not a primitive.)
+ */
+export * from './canonicalize.js';
+export * from './env.js';
+export * from './logger.js';
+export * from './ssrf.js';
+export * from './robots-txt.js';

package/dist/index.js

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * @bluefields/primitives — pure, secret-free infrastructure primitives shared
+ * by the open engine (fetcher/extractor) and the private control plane.
+ *
+ * Carved out of @bluefields/control so the engine can be open-sourced without
+ * dragging in billing/customers/quotas. Everything here is deterministic
+ * library code with no database, no secrets, and no business/policy: URL
+ * canonicalization, env parsing, structured logging, SSRF-safe fetching, and
+ * robots.txt parsing/evaluation. (Plan-based robots POLICY stays in control —
+ * that is business logic, not a primitive.)
+ */
+export * from './canonicalize.js';
+export * from './env.js';
+export * from './logger.js';
+export * from './ssrf.js';
+export * from './robots-txt.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC;;;;;;;;;;GAUG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC"}

package/dist/logger.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Structured logging (pino, JSON output).
+ *
+ * One root logger; child loggers per layer to attach consistent bindings
+ * without callers having to remember them.
+ *
+ * ## Conventions
+ *
+ * Every log call passes a structured object as the first argument with these
+ * fields when relevant:
+ *   - layer:           'poll' | 'fetcher' | 'extractor' | 'differ' | 'events' | 'api' | 'control'
+ *   - subscription_id: UUID
+ *   - event_id:        UUID
+ *   - customer_id:     UUID
+ *   - duration_ms:     int
+ *   - outcome:         'ok' | 'cached' | 'blocked' | 'skipped' | 'error' | 'partial'
+ *
+ * The message string is the second argument and stays short — searchable
+ * tokens, not prose.
+ *
+ * ## What NOT to log
+ *
+ * Configured via pino's `redact` so accidental inclusion in a bindings
+ * object or a log argument silently becomes `[Redacted]`:
+ *   - webhook secrets (current + previous)
+ *   - API keys (full + hash)
+ *   - Stripe webhook signing secret
+ *   - Authorization headers + bearer tokens
+ *   - Anthropic API key
+ *   - Resend API key
+ *   - Raw HTML / extraction body — caller's responsibility (we just don't log it)
+ *
+ * Raw HTML and raw extraction output aren't redactable by path (the field
+ * names vary by call site); the convention is: log size + hash, never bytes.
+ * Reviewers should grep `logger.*(.*html|.*markdown|.*structuredData|.*body:)`
+ * before merging changes that touch logging.
+ */
+import { type Logger as PinoLogger } from 'pino';
+export type Logger = PinoLogger;
+/** Layers the logger knows about — keep in sync with the four-layer rule. */
+export type LogLayer = 'poll' | 'fetcher' | 'extractor' | 'differ' | 'events' | 'api' | 'control' | 'crawl';
+/** Outcome enum — same set used in PollOutcome and DeliveryHttpResult. */
+export type LogOutcome = 'ok' | 'cached' | 'blocked' | 'skipped' | 'error' | 'partial' | 'timeout' | 'non_2xx';
+/**
+ * Build the root logger. Idempotent — subsequent calls return the same
+ * instance. Tests can pass a custom `level` override.
+ */
+export declare function getRootLogger(env?: NodeJS.ProcessEnv): Logger;
+/**
+ * Build a child logger pre-bound to a layer (and optional subscription /
+ * customer / event ids). Pass through callers so log calls don't have to
+ * repeat the same bindings.
+ */
+export declare function createLayerLogger(layer: LogLayer, bindings?: LayerBindings, env?: NodeJS.ProcessEnv): Logger;
+export interface LayerBindings {
+    subscription_id?: string;
+    event_id?: string;
+    customer_id?: string;
+    /** Inngest function id for grouping per-function logs. */
+    inngest_function?: string;
+}
+/** Test-only — reset the cached root logger. */
+export declare function _resetLoggerForTests(): void;