@tricoteuses/tisseuse 0.1.0

package/dist/index.js

@@ -0,0 +1,707 @@
+function _(u, n) {
+  throw `Unexpected type ${u}: ${n}`;
+}
+const N = /^<\/?(!DOCTYPE|\?XML|[A-Z][A-Z0-9]*)/i;
+function H(u, n) {
+  return (t) => {
+    const s = [];
+    let e = t;
+    for (const i of n) {
+      const r = i(e);
+      (r.transformations !== void 0 || r.sourceMap.length !== 0) && (s.push(r), e = r.output);
+    }
+    return { input: t, output: e, title: u, transformations: s };
+  };
+}
+function* $(u) {
+  if (u.transformations === void 0)
+    yield u;
+  else
+    for (const n of u.transformations)
+      yield* $(n);
+}
+function* U(u) {
+  const n = [
+    ...$(u)
+  ].reverse();
+  let t = {
+    position: { start: 0, stop: 0 }
+  };
+  const s = n.map(
+    (i) => {
+      const r = y(
+        i
+      );
+      return r.next(t), r;
+    }
+  );
+  let e = yield t;
+  for (; e !== void 0; ) {
+    t = {
+      position: e
+    };
+    for (const i of s) {
+      const r = i.next(t);
+      if (r.done)
+        return;
+      t = r.value;
+    }
+    e = yield t;
+  }
+  for (const i of s)
+    i.next(void 0);
+}
+function* y(u) {
+  let n, t = "looking_for_start_segment", s = yield {
+    position: { start: 0, stop: 0 }
+  };
+  const e = [
+    { inputIndex: 0, inputLength: 0, outputIndex: 0, outputLength: 0 },
+    ...u.sourceMap,
+    {
+      inputIndex: Number.MAX_SAFE_INTEGER,
+      inputLength: 0,
+      outputIndex: Number.MAX_SAFE_INTEGER,
+      outputLength: 0
+    }
+  ];
+  for (const [i, r] of e.entries())
+    for (let c = !1; !c; ) {
+      if (s === void 0)
+        return;
+      let { position: o } = s, p, l, d, g;
+      switch (t) {
+        case "looking_for_start_segment": {
+          const x = e[i + 1];
+          x !== void 0 && x.outputIndex + x.outputLength <= o.start ? c = !0 : (n = i, t = "looking_for_stop_segment");
+          break;
+        }
+        case "looking_for_stop_segment": {
+          if (r.outputIndex < o.stop)
+            c = !0;
+          else {
+            const x = e[n], T = i, f = r;
+            let m = o.start < x.outputIndex + x.outputLength ? n : n + 1, a = o.stop > f.outputIndex ? T : T - 1;
+            for (let M = !0; M; ) {
+              M = !1, p = void 0, l = void 0, d = void 0, g = void 0;
+              for (let P = m; P <= a; P++) {
+                const E = e[P].matchingSegmentIndex;
+                if (E !== void 0) {
+                  if (E + 1 < m) {
+                    let R = !1;
+                    const L = e[E + 1];
+                    if (L.openingTag !== void 0 && L.outputIndex + L.outputLength < o.start) {
+                      const A = L.openingTag.match(N);
+                      if (A !== null) {
+                        const v = A[1], O = v.toUpperCase();
+                        [
+                          "B",
+                          "EM",
+                          "I",
+                          "SPAN",
+                          "STRONG",
+                          "SUB",
+                          "SUP"
+                        ].includes(O) && (d = `${d ?? ""}</${v}>`, p = `${L.openingTag}${p ?? ""}`, R = !0);
+                      }
+                    }
+                    if (!R) {
+                      p = void 0, d = void 0, m = E + 1, o = {
+                        start: e[m].outputIndex,
+                        stop: o.stop
+                      }, M = !0;
+                      break;
+                    }
+                  }
+                  if (E + 1 > a) {
+                    let R = !1;
+                    const L = e[P], A = e[E + 1];
+                    if (L.openingTag !== void 0 && o.stop < A.outputIndex) {
+                      const v = L.openingTag.match(N);
+                      if (v !== null) {
+                        const O = v[1], F = O.toUpperCase();
+                        [
+                          "B",
+                          "EM",
+                          "I",
+                          "SPAN",
+                          "STRONG",
+                          "SUB",
+                          "SUP"
+                        ].includes(F) && (l = `${l ?? ""}</${O}>`, g = `${L.openingTag}${g ?? ""}`, R = !0);
+                      }
+                    }
+                    if (!R) {
+                      l = void 0, g = void 0, a = E + 1;
+                      const v = e[a];
+                      o = {
+                        start: o.start,
+                        stop: v.outputIndex + v.outputLength
+                      }, M = !0;
+                      break;
+                    }
+                  }
+                }
+              }
+            }
+            const h = e[m - 1], S = h.inputIndex + h.inputLength + o.start - (h.outputIndex + h.outputLength), I = e[a], C = I.inputIndex + I.inputLength + o.stop - (I.outputIndex + I.outputLength);
+            s = yield Object.fromEntries(
+              Object.entries({
+                innerPrefix: `${p ?? ""}${s.innerPrefix ?? ""}` || void 0,
+                innerSuffix: `${s.innerSuffix ?? ""}${l ?? ""}` || void 0,
+                outerPrefix: `${s.outerPrefix ?? ""}${d ?? ""}` || void 0,
+                outerSuffix: `${g ?? ""}${s.outerSuffix ?? ""}` || void 0,
+                position: {
+                  start: S,
+                  stop: C
+                }
+              }).filter(([, M]) => M !== void 0)
+            ), t = "looking_for_start_segment";
+          }
+          break;
+        }
+        default:
+          _(
+            "iterOriginalMergedPositionsFromTransformedUsingTransformationLeaf.state",
+            t
+          );
+      }
+    }
+}
+function q(u, n) {
+  const t = U(u);
+  return t.next({ start: 0, stop: 0 }), n.map((s) => {
+    const e = t.next(s);
+    if (e.done)
+      throw new Error(
+        `Reverse transformation of position failed: ${s}`
+      );
+    return e.value;
+  });
+}
+function W(u, n) {
+  for (const { sourceMap: t } of [
+    ...$(u)
+  ].reverse())
+    n = b(
+      t,
+      n
+    );
+  return n;
+}
+function b(u, n) {
+  const t = [];
+  u = [
+    { inputIndex: 0, inputLength: 0, outputIndex: 0, outputLength: 0 },
+    ...u,
+    {
+      inputIndex: Number.MAX_SAFE_INTEGER,
+      inputLength: 0,
+      outputIndex: Number.MAX_SAFE_INTEGER,
+      outputLength: 0
+    }
+  ];
+  let s = 0, e = u[s];
+  for (const i of n) {
+    let { start: r } = i;
+    const { stop: c } = i;
+    t: for (let o = !1; !o; ) {
+      for (; e.outputIndex + e.outputLength <= r; s++, e = u[s]) ;
+      let p = s;
+      const l = u[p - 1];
+      let d = l.inputIndex + l.inputLength + r - (l.outputIndex + l.outputLength), g;
+      for (g = p - 1; u[g + 1].outputIndex < c; g++) ;
+      const x = u[g];
+      let T = x.inputIndex + x.inputLength + c - (x.outputIndex + x.outputLength);
+      for (let f = p; f <= g; f++) {
+        const m = u[f], a = m.matchingSegmentIndex;
+        if (a !== void 0) {
+          if (a + 1 < p) {
+            const h = u[a + 1];
+            if (h.outputIndex < r) {
+              m.inputIndex > d && t.push({
+                start: d,
+                stop: m.inputIndex
+              }), r = m.outputIndex + m.outputLength;
+              for (let S = f, I = m; I.outputIndex + I.outputLength === r; S++, I = u[S])
+                s = S;
+              continue t;
+            }
+            p = a + 1, d = h.inputIndex;
+          } else if (a + 1 > g) {
+            const h = u[a + 1];
+            if (h.outputIndex + h.outputLength > c) {
+              m.inputIndex > d && t.push({
+                start: d,
+                stop: m.inputIndex
+              }), r = m.outputIndex + m.outputLength;
+              for (let S = f, I = m; I.outputIndex + I.outputLength === r; S++, I = u[S])
+                s = S;
+              continue t;
+            }
+            g = a + 1, T = h.inputIndex + h.inputLength;
+          }
+        }
+      }
+      t.push({
+        start: d,
+        stop: T
+      }), o = !0;
+    }
+  }
+  return t;
+}
+const B = {
+  amp: "&",
+  apos: "'",
+  asymp: "≈",
+  copy: "©",
+  deg: "°",
+  euro: "€",
+  gt: ">",
+  lt: "<",
+  mdash: "—",
+  nbsp: " ",
+  ndash: "–",
+  ne: "≠",
+  pound: "£",
+  quot: '"',
+  reg: "®",
+  trade: "™"
+};
+function k({
+  removeAWithHref: u
+} = {}) {
+  return (n) => {
+    let t = 0, s = 0, e = [], i = [];
+    const r = [], c = "Conversion des éléments HTML en texte";
+    for (; t < n.length; ) {
+      const o = n.indexOf("<", t);
+      if (o === -1)
+        return e.push(n.slice(t).replace(/[\n\r]/g, " ")), {
+          input: n,
+          output: e.join(""),
+          sourceMap: i,
+          title: c
+        };
+      const p = n.indexOf(">", o);
+      if (p === -1)
+        return e.push(n.slice(t).replace(/[\n\r]/g, " ")), {
+          input: n,
+          output: e.join(""),
+          sourceMap: i,
+          title: c
+        };
+      const l = n.slice(o, p + 1), d = l.startsWith("</"), g = l.length, x = l.match(N);
+      if (x === null) {
+        e.push(
+          n.slice(t, p + 1).replace(/[\n\r]/g, " ")
+        ), t = p + 1;
+        continue;
+      }
+      const f = x[1].toUpperCase();
+      if (l.endsWith("/>") || [
+        "!DOCTYPE",
+        "?XML",
+        "AREA",
+        "BASE",
+        "BR",
+        "COL",
+        "EMBED",
+        "HR",
+        "IMG",
+        "INPUT",
+        "LINK",
+        "META",
+        "PARAM",
+        "SOURCE",
+        "TRACK",
+        "WBR"
+      ].includes(f))
+        if (d)
+          o > t && e.push(
+            n.slice(t, o).replace(/[\n\r]/g, " ")
+          ), i.push({
+            inputIndex: o,
+            inputLength: g,
+            outputIndex: o + s,
+            outputLength: 0
+          }), s -= g;
+        else if (["BR", "HR"].includes(f)) {
+          o > t && e.push(
+            n.slice(t, o).replace(/[\n\r]/g, " ")
+          );
+          const a = p + 1 - o;
+          e.push(`
+`), i.push({
+            inputIndex: o,
+            inputLength: a,
+            openingTag: l,
+            outputIndex: o + s,
+            outputLength: 1
+          }), s += 1 - a;
+        } else ["!DOCTYPE", "?XML", "COL", "IMG", "INPUT"].includes(f) ? (o > t && e.push(
+          n.slice(t, o).replace(/[\n\r]/g, " ")
+        ), i.push({
+          inputIndex: o,
+          inputLength: g,
+          openingTag: l,
+          outputIndex: o + s,
+          outputLength: 0
+        }), s -= g) : e.push(
+          n.slice(t, p + 1).replace(/[\n\r]/g, " ")
+        );
+      else if (d) {
+        const a = r.at(-1);
+        if (f === a?.name)
+          switch (r.pop(), a.action) {
+            case void 0: {
+              e.push(
+                n.slice(t, p + 1).replace(/[\n\r]/g, " ")
+              );
+              break;
+            }
+            case "ignore": {
+              e = a.outputFragments, s = a.outputOffset, i = a.sourceMap;
+              const h = p + 1 - a.inputIndex;
+              i.push({
+                inputIndex: a.inputIndex,
+                inputLength: h,
+                outputIndex: a.inputIndex + s,
+                outputLength: 0
+              }), s = s - h;
+              break;
+            }
+            case "keep_content": {
+              e.push(
+                n.slice(t, o).replace(/[\n\r]/g, " ")
+              ), a.closingTagReplacement.length !== 0 && e.push(a.closingTagReplacement);
+              const h = p + 1 - o;
+              i[a.openingSegmentIndex].matchingSegmentIndex = i.length, i.push({
+                inputIndex: o,
+                inputLength: h,
+                matchingSegmentIndex: a.openingSegmentIndex,
+                outputIndex: o + s,
+                outputLength: a.closingTagReplacement.length
+              }), s += a.closingTagReplacement.length - h;
+              break;
+            }
+            default:
+              _("TagInfos.action", a);
+          }
+        else
+          o > t && e.push(
+            n.slice(t, o).replace(/[\n\r]/g, " ")
+          ), i.push({
+            inputIndex: o,
+            inputLength: g,
+            outputIndex: o + s,
+            outputLength: 0
+          }), s -= g;
+      } else if (["COLGROUP", "HEAD", "SCRIPT", "STYLE"].includes(f) || u && f === "A" && / href=/i.test(l))
+        o > t && e.push(
+          n.slice(t, o).replace(/[\n\r]/g, " ")
+        ), r.push({
+          action: "ignore",
+          inputIndex: o,
+          name: f,
+          // Backup outputFragments, outputOffset & sourceMap, because
+          // every changes made inside ignored element will be ignored.
+          outputFragments: e,
+          outputOffset: s,
+          sourceMap: i
+        }), e = [], i = [];
+      else if ([
+        "A",
+        // When removeAWithHref is false or no href
+        "B",
+        "BODY",
+        "DL",
+        "EM",
+        "HTML",
+        "I",
+        "OL",
+        "SPAN",
+        "STRONG",
+        "SUB",
+        "SUP",
+        "TABLE",
+        "TBODY",
+        "THEAD",
+        "TR",
+        "UL"
+      ].includes(f)) {
+        o > t && e.push(
+          n.slice(t, o).replace(/[\n\r]/g, " ")
+        );
+        const a = i.length;
+        i.push({
+          inputIndex: o,
+          inputLength: g,
+          openingTag: l,
+          outputIndex: o + s,
+          outputLength: 0
+        }), s -= g, r.push({
+          action: "keep_content",
+          closingTagReplacement: "",
+          name: f,
+          openingSegmentIndex: a
+        });
+      } else if ([
+        "CAPTION",
+        "DD",
+        "DT",
+        "H1",
+        "H2",
+        "H3",
+        "H4",
+        "H5",
+        "H6",
+        "DIV",
+        "FORM",
+        "LI",
+        "P",
+        "TD",
+        "TH"
+      ].includes(f)) {
+        o > t && e.push(
+          n.slice(t, o).replace(/[\n\r]/g, " ")
+        ), e.push(`
+`);
+        const a = i.length;
+        i.push({
+          inputIndex: o,
+          inputLength: g,
+          openingTag: l,
+          outputIndex: o + s,
+          outputLength: 1
+        }), s += 1 - g, r.push({
+          action: "keep_content",
+          closingTagReplacement: `
+`,
+          name: f,
+          openingSegmentIndex: a
+        });
+      } else
+        e.push(
+          n.slice(t, p + 1).replace(/[\n\r]/g, " ")
+        ), r.push({
+          name: f
+        });
+      t = p + 1;
+    }
+    return {
+      input: n,
+      output: e.join(""),
+      sourceMap: i,
+      title: c
+    };
+  };
+}
+function D(u) {
+  let n = 0;
+  const t = [], s = u.replace(
+    /&(amp|apos|asymp|copy|deg|euro|gt|lt|mdash|nbsp|ndash|ne|pound|quot|reg|trade);/gi,
+    (e, i, r) => {
+      const c = B[i.toLowerCase()];
+      return t.push({
+        inputIndex: r,
+        inputLength: e.length,
+        outputIndex: r + n,
+        outputLength: c.length
+      }), n += c.length - e.length, c;
+    }
+  );
+  return {
+    input: u,
+    output: s,
+    sourceMap: t,
+    title: "Décodage des entités HTML nommées"
+  };
+}
+function G(u) {
+  let n = 0;
+  const t = [], s = u.replace(
+    /&#(?:(\d+)|x([0-9A-F]+));/gi,
+    (e, i, r, c) => {
+      const o = parseInt(
+        i ?? r,
+        i === void 0 ? 16 : 10
+      ), p = String.fromCharCode(o);
+      return t.push({
+        inputIndex: c,
+        inputLength: e.length,
+        outputIndex: c + n,
+        outputLength: p.length
+      }), n += p.length - e.length, p;
+    }
+  );
+  return {
+    input: u,
+    output: s,
+    sourceMap: t,
+    title: "Décodage des entités HTML numériques"
+  };
+}
+function w(u, n) {
+  return (t) => {
+    const s = [], e = t.replaceAll(u, (r, ...c) => {
+      const o = c.at(-2);
+      let p = n;
+      for (const [l, d] of c.slice(0, -2).entries())
+        p = p.replaceAll(`$${l + 1}`, d);
+      return s.push({
+        inputIndex: o,
+        inputLength: r.length,
+        // Note: `outputIndex` is added below.
+        outputLength: p.length
+      }), p;
+    });
+    let i = 0;
+    for (const r of s)
+      r.outputIndex = r.inputIndex + i, i += r.outputLength - r.inputLength;
+    return {
+      input: t,
+      output: e,
+      sourceMap: s,
+      title: `Remplacement de ${u} par ${JSON.stringify(n)}`
+    };
+  };
+}
+function X(u) {
+  const n = [];
+  let t = u;
+  for (const [s, e] of [
+    // Note: The most englobing patterns must be first.
+    // Remove HTML comment.
+    [/<!--.*?-->/gs, ""],
+    // Remove <script> element.
+    [/<script.*?>.*?<\/script>/gis, ""],
+    // Remove <script> element.
+    [/<style.*?>.*?<\/style>/gis, ""],
+    // Ensure that there is always a space after "n°".
+    [/(\sn°)([^\s])/gi, "$1 $2"],
+    // Remove Sénat "pastillage":
+    // - \uF04B-\uF054 are circled numbers 0-9.
+    // - \uF031-\uF039 are left-half circled numbers 1-9.
+    // - \uF041-\uF04A are numbers 0-9 with a circle fragment on their top & bottom only.
+    // - \uF061-\uF06A are right-half circled numbers 0-9.
+    [/[\uF031-\uF039\uF041-\uF054\uF061-\uF06A]/g, ""]
+  ]) {
+    const i = w(s, e)(t);
+    i.sourceMap.length !== 0 && (n.push(i), t = i.output);
+  }
+  return {
+    input: u,
+    output: t,
+    title: "Suppression des commentaires, scripts et styles HTML et nettoyage d'expressions",
+    transformations: n
+  };
+}
+function K({
+  removeAWithHref: u
+} = {}) {
+  return (n) => H("Simplification du HTML", [
+    D,
+    G,
+    X,
+    j,
+    k({ removeAWithHref: u }),
+    Y
+  ])(n);
+}
+function Y(u) {
+  const n = [];
+  let t = u;
+  for (const [s, e, i] of [
+    ["Remplacement des espaces multiples par une espace unique", /  +/g, " "],
+    ["Suppression d'une espace en début de ligne", /^ /gm, ""],
+    ["Suppression d'une espace en fin de ligne", / $/gm, ""],
+    [
+      "Remplacement des sauts de lignes multiples par un saut de ligne unique",
+      /\n\n+/g,
+      `
+`
+    ],
+    ["Suppression d'un saut de ligne en début de texte", /^\n/g, ""],
+    ["Suppression d'un saut de ligne en fin de texte", /\n$/g, ""]
+  ]) {
+    let r = 0;
+    const c = [], o = t.replaceAll(e, (p, ...l) => {
+      const d = l.at(-2);
+      let g = i;
+      for (const [x, T] of l.slice(0, -2).entries())
+        g = g.replaceAll(`$${x + 1}`, T);
+      return c.push({
+        inputIndex: d,
+        inputLength: p.length,
+        outputIndex: d + r,
+        outputLength: g.length
+      }), r += g.length - p.length, g;
+    });
+    c.length !== 0 && (n.push({
+      input: t,
+      output: o,
+      sourceMap: c,
+      title: s
+    }), t = o);
+  }
+  return {
+    input: u,
+    output: t,
+    title: "Simplification du texte",
+    transformations: n
+  };
+}
+function j(u) {
+  const n = [];
+  let t = u;
+  for (const [e, i] of [
+    // Replace U+00A0 (no-break space) and tab with a normal space.
+    [/[ \t]/g, " "],
+    // Replace three non-ASCII dashes (U+2010, U+2011 et U+2013) with a minus sign.
+    [/[‐‑–]/g, "-"],
+    // Replace non-ASCII apostrophe.
+    [/’/g, "'"],
+    // Replace İ (I with a point) with normal I.
+    // The İ can be used, probably to differentiate the letter I from the Roman numeral I.
+    // For example: Article 199 decies İ of the General Tax Code.
+    // But Légifrance uses a classic I…
+    ["İ", "I"]
+  ])
+    t = t.replaceAll(e, (r, ...c) => {
+      const o = c.at(-2), p = {
+        inputIndex: o,
+        inputLength: 1,
+        outputIndex: o,
+        // Note: `outputIndex` is added below.
+        outputLength: 1
+      }, l = n.findIndex(
+        (d) => d.inputIndex > o
+      );
+      return l === -1 ? n.push(p) : n.splice(l, 0, p), i;
+    });
+  let s = 0;
+  for (const e of n)
+    e.outputIndex = e.inputIndex + s, s += e.outputLength - e.inputLength;
+  return {
+    input: u,
+    output: t,
+    sourceMap: n,
+    title: "Simplification des caractères unicodes"
+  };
+}
+export {
+  H as chainTransformers,
+  k as convertHtmlElementsToText,
+  D as decodeNamedHtmlEntities,
+  G as decodeNumericHtmlEntities,
+  U as iterOriginalMergedPositionsFromTransformed,
+  q as originalMergedPositionsFromTransformed,
+  W as originalSplitPositionsFromTransformed,
+  w as replacePattern,
+  X as replacePatterns,
+  K as simplifyHtml,
+  Y as simplifyText,
+  j as simplifyUnicodeCharacters
+};

package/dist/lib/asserts.d.ts

@@ -0,0 +1 @@
+export declare function assertNever(type: string, value: never): never;

package/dist/lib/index.d.ts

@@ -0,0 +1,3 @@
+export type { TextPosition } from './text_parsers/positions.js';
+export { convertHtmlElementsToText, decodeNamedHtmlEntities, decodeNumericHtmlEntities, replacePattern, replacePatterns, simplifyHtml, simplifyText, simplifyUnicodeCharacters, } from './text_parsers/simplifiers.js';
+export { chainTransformers, iterOriginalMergedPositionsFromTransformed, originalMergedPositionsFromTransformed, originalSplitPositionsFromTransformed, type FragmentReverseTransformation, type SourceMapSegment, type Transformation, type TransformationLeaf, type TransformationNode, type Transformer, type TransformerLeaf, type TransformerNode, } from './text_parsers/transformers.js';