@framers/agentos-ext-citation-check 0.1.4 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/parse.js CHANGED
@@ -5,7 +5,8 @@
5
5
  * 2026)`), in a markdown link, between angle brackets or bare, on the same line or a later line of the paragraph;
6
6
  * several quotations waiting in a sentence share the address that follows them. A quotation of four words or more that
7
7
  * no address follows before its sentence ends is uncited; a shorter one is a term or a title and is passed over.
8
- * Fenced code, code spans and escaped quotation marks hold no claim. JSON: an array of `{ claim, url, exact }`.
8
+ * Fenced code, code spans, escaped quotation marks and a link's title hold no claim; a quotation's emphasis marks are
9
+ * removed from its words. JSON: an array of `{ claim, url, exact }`.
9
10
  *
10
11
  * @module agentos/extensions/research/citation-check/parse
11
12
  */
@@ -36,6 +37,28 @@ const QUOTE_MARKER = /^ {0,3}>/u;
36
37
  const ADDRESS_END = /[\s<>"“”„`]/u;
37
38
  /** Punctuation that ends a sentence or a markdown span after an address, not the address itself. */
38
39
  const TRAILING = /[.,;:!?'*_~]/u;
40
+ /** A markdown link's title after its address and before the closing parenthesis: `"…"`, `'…'` or `(…)`. */
41
+ const LINK_TITLE = /^>?\s+(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\((?:[^()\\]|\\.)*\))(?=\s*\))/u;
42
+ /**
43
+ * Emphasis: a run of `*` and the same run closing it, each against the words they wrap, or the same with `_` at the
44
+ * words' edges (an `_` inside a word, as in `snake_case`, is no emphasis); an escaped mark is none. An unpaired mark, as
45
+ * in `2*3` or `char*`, is text.
46
+ */
47
+ const EMPHASIS = [
48
+ /(?<!\\)(\*+)(?=\S)(.+?)(?<=[^\s\\])\1/gsu,
49
+ /(?<![\p{L}\p{N}\\])(_+)(?=\S)(.+?)(?<=[^\s\\])\1(?![\p{L}\p{N}])/gsu,
50
+ ];
51
+ /** A quotation's text with its emphasis marks removed, inner emphasis included. */
52
+ function withoutEmphasis(raw) {
53
+ let text = raw;
54
+ for (let pass = 0; pass < 4; pass += 1) {
55
+ const next = EMPHASIS.reduce((value, pattern) => value.replace(pattern, '$2'), text);
56
+ if (next === text)
57
+ break;
58
+ text = next;
59
+ }
60
+ return text;
61
+ }
39
62
  /** Whether a string is an absolute http or https address with a host. */
40
63
  export function isHttpUrl(value) {
41
64
  try {
@@ -103,9 +126,21 @@ function closingMark(text, from, closers) {
103
126
  }
104
127
  return -1;
105
128
  }
106
- /** A quotation's words: escapes and the markdown marks for bold and code removed, white space collapsed. */
129
+ /** A code span in a quotation: a run of backticks, its text, and the same run closing it. */
130
+ const CODE_SPAN = /(?<![\\`])(`+)(?!`)([\s\S]*?[^`])\1(?!`)/u;
131
+ /**
132
+ * A quotation's words: outside code spans, the markdown marks for emphasis removed, then escapes and stray backticks;
133
+ * a code span's text kept as it stands, its backticks removed; white space collapsed.
134
+ */
107
135
  function cleanQuotation(raw) {
108
- return collapse(raw.replace(/\\(.)/gu, '$1').replace(/\*\*|__|`/gu, ''));
136
+ const prose = (text) => withoutEmphasis(text).replace(/\\(.)/gu, '$1').replace(/`/gu, '');
137
+ let out = '';
138
+ let rest = raw;
139
+ for (let span = CODE_SPAN.exec(rest); span !== null; span = CODE_SPAN.exec(rest)) {
140
+ out += prose(rest.slice(0, span.index)) + (span[2] ?? '');
141
+ rest = rest.slice(span.index + span[0].length);
142
+ }
143
+ return collapse(out + prose(rest));
109
144
  }
110
145
  /** How many times a character stands in a string. */
111
146
  function count(text, char) {
@@ -133,6 +168,18 @@ function addressAt(text, start) {
133
168
  return url;
134
169
  }
135
170
  }
171
+ /**
172
+ * Whether the address at `start` is a markdown link's destination: `](` before it, then any white space, then `<` for
173
+ * an address in angle brackets.
174
+ */
175
+ function linkDestination(text, start) {
176
+ let at = start;
177
+ if (text[at - 1] === '<')
178
+ at -= 1;
179
+ while (at > 0 && /\s/u.test(text[at - 1] ?? ''))
180
+ at -= 1;
181
+ return text[at - 1] === '(' && text[at - 2] === ']';
182
+ }
136
183
  /** The claims of one unit, appended to `out` in the order they stand. */
137
184
  function scanUnit(unit, out) {
138
185
  let text = '';
@@ -190,7 +237,11 @@ function scanUnit(unit, out) {
190
237
  out.push({ line: quote.line, claim: quote.claim, url, exact: true });
191
238
  waiting = [];
192
239
  }
240
+ const inLink = linkDestination(text, index);
193
241
  index += Math.max(url.length, 1);
242
+ // A link's title, `[text](address "Title")`, is no quotation: the scan goes on at the closing parenthesis.
243
+ if (inLink)
244
+ index += LINK_TITLE.exec(text.slice(index))?.[0].length ?? 0;
194
245
  continue;
195
246
  }
196
247
  if (char === '(' || char === '[') {
package/dist/parse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"parse.js","sourceRoot":"","sources":["../src/parse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAMhG,8GAA8G;AAC9G,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAC3C,iDAAiD;IACjD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,uEAAuE;AACvE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,0EAA0E;AAC1E,MAAM,OAAO,GAAqC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAEpF,oFAAoF;AACpF,MAAM,KAAK,GAAG,uBAAuB,CAAC;AACtC,iFAAiF;AACjF,MAAM,WAAW,GAAG,kDAAkD,CAAC;AACvE,gEAAgE;AAChE,MAAM,KAAK,GAAG,4BAA4B,CAAC;AAC3C,wBAAwB;AACxB,MAAM,IAAI,GAAG,wBAAwB,CAAC;AACtC,kCAAkC;AAClC,MAAM,YAAY,GAAG,WAAW,CAAC;AACjC,+FAA+F;AAC/F,MAAM,WAAW,GAAG,cAAc,CAAC;AACnC,oGAAoG;AACpG,MAAM,QAAQ,GAAG,eAAe,CAAC;AAEjC,yEAAyE;AACzE,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,CAAC;IACxF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAQD,gIAAgI;AAChI,SAAS,KAAK,CAAC,IAAY;IACzB,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAe,EAAE,CAAC;IAC1B,IAAI,KAAK,GAA4C,IAAI,CAAC;IAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC;QACvB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;YAC5H,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC;YACR,KAAK,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClE,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,KAAK,EAAE,CAAC;YACR,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC1B,KAAK,EAAE,CAAC;YACR,OAAO;QACT,CAAC;QACD,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,KAAK,EAAE,CAAC;QAC7D,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,KAAK,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sHAAsH;AACtH,SAAS,WAAW,CAAC,IAAY,EAAE,IAAY,EAAE,OAAe;IAC9D,KAAK,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,EAAE,IAAI,CAAC,CAAC;YACR,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,4GAA4G;AAC5G,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,qDAAqD;AACrD,SAAS,KAAK,CAAC,IAAY,EAAE,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,KAAa;IAC5C,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAAE,GAAG,IAAI,CAAC,CAAC;IACzE,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACjC,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aAC3C,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aAC9E,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;;YAC9E,OAAO,GAAG,CAAC;IAClB,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,SAAS,QAAQ,CAAC,IAAgB,EAAE,GAAiB;IACnD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC;IAC3B,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE;QACxC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM;YAAE,KAAK,IAAI,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC;IACF,IAAI,OAAO,GAA2C,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,iBAAiB;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjI,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;IACf,CAAC,CAAC;IACF,6GAA6G;IAC7G,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAClD,KAAK,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;YAC3D,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,KAAK,IAAI,CAAC,CAAC;gBACX,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACzD,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1E,KAAK,GAAG,GAAG,GAAG,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,KAAK,MAAM,KAAK,IAAI,OAAO;oBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClG,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxD,IAAI,GAAG,GAAG,KAAK,CAAC;YAChB,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;gBAAE,GAAG,IAAI,CAAC,CAAC;YACtE,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC;gBAAE,MAAM,EAAE,CAAC;YACxH,KAAK,GAAG,GAAG,CAAC;YACZ,SAAS;QACX,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,MAAM,EAAE,CAAC;AACX,CAAC;AAED,yCAAyC;AACzC,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6GAA6G;AAC7G,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,KAAK,IAAI;YAAE,IAAI,IAAI,CAAC,CAAC;QAC7B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,OAAO;gBAAE,OAAO,GAAG,KAAK,CAAC;iBACxB,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAC;iBAClC,IAAI,IAAI,KAAK,GAAG;gBAAE,QAAQ,GAAG,KAAK,CAAC;YACxC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,KAAK,GAAG;YAAE,QAAQ,GAAG,IAAI,CAAC;aAC7B,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACtC,KAAK,IAAI,CAAC,CAAC;YACX,IAAI,KAAK,KAAK,CAAC;gBAAE,SAAS,GAAG,IAAI,CAAC;QACpC,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;YAAE,KAAK,IAAI,CAAC,CAAC;aAC/C,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS,CAAC,IAAY,EAAE,KAAc;IAC7C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,kBAAkB,CAAC,mFAAmF,CAAC,CAAC;IAC7I,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAc,EAAE,KAAK,EAAc,EAAE;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,SAAS,KAAK,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC;QAClD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QACnI,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,KAAgC,CAAC;QAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,eAAe,CAAC,CAAC;QAC/G,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;QACjI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,+BAA+B,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,OAAO,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAC;QACzH,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;IACtG,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,UAAyC,EAAE;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IACxC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACpE,MAAM,IAAI,kBAAkB,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,8BAA8B,CAAC,CAAC;IACvG,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnF,IAAI,KAAc,CAAC;QACnB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,KAAK,MAAM;gBAAE,MAAM,IAAI,kBAAkB,CAAC,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjI,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}
1
+ {"version":3,"file":"parse.js","sourceRoot":"","sources":["../src/parse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAMhG,8GAA8G;AAC9G,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAC3C,iDAAiD;IACjD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,uEAAuE;AACvE,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,0EAA0E;AAC1E,MAAM,OAAO,GAAqC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAEpF,oFAAoF;AACpF,MAAM,KAAK,GAAG,uBAAuB,CAAC;AACtC,iFAAiF;AACjF,MAAM,WAAW,GAAG,kDAAkD,CAAC;AACvE,gEAAgE;AAChE,MAAM,KAAK,GAAG,4BAA4B,CAAC;AAC3C,wBAAwB;AACxB,MAAM,IAAI,GAAG,wBAAwB,CAAC;AACtC,kCAAkC;AAClC,MAAM,YAAY,GAAG,WAAW,CAAC;AACjC,+FAA+F;AAC/F,MAAM,WAAW,GAAG,cAAc,CAAC;AACnC,oGAAoG;AACpG,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,2GAA2G;AAC3G,MAAM,UAAU,GAAG,8EAA8E,CAAC;AAClG;;;;GAIG;AACH,MAAM,QAAQ,GAAG;IACf,0CAA0C;IAC1C,qEAAqE;CACtE,CAAC;AAEF,mFAAmF;AACnF,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,IAAI,GAAG,GAAG,CAAC;IACf,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,IAAI,IAAI,KAAK,IAAI;YAAE,MAAM;QACzB,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,CAAC;IACxF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAQD,gIAAgI;AAChI,SAAS,KAAK,CAAC,IAAY;IACzB,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAe,EAAE,CAAC;IAC1B,IAAI,KAAK,GAA4C,IAAI,CAAC;IAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC;QACvB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;YAC5H,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC;YACR,KAAK,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClE,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,KAAK,EAAE,CAAC;YACR,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC1B,KAAK,EAAE,CAAC;YACR,OAAO;QACT,CAAC;QACD,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,KAAK,EAAE,CAAC;QAC7D,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,KAAK,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sHAAsH;AACtH,SAAS,WAAW,CAAC,IAAY,EAAE,IAAY,EAAE,OAAe;IAC9D,KAAK,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,EAAE,IAAI,CAAC,CAAC;YACR,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,6FAA6F;AAC7F,MAAM,SAAS,GAAG,2CAA2C,CAAC;AAE9D;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1G,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,IAAI,GAAG,GAAG,CAAC;IACf,KAAK,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjF,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,QAAQ,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,qDAAqD;AACrD,SAAS,KAAK,CAAC,IAAY,EAAE,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,KAAa;IAC5C,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAAE,GAAG,IAAI,CAAC,CAAC;IACzE,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACjC,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aAC3C,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aAC9E,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;YAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;;YAC9E,OAAO,GAAG,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,KAAa;IAClD,IAAI,EAAE,GAAG,KAAK,CAAC;IACf,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG;QAAE,EAAE,IAAI,CAAC,CAAC;IAClC,OAAO,EAAE,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,EAAE,IAAI,CAAC,CAAC;IACzD,OAAO,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC;AACtD,CAAC;AAED,yEAAyE;AACzE,SAAS,QAAQ,CAAC,IAAgB,EAAE,GAAiB;IACnD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC;IAC3B,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE;QACxC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM;YAAE,KAAK,IAAI,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC;IACF,IAAI,OAAO,GAA2C,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,iBAAiB;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjI,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;IACf,CAAC,CAAC;IACF,6GAA6G;IAC7G,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAClD,KAAK,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;YAC3D,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,KAAK,IAAI,CAAC,CAAC;gBACX,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACzD,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1E,KAAK,GAAG,GAAG,GAAG,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,KAAK,MAAM,KAAK,IAAI,OAAO;oBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClG,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC5C,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjC,2GAA2G;YAC3G,IAAI,MAAM;gBAAE,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;YACzE,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxD,IAAI,GAAG,GAAG,KAAK,CAAC;YAChB,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;gBAAE,GAAG,IAAI,CAAC,CAAC;YACtE,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC;gBAAE,MAAM,EAAE,CAAC;YACxH,KAAK,GAAG,GAAG,CAAC;YACZ,SAAS;QACX,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,MAAM,EAAE,CAAC;AACX,CAAC;AAED,yCAAyC;AACzC,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6GAA6G;AAC7G,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,KAAK,IAAI;YAAE,IAAI,IAAI,CAAC,CAAC;QAC7B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,OAAO;gBAAE,OAAO,GAAG,KAAK,CAAC;iBACxB,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAC;iBAClC,IAAI,IAAI,KAAK,GAAG;gBAAE,QAAQ,GAAG,KAAK,CAAC;YACxC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,KAAK,GAAG;YAAE,QAAQ,GAAG,IAAI,CAAC;aAC7B,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACtC,KAAK,IAAI,CAAC,CAAC;YACX,IAAI,KAAK,KAAK,CAAC;gBAAE,SAAS,GAAG,IAAI,CAAC;QACpC,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;YAAE,KAAK,IAAI,CAAC,CAAC;aAC/C,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4CAA4C;AAC5C,SAAS,SAAS,CAAC,IAAY,EAAE,KAAc;IAC7C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,kBAAkB,CAAC,mFAAmF,CAAC,CAAC;IAC7I,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAc,EAAE,KAAK,EAAc,EAAE;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,SAAS,KAAK,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC;QAClD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QACnI,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,KAAgC,CAAC;QAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,eAAe,CAAC,CAAC;QAC/G,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;QACjI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,+BAA+B,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,OAAO,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAC;QACzH,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;IACtG,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,UAAyC,EAAE;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IACxC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACpE,MAAM,IAAI,kBAAkB,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,8BAA8B,CAAC,CAAC;IACvG,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnF,IAAI,KAAc,CAAC;QACnB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,KAAK,MAAM;gBAAE,MAAM,IAAI,kBAAkB,CAAC,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjI,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}
@@ -0,0 +1,90 @@
1
+ /**
2
+ * @file robots.ts
3
+ * @description robots.txt as RFC 9309 gives it (https://www.rfc-editor.org/rfc/rfc9309.html). The file of a page's
4
+ * origin is read through the transport the page is read through, under a deadline of its own, and parsed to its
5
+ * first `maxBytes` bytes (the RFC's floor is 500 KiB; a line the limit cuts is left out). Its groups are matched by
6
+ * product token without case, the `*` group when none names the token; within the groups that apply, the longest
7
+ * matching rule decides, an `Allow` over a `Disallow` of the same length, `*` matching any run of characters and a
8
+ * final `$` the end of the path and query. A 400-range answer allows every page (2.3.1.3); a 500-range answer, a
9
+ * failed request or a passed deadline refuses every page (2.3.1.4); a file past five redirects is taken as
10
+ * unavailable (2.3.1.2). A refusal of the transport's egress policy, for the file's address or a hop of it, is the
11
+ * policy's answer and not the site's: it is thrown, so the page ends with the policy's reason. An origin's answer is
12
+ * kept for `cacheMs` per transport and parsing limit, save one the host's rule decided, since the next read may be
13
+ * under another rule.
14
+ *
15
+ * @module agentos/extensions/research/citation-check/robots
16
+ */
17
+ import type { Transport } from './page.js';
18
+ /** How robots.txt is read before a page and before each redirect hop. */
19
+ export interface RobotsRule {
20
+ /** The product token groups are matched against, without case; `*` obeys the `*` group alone. */
21
+ token: string;
22
+ /** The deadline of the robots.txt request, in milliseconds. */
23
+ timeoutMs: number;
24
+ /** The parsing limit, in bytes: what lies past it is ignored. RFC 9309 sets its floor at 500 KiB. */
25
+ maxBytes: number;
26
+ /** How long an origin's answer is kept, in milliseconds. */
27
+ cacheMs: number;
28
+ }
29
+ /** The parsing limit RFC 9309's section 2.5 sets as the least a reader parses: 500 KiB, written as 512,000 bytes. */
30
+ export declare const ROBOTS_PARSING_FLOOR = 512000;
31
+ /** One `Allow` or `Disallow` line: its path pattern with its percent-encoding made one form. */
32
+ export interface RobotsLine {
33
+ allow: boolean;
34
+ pattern: string;
35
+ }
36
+ /** One group: the product tokens of its `User-agent` lines, in lower case, and its rules. */
37
+ export interface RobotsGroup {
38
+ agents: string[];
39
+ rules: RobotsLine[];
40
+ }
41
+ /** A robots.txt as read: its groups, or every page allowed (no file) or refused (no answer). */
42
+ export type RobotsFile = {
43
+ kind: 'rules';
44
+ groups: RobotsGroup[];
45
+ } | {
46
+ kind: 'allow-all';
47
+ } | {
48
+ kind: 'disallow-all';
49
+ };
50
+ /**
51
+ * Parses a robots.txt's text into its groups. Consecutive `User-agent` lines open one group, which its `Allow` and
52
+ * `Disallow` lines fill; a rule before any `User-agent` line, an empty rule and every other line (`Sitemap`,
53
+ * `Crawl-delay`, a comment) are passed over.
54
+ */
55
+ export declare function parseRobots(text: string): RobotsFile;
56
+ /**
57
+ * Whether robots.txt lets a reader with this product token read the address. `/robots.txt` itself is always allowed.
58
+ *
59
+ * @param file - The file, from {@link readRobots} or {@link parseRobots}.
60
+ * @param token - The reader's product token; `*` obeys the `*` group alone.
61
+ * @param url - The address to read.
62
+ */
63
+ export declare function robotsAllows(file: RobotsFile, token: string, url: string): boolean;
64
+ /** How {@link readRobots} reads an origin's file. */
65
+ export interface ReadRobotsOptions {
66
+ /** The transport the file is requested through, the one the pages are read through. */
67
+ transport: Transport;
68
+ /** The token, the deadline, the parsing limit and how long an answer is kept. */
69
+ robots: RobotsRule;
70
+ /** The host's rule, which the transport applies to the file's address and to each of its redirect hops. */
71
+ allowUrl?: (url: string) => boolean;
72
+ /** The caller's cancellation, which is thrown. */
73
+ signal?: AbortSignal;
74
+ /** The clock the kept answers are timed by; `Date.now` when left out. */
75
+ now?: () => number;
76
+ }
77
+ /**
78
+ * Reads the robots.txt of an address's origin through the transport, or the answer the transport keeps for it.
79
+ * A 2xx answer is parsed; a 400-range answer allows every page; a 500-range answer, a failed request, a passed
80
+ * deadline or a hop the host rule refuses refuses every page; a file past five redirects allows every page. An answer
81
+ * is kept for `cacheMs` per transport, origin and parsing limit, save one the host's rule decided.
82
+ *
83
+ * @param url - Any address of the origin.
84
+ * @param options - The transport, the rule, the host rule for the file's hops, the caller's cancellation.
85
+ * @throws The caller's cancellation reason, when `signal` aborts; a `TypeError` for an address that does not parse;
86
+ * the transport's `PageRefusal` (`refused`, with a detail other than `host`) when its egress policy refuses the file's
87
+ * address or a hop of it, which is kept for no later read.
88
+ */
89
+ export declare function readRobots(url: string, options: ReadRobotsOptions): Promise<RobotsFile>;
90
+ //# sourceMappingURL=robots.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,WAAW,CAAC;AAE5D,yEAAyE;AACzE,MAAM,WAAW,UAAU;IACzB,iGAAiG;IACjG,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,qGAAqG;IACrG,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qHAAqH;AACrH,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,gGAAgG;AAChG,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,6FAA6F;AAC7F,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,gGAAgG;AAChG,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC;AAkCrH;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAwBpD;AA8BD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAwBlF;AAED,qDAAqD;AACrD,MAAM,WAAW,iBAAiB;IAChC,uFAAuF;IACvF,SAAS,EAAE,SAAS,CAAC;IACrB,iFAAiF;IACjF,MAAM,EAAE,UAAU,CAAC;IACnB,2GAA2G;IAC3G,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,kDAAkD;IAClD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAkED;;;;;;;;;;;GAWG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiB7F"}
package/dist/robots.js ADDED
@@ -0,0 +1,249 @@
1
+ /**
2
+ * @file robots.ts
3
+ * @description robots.txt as RFC 9309 gives it (https://www.rfc-editor.org/rfc/rfc9309.html). The file of a page's
4
+ * origin is read through the transport the page is read through, under a deadline of its own, and parsed to its
5
+ * first `maxBytes` bytes (the RFC's floor is 500 KiB; a line the limit cuts is left out). Its groups are matched by
6
+ * product token without case, the `*` group when none names the token; within the groups that apply, the longest
7
+ * matching rule decides, an `Allow` over a `Disallow` of the same length, `*` matching any run of characters and a
8
+ * final `$` the end of the path and query. A 400-range answer allows every page (2.3.1.3); a 500-range answer, a
9
+ * failed request or a passed deadline refuses every page (2.3.1.4); a file past five redirects is taken as
10
+ * unavailable (2.3.1.2). A refusal of the transport's egress policy, for the file's address or a hop of it, is the
11
+ * policy's answer and not the site's: it is thrown, so the page ends with the policy's reason. An origin's answer is
12
+ * kept for `cacheMs` per transport and parsing limit, save one the host's rule decided, since the next read may be
13
+ * under another rule.
14
+ *
15
+ * @module agentos/extensions/research/citation-check/robots
16
+ */
17
+ /** The parsing limit RFC 9309's section 2.5 sets as the least a reader parses: 500 KiB, written as 512,000 bytes. */
18
+ export const ROBOTS_PARSING_FLOOR = 512_000;
19
+ const ALLOW_ALL = Object.freeze({ kind: 'allow-all' });
20
+ const DISALLOW_ALL = Object.freeze({ kind: 'disallow-all' });
21
+ /** The characters RFC 3986 calls unreserved, which a percent-encoding of them stands for. */
22
+ const UNRESERVED = /^[A-Za-z0-9\-._~]$/u;
23
+ /** The most origins one transport keeps an answer for; the oldest goes first. */
24
+ const CACHE_ENTRIES = 1_000;
25
+ /**
26
+ * A path in one form for comparison: characters outside printable ASCII percent-encoded in UTF-8, an encoded
27
+ * unreserved character decoded, and every other encoding in upper case (RFC 9309, 2.2.2).
28
+ */
29
+ function normalizePath(path) {
30
+ let encoded = '';
31
+ for (const char of path) {
32
+ const code = char.codePointAt(0);
33
+ if (code > 0x20 && code < 0x7f) {
34
+ encoded += char;
35
+ continue;
36
+ }
37
+ try {
38
+ encoded += encodeURIComponent(char);
39
+ }
40
+ catch {
41
+ // A lone surrogate has no UTF-8 form; it is left out.
42
+ }
43
+ }
44
+ return encoded.replace(/%([0-9A-Fa-f]{2})/gu, (_whole, hex) => {
45
+ const char = String.fromCharCode(Number.parseInt(hex, 16));
46
+ return UNRESERVED.test(char) ? char : `%${hex.toUpperCase()}`;
47
+ });
48
+ }
49
+ /**
50
+ * Parses a robots.txt's text into its groups. Consecutive `User-agent` lines open one group, which its `Allow` and
51
+ * `Disallow` lines fill; a rule before any `User-agent` line, an empty rule and every other line (`Sitemap`,
52
+ * `Crawl-delay`, a comment) are passed over.
53
+ */
54
+ export function parseRobots(text) {
55
+ const groups = [];
56
+ let current = null;
57
+ let readingAgents = false;
58
+ for (const raw of text.replace(/^/u, '').split(/\r\n|\r|\n/u)) {
59
+ const line = raw.replace(/#.*$/u, '').trim();
60
+ const colon = line.indexOf(':');
61
+ if (colon <= 0)
62
+ continue;
63
+ const field = line.slice(0, colon).trim().toLowerCase();
64
+ const value = line.slice(colon + 1).trim();
65
+ if (field === 'user-agent') {
66
+ if (current === null || !readingAgents) {
67
+ current = { agents: [], rules: [] };
68
+ groups.push(current);
69
+ readingAgents = true;
70
+ }
71
+ if (value !== '')
72
+ current.agents.push(value.toLowerCase());
73
+ }
74
+ else if (field === 'allow' || field === 'disallow') {
75
+ if (current === null)
76
+ continue;
77
+ readingAgents = false;
78
+ if (value !== '')
79
+ current.rules.push({ allow: field === 'allow', pattern: normalizePath(value) });
80
+ }
81
+ }
82
+ return { kind: 'rules', groups };
83
+ }
84
+ /** Whether a group's `User-agent` value names the token: the same word without case, or the word with a version after it. */
85
+ function namesToken(agent, token) {
86
+ return agent === token || (token !== '*' && agent.startsWith(`${token}/`));
87
+ }
88
+ /**
89
+ * Whether a rule's pattern matches a path and query from their start: `*` any run of characters, a final `$` the end.
90
+ * Each piece between wildcards is found at its first place after the one before it, which is where it leaves the most
91
+ * path for the pieces after it, so the time grows with the pattern and the path and never with the ways to split them.
92
+ */
93
+ function matches(pattern, path) {
94
+ const anchored = pattern.endsWith('$');
95
+ const pieces = (anchored ? pattern.slice(0, -1) : pattern).split('*');
96
+ const first = pieces[0];
97
+ if (!path.startsWith(first))
98
+ return false;
99
+ if (pieces.length === 1)
100
+ return !anchored || path.length === first.length;
101
+ let at = first.length;
102
+ for (const piece of pieces.slice(1, -1)) {
103
+ if (piece === '')
104
+ continue;
105
+ const found = path.indexOf(piece, at);
106
+ if (found === -1)
107
+ return false;
108
+ at = found + piece.length;
109
+ }
110
+ const last = pieces[pieces.length - 1];
111
+ if (anchored)
112
+ return path.length - last.length >= at && path.endsWith(last);
113
+ return path.indexOf(last, at) !== -1;
114
+ }
115
+ /**
116
+ * Whether robots.txt lets a reader with this product token read the address. `/robots.txt` itself is always allowed.
117
+ *
118
+ * @param file - The file, from {@link readRobots} or {@link parseRobots}.
119
+ * @param token - The reader's product token; `*` obeys the `*` group alone.
120
+ * @param url - The address to read.
121
+ */
122
+ export function robotsAllows(file, token, url) {
123
+ let target;
124
+ try {
125
+ target = new URL(url);
126
+ }
127
+ catch {
128
+ return false;
129
+ }
130
+ if (target.pathname === '/robots.txt')
131
+ return true;
132
+ if (file.kind === 'allow-all')
133
+ return true;
134
+ if (file.kind === 'disallow-all')
135
+ return false;
136
+ const path = normalizePath(`${target.pathname}${target.search}`) || '/';
137
+ const wanted = token.trim().toLowerCase();
138
+ let groups = file.groups.filter((group) => group.agents.some((agent) => namesToken(agent, wanted)));
139
+ if (groups.length === 0)
140
+ groups = file.groups.filter((group) => group.agents.includes('*'));
141
+ let best = null;
142
+ for (const group of groups) {
143
+ for (const rule of group.rules) {
144
+ if (!matches(rule.pattern, path))
145
+ continue;
146
+ const longer = best === null || rule.pattern.length > best.pattern.length;
147
+ const allowOverDisallow = best !== null && rule.pattern.length === best.pattern.length && rule.allow && !best.allow;
148
+ if (longer || allowOverDisallow)
149
+ best = rule;
150
+ }
151
+ }
152
+ return best === null || best.allow;
153
+ }
154
+ /** The answers each transport keeps, by origin and parsing limit, with when each was read. */
155
+ const kept = new WeakMap();
156
+ /** A transport's refusal, by its failure and its detail, or null for another error. */
157
+ function refusalOf(error) {
158
+ if (!(error instanceof Error) || error.name !== 'PageRefusal')
159
+ return null;
160
+ const { failure, detail } = error;
161
+ return { failure, detail };
162
+ }
163
+ /** A file's bytes as text, held to the parsing limit; past it, the line the limit cuts is left out. */
164
+ function robotsText(bytes, limit) {
165
+ let held = bytes;
166
+ if (bytes.byteLength > limit) {
167
+ held = bytes.subarray(0, limit);
168
+ const lastBreak = Math.max(held.lastIndexOf(0x0a), held.lastIndexOf(0x0d));
169
+ held = held.subarray(0, lastBreak + 1);
170
+ }
171
+ return new TextDecoder('utf-8').decode(held);
172
+ }
173
+ /** Requests the file and reads what its answer means. */
174
+ async function fetchRobots(robotsUrl, options) {
175
+ const deadline = AbortSignal.timeout(options.robots.timeoutMs);
176
+ const signal = options.signal ? AbortSignal.any([deadline, options.signal]) : deadline;
177
+ let answer;
178
+ try {
179
+ answer = await options.transport(robotsUrl, signal, options.allowUrl !== undefined ? { allowUrl: options.allowUrl } : undefined);
180
+ }
181
+ catch (error) {
182
+ if (options.signal?.aborted)
183
+ throw options.signal.reason;
184
+ const refusal = refusalOf(error);
185
+ if (refusal?.failure === 'redirects')
186
+ return { file: ALLOW_ALL, keep: true };
187
+ if (refusal?.failure === 'refused') {
188
+ // The host's rule refused a hop: the rule is the caller's, so the next read, under its own rule, asks again.
189
+ if (refusal.detail === 'host')
190
+ return { file: DISALLOW_ALL, keep: false };
191
+ // The egress policy refused the file's address or a hop of it before a byte was sent: the policy decided, not the
192
+ // site, so its refusal ends the page with its own reason, and nothing is kept for it.
193
+ throw error;
194
+ }
195
+ return { file: DISALLOW_ALL, keep: true };
196
+ }
197
+ try {
198
+ if (answer.status >= 200 && answer.status < 300) {
199
+ const limit = options.robots.maxBytes;
200
+ // One byte past the limit says whether the file runs past it.
201
+ const bytes = answer.readPrefix !== undefined ? await answer.readPrefix(limit + 1) : await answer.read(limit + 1);
202
+ if (bytes === null)
203
+ return { file: DISALLOW_ALL, keep: true };
204
+ return { file: parseRobots(robotsText(bytes, limit)), keep: true };
205
+ }
206
+ await answer.discard();
207
+ return { file: answer.status >= 500 ? DISALLOW_ALL : ALLOW_ALL, keep: true };
208
+ }
209
+ catch {
210
+ await answer.discard().catch(() => undefined);
211
+ if (options.signal?.aborted)
212
+ throw options.signal.reason;
213
+ return { file: DISALLOW_ALL, keep: true };
214
+ }
215
+ }
216
+ /**
217
+ * Reads the robots.txt of an address's origin through the transport, or the answer the transport keeps for it.
218
+ * A 2xx answer is parsed; a 400-range answer allows every page; a 500-range answer, a failed request, a passed
219
+ * deadline or a hop the host rule refuses refuses every page; a file past five redirects allows every page. An answer
220
+ * is kept for `cacheMs` per transport, origin and parsing limit, save one the host's rule decided.
221
+ *
222
+ * @param url - Any address of the origin.
223
+ * @param options - The transport, the rule, the host rule for the file's hops, the caller's cancellation.
224
+ * @throws The caller's cancellation reason, when `signal` aborts; a `TypeError` for an address that does not parse;
225
+ * the transport's `PageRefusal` (`refused`, with a detail other than `host`) when its egress policy refuses the file's
226
+ * address or a hop of it, which is kept for no later read.
227
+ */
228
+ export async function readRobots(url, options) {
229
+ const origin = new URL(url).origin;
230
+ const now = options.now ?? Date.now;
231
+ let answers = kept.get(options.transport);
232
+ if (answers === undefined) {
233
+ answers = new Map();
234
+ kept.set(options.transport, answers);
235
+ }
236
+ // A file parsed to a shorter limit may lack a rule a longer one reads, so each limit keeps its own answer.
237
+ const key = `${options.robots.maxBytes} ${origin}`;
238
+ const held = answers.get(key);
239
+ if (held !== undefined && now() - held.at < options.robots.cacheMs)
240
+ return held.file;
241
+ const { file, keep } = await fetchRobots(`${origin}/robots.txt`, options);
242
+ answers.delete(key);
243
+ if (keep)
244
+ answers.set(key, { at: now(), file });
245
+ if (answers.size > CACHE_ENTRIES)
246
+ answers.delete(answers.keys().next().value);
247
+ return file;
248
+ }
249
+ //# sourceMappingURL=robots.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAgBH,qHAAqH;AACrH,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAiB5C,MAAM,SAAS,GAAe,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,WAAoB,EAAE,CAAC,CAAC;AAC5E,MAAM,YAAY,GAAe,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,cAAuB,EAAE,CAAC,CAAC;AAElF,6FAA6F;AAC7F,MAAM,UAAU,GAAG,qBAAqB,CAAC;AACzC,iFAAiF;AACjF,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,CAAC;QAC3C,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,OAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC,MAAM,EAAE,GAAW,EAAE,EAAE;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,CAAC;YAAE,SAAS;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvC,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YACD,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACrD,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS;YAC/B,aAAa,GAAG,KAAK,CAAC;YACtB,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,6HAA6H;AAC7H,SAAS,UAAU,CAAC,KAAa,EAAE,KAAa;IAC9C,OAAO,KAAK,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,OAAe,EAAE,IAAY;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;IAClC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,IAAI,KAAK,KAAK,EAAE;YAAE,SAAS;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/B,EAAE,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;IACjD,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAgB,EAAE,KAAa,EAAE,GAAW;IACvE,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACpG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5F,IAAI,IAAI,GAAsB,IAAI,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,SAAS;YAC3C,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAC1E,MAAM,iBAAiB,GAAG,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACpH,IAAI,MAAM,IAAI,iBAAiB;gBAAE,IAAI,GAAG,IAAI,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;AACrC,CAAC;AAgBD,8FAA8F;AAC9F,MAAM,IAAI,GAAG,IAAI,OAAO,EAA4D,CAAC;AAErF,uFAAuF;AACvF,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IAC3E,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAwD,CAAC;IACrF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAQD,uGAAuG;AACvG,SAAS,UAAU,CAAC,KAAiB,EAAE,KAAa;IAClD,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,EAAE,CAAC;QAC7B,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,OAA0B;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvF,IAAI,MAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnI,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,OAAO,EAAE,OAAO,KAAK,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC7E,IAAI,OAAO,EAAE,OAAO,KAAK,SAAS,EAAE,CAAC;YACnC,6GAA6G;YAC7G,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAC1E,kHAAkH;YAClH,sFAAsF;YACtF,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;YACtC,8DAA8D;YAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAClH,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9D,OAAO,EAAE,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrE,CAAC;QACD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC/E,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,OAA0B;IACtE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,2GAA2G;IAC3G,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC;IACrF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,MAAM,aAAa,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpB,IAAI,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,IAAI,GAAG,aAAa;QAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAe,CAAC,CAAC;IACxF,OAAO,IAAI,CAAC;AACd,CAAC"}
package/dist/types.d.ts CHANGED
@@ -77,9 +77,9 @@ export interface CitationCheck {
77
77
  }
78
78
  /**
79
79
  * Why a page was not read. A page is unreachable for `timeout` (no answer by the deadline), `network` (no connection,
80
- * or a name that does not resolve), `status` (an answer that is not 2xx), `refused` (the egress policy refused the
81
- * address) or `redirects` (too many redirects); it is unreadable for `too_large` (past the size cap), `pdf`, `binary`
82
- * (another type that is not text) or `no_text` (no text a reader sees).
80
+ * or a name that does not resolve), `status` (an answer that is not 2xx), `refused` (the egress policy, the host's rule
81
+ * or robots.txt refused the address or a redirect target) or `redirects` (too many redirects); it is unreadable for
82
+ * `too_large` (past the size cap), `pdf`, `binary` (another type that is not text) or `no_text` (no text a reader sees).
83
83
  */
84
84
  export type PageFailure = 'timeout' | 'network' | 'status' | 'refused' | 'redirects' | 'too_large' | 'pdf' | 'binary' | 'no_text';
85
85
  /** How one page fared. */
@@ -94,9 +94,12 @@ export interface PageStatus {
94
94
  status: number | null;
95
95
  /** Why it was not read; null when it was. */
96
96
  failure: PageFailure | null;
97
- /** More about the failure: the content type of a binary answer, the egress policy's reason; null otherwise. */
98
- detail: string | null;
99
- /** How its text was read: the plain request, or the web scraper's fallback; null when it was not read. */
97
+ /**
98
+ * More about the failure: the content type of a binary answer, the egress policy's reason, `host` for an address the
99
+ * host's rule refused, `robots` for one robots.txt disallows; null otherwise.
100
+ */
101
+ detail: 'host' | 'robots' | (string & {}) | null;
102
+ /** How its text was read: the plain request, or a fallback (the host's reader or the web scraper); null when it was not read. */
100
103
  via: 'fetch' | 'fallback' | null;
101
104
  /** How many sentences it gave. */
102
105
  passages: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,sHAAsH;IACtH,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,gBAAgB,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC;AAElH,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EAUvD,CAAC;AAEF,6GAA6G;AAC7G,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,CAAC;AAE/D,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,+GAA+G;IAC/G,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,6FAA6F;IAC7F,KAAK,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,eAAe,CAAC;IACzB,yFAAyF;IACzF,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uGAAuG;IACvG,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,wGAAwG;IACxG,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;;OAGG;IACH,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElI,0BAA0B;AAC1B,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;IAC/C,mFAAmF;IACnF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qDAAqD;IACrD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6CAA6C;IAC7C,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,+GAA+G;IAC/G,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,0GAA0G;IAC1G,GAAG,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IACjC,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACxC,uCAAuC;IACvC,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,+DAA+D;IAC/D,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,0BAA0B;AAC1B,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,OAAO,EAAE,SAAS,GAAG,WAAW,CAAC;IACjC,2DAA2D;IAC3D,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,kFAAkF;IAClF,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,sFAAsF;IACtF,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7D,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC/D;AAED,0GAA0G;AAC1G,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE/D,6FAA6F;AAC7F,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,sHAAsH;IACtH,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,gBAAgB,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC;AAElH,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EAUvD,CAAC;AAEF,6GAA6G;AAC7G,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,CAAC;AAE/D,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,+GAA+G;IAC/G,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,6FAA6F;IAC7F,KAAK,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,eAAe,CAAC;IACzB,yFAAyF;IACzF,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uGAAuG;IACvG,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,wGAAwG;IACxG,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;;OAGG;IACH,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElI,0BAA0B;AAC1B,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;IAC/C,mFAAmF;IACnF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qDAAqD;IACrD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6CAA6C;IAC7C,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IACjD,iIAAiI;IACjI,GAAG,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IACjC,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACxC,uCAAuC;IACvC,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,+DAA+D;IAC/D,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,0BAA0B;AAC1B,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,OAAO,EAAE,SAAS,GAAG,WAAW,CAAC;IACjC,2DAA2D;IAC3D,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,kFAAkF;IAClF,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,sFAAsF;IACtF,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7D,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC/D;AAED,0GAA0G;AAC1G,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE/D,6FAA6F;AAC7F,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC"}