bareloop 0.49.0 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/job.js CHANGED
@@ -170,7 +170,7 @@ export const CONDITION_KEYS = Object.freeze(['providerPath', 'closeVerbosity', '
170
170
  // `steps` stays in the field list ONLY so a retired spec reds by name
171
171
  // (`shape-retired`) instead of falling through to a generic unknown-field —
172
172
  // the operator gets told what happened, not just that something is wrong.
173
- const JOB_FIELDS = ['schema', 'job', 'description', 'provider', 'baseUrl', 'conditions', 'cadence', 'trigger', 'budgetUsd', 'maxWallMs', 'closeTimeoutMs', 'model', 'thinking', 'judge', 'writeScope', 'steps', 'escalation', 'goal', 'verdictType', 'close', 'closeDecl', 'checks', 'tools', 'jobLines', 'inputs'];
173
+ const JOB_FIELDS = ['schema', 'job', 'description', 'provider', 'baseUrl', 'conditions', 'cadence', 'trigger', 'budgetUsd', 'maxWallMs', 'closeTimeoutMs', 'model', 'thinking', 'handoffNote', 'judge', 'writeScope', 'steps', 'escalation', 'goal', 'verdictType', 'close', 'closeDecl', 'checks', 'tools', 'jobLines', 'inputs'];
174
174
  /** the exact fields of a signed `judge` override — nested objects red unknown
175
175
  * keys too, because no smuggling level exists in a signed spec */
176
176
  const JUDGE_FIELDS = ['provider', 'model'];
@@ -349,6 +349,11 @@ export function validateJob(input, { shellCapUsd = 2 } = {}) {
349
349
  if (spec.thinking !== undefined && typeof spec.thinking !== 'boolean') {
350
350
  red('invalid-value', 'thinking', 'true or false — the job\'s "Thinking on" tick (absent = off)');
351
351
  }
352
+ // THE HANDOFF NOTE (src/handoff.js): a declared job condition, signed like `thinking`. A boolean; ABSENT = on (an old spec keeps its
353
+ // bytes, so its hash), only an explicit `false` turns the note off. The switch is hashed; the note's text never is.
354
+ if (spec.handoffNote !== undefined && typeof spec.handoffNote !== 'boolean') {
355
+ red('invalid-value', 'handoffNote', 'true or false (absent = on): whether the next step/fix worker is handed a map of the files earlier workers read');
356
+ }
352
357
 
353
358
  // JUDGE — PRD item 32.1 (hamr, 2026-09-10: the judge "defaults to whatever
354
359
  // llm is used and can be overriden"). OPTIONAL, and the OVERRIDE half: absent,
@@ -3,15 +3,18 @@
3
3
  // block all read these words; no other file writes a drop reason (a test greps for it). Plain words, written by CODE from the drop's reason
4
4
  // code, never a raw code and never model-written. `<label>` is the page's own word.
5
5
 
6
+ import { FACT_LANGS } from './facts.js';
7
+
6
8
  /** @typedef {import('./pagerules.js').Dropped} Dropped */
7
9
 
8
10
  /** reason code -> the plain sentence (`d.word` = the page's word, `d.fact` = the fact's name, `d.other` = what it was compared to) */
9
11
  const WORDS = /** @type {Record<string, (d: Dropped) => string>} */ ({
10
12
  'not-on-page': (d) => `"${d.word}" is not written on the page.`,
11
13
  'no-value': (d) => `The page never shows a number right after or right before "${d.word}".`,
14
+ 'reads-no-amount': (d) => `The page never shows an amount next to "${d.word}", so it was not used to write the price.`,
12
15
  'n2': (d) => `"${d.word}" is written more than once inside one line of the page, so one result cannot show it once.`,
13
16
  'r1': (d) => `"${d.word}" does not show the same values as "${d.other}", so it was not used as a second way to read it.`,
14
- 'sign-twice': (d) => `"${d.word}" was listed twice, as a unit and as a currency; it is used once, as the currency.`,
17
+ 'sign-twice': (d) => `"${d.word}" was listed twice, as a unit and as a currency; it is used once, as the ${d.other === 'unit' ? 'unit' : 'currency'}.`,
15
18
  'r1l': (d) => d.word === d.other
16
19
  ? `"${d.word}" was listed twice, as a unit and as a currency; it is used once, as the first way to write the price.`
17
20
  : `"${d.word}" does not show the same amounts as "${d.other}", so it was not used as a second way to write the price.`,
@@ -23,6 +26,8 @@ const WORDS = /** @type {Record<string, (d: Dropped) => string>} */ ({
23
26
  'unit-too-long': (d) => `"${d.word}" is longer than 100 characters, so it was not used.`,
24
27
  'reads-no-result': () => "the price rule could not read a price in the page's results, so it is not checked.",
25
28
  'reads-no-time': (d) => `the page never shows a time right after "${d.other}", so it is not checked.`,
29
+ 'lang-not-on-page': (d) => `the page shows no month name in ${FACT_LANGS[/** @type {keyof typeof FACT_LANGS} */ (d.other ?? '')] ?? d.other}, so the date is not checked.`,
30
+ 'decimal-not-on-page': () => 'the page shows no amount written with a comma before the decimals, so it is not checked.',
26
31
  'phrase-no-result': (d) => `"${d.other}" is on the page but in none of its results, so it is not checked.`,
27
32
  'person-left': () => 'You chose to leave it Not checked.',
28
33
  'bad-shape': (d) => `The AI named a kind of value for "${d.fact}" that a check cannot read.`,
@@ -68,7 +73,7 @@ export function dropLine(row, jobLines, rulesOf) {
68
73
  // HOW THE PAGE WAS READ (run mvij7kuk, flight-c1): a drop of ONE WORD whose fact is still checked is a NOTE about the form code did not use, not
69
74
  // something left unchecked. The sentences for those reasons are WORD-level; every other sentence (a whole fact left out) keeps the line under
70
75
  // Not checked. Classified from the line's own words, so it holds for a signed spec already on disk.
71
- const WORD_REASONS = ['not-on-page', 'no-value', 'n2', 'r1', 'sign-twice', 'r1l', 'limit-currency', 'limit-unit', 'unit-too-long'];
76
+ const WORD_REASONS = ['not-on-page', 'no-value', 'reads-no-amount', 'n2', 'r1', 'sign-twice', 'r1l', 'limit-currency', 'limit-unit', 'unit-too-long'];
72
77
  /** @param {string} s */
73
78
  const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
74
79
  const SENTENCE_RE = (() => {
@@ -76,7 +81,7 @@ const SENTENCE_RE = (() => {
76
81
  /** @type {string[]} */
77
82
  const alts = [];
78
83
  for (const reason of WORD_REASONS) {
79
- const probes = reason === 'r1l' ? [{ word: W, other: W }, { word: W, other: O }] : [{ word: W, other: O }];
84
+ const probes = reason === 'r1l' ? [{ word: W, other: W }, { word: W, other: O }] : reason === 'sign-twice' ? [{ word: W, other: 'currency' }, { word: W, other: 'unit' }] : [{ word: W, other: O }];
80
85
  for (const p of probes) alts.push(esc(WORDS[reason](/** @type {any} */ ({ ...p, reason, line: 0, fact: 'f' }))).split(esc(W)).join('[^"]+').split(esc(O)).join('[^"]+'));
81
86
  }
82
87
  return new RegExp(`^(?:(?:${alts.join('|')})(?: |$))+$`);
package/src/pagedry.js CHANGED
@@ -8,13 +8,16 @@
8
8
 
9
9
  import { createHash } from 'node:crypto';
10
10
  import { gradeStep } from './rubricstep.js';
11
- import { linesOf, numberAtEnd } from './pagerules.js';
11
+ import { linesOf, numberAtEnd, numberAtEndComma, NUM_COMMA } from './pagerules.js';
12
12
 
13
13
  /** @typedef {import('./pagerules.js').KeptFact} KeptFact @typedef {import('./pagerules.js').Dropped} Dropped */
14
14
 
15
15
  /** a number as the page writes it, thousands commas allowed */
16
16
  const NUM = '\\d[\\d,]*(?:\\.\\d+)?';
17
17
  const AFTER_LABEL = new RegExp(`^\\s*(?:[$€£]\\s*)?(${NUM})`);
18
+ const AFTER_LABEL_COMMA = new RegExp(`^\\s*(?:[$€£]\\s*)?(${NUM_COMMA})`);
19
+ /** the symbols a currency may be; one right before a number reads the number after it */
20
+ const SIGNS = ['$', '€', '£'];
18
21
  /** the label every dry-read claim is made under (the checker only needs one name; the fact's own is not read) */
19
22
  const CLAIM = 'price';
20
23
  const SOURCE = 'page';
@@ -38,12 +41,20 @@ async function greenLines(text, sha, fact, claim, lines) {
38
41
 
39
42
  /**
40
43
  * The result lines of a kept money fact: with a label, every page line where a kept label is followed by a number (one sign between allowed);
41
- * with no label (U1 moved it to a unit, e.g. "€533 total"), every line where a number is followed by a kept unit.
44
+ * with no label (U1 moved it to a unit, e.g. "€533 total", or a sign alone, e.g. "48,49€"), every line where a number sits right before a kept
45
+ * unit or sign, or right after a kept sign. A fact that signs `decimal: ","` is read in that form: groups of three joined by "." or a space, a comma
46
+ * before the decimals. The value is always the CANONICAL amount ("1057.65"): the typed value a claim carries, never the page's spelling.
47
+ * A LOOSE finder (it only collects): whether the run's checker can read the line is `gradeStep`'s to say.
42
48
  * @param {string} text @param {KeptFact} k @returns {{quote: string, value: string}[]}
43
49
  */
44
50
  export function moneyResultLines(text, k) {
45
51
  const raw = text.split('\n');
46
52
  const squashed = linesOf(text);
53
+ const comma = k.decimal === ',';
54
+ const afterLabel = comma ? AFTER_LABEL_COMMA : AFTER_LABEL;
55
+ const endNumber = comma ? numberAtEndComma : numberAtEnd;
56
+ /** the page's amount as a canonical decimal string */
57
+ const canon = (/** @type {string} */ a) => (comma ? a.replace(/[. ]/g, '').replace(',', '.') : a.replace(/,/g, ''));
47
58
  /** @type {{quote: string, value: string}[]} */
48
59
  const out = [];
49
60
  squashed.forEach((ln, i) => {
@@ -51,16 +62,19 @@ export function moneyResultLines(text, k) {
51
62
  let amount;
52
63
  if (k.words.length) {
53
64
  for (const w of k.words) {
54
- for (let at = ln.indexOf(w); at !== -1 && amount === undefined; at = ln.indexOf(w, at + w.length)) amount = AFTER_LABEL.exec(ln.slice(at + w.length))?.[1];
65
+ for (let at = ln.indexOf(w); at !== -1 && amount === undefined; at = ln.indexOf(w, at + w.length)) amount = afterLabel.exec(ln.slice(at + w.length))?.[1];
55
66
  if (amount !== undefined) break;
56
67
  }
57
68
  } else {
58
- for (const u of k.unit ?? []) {
59
- for (let at = ln.indexOf(u); at !== -1 && amount === undefined; at = ln.indexOf(u, at + u.length)) amount = numberAtEnd(ln.slice(0, at));
69
+ for (const u of [...(k.unit ?? []), ...(k.currency ?? [])]) {
70
+ for (let at = ln.indexOf(u); at !== -1 && amount === undefined; at = ln.indexOf(u, at + u.length)) {
71
+ amount = endNumber(ln.slice(0, at));
72
+ if (amount === undefined && SIGNS.includes(u)) amount = afterLabel.exec(ln.slice(at + u.length))?.[1];
73
+ }
60
74
  if (amount !== undefined) break;
61
75
  }
62
76
  }
63
- if (amount !== undefined) out.push({ quote: raw[i].trim(), value: amount.replace(/,/g, '') });
77
+ if (amount !== undefined) out.push({ quote: raw[i].trim(), value: canon(amount) });
64
78
  });
65
79
  return out;
66
80
  }
@@ -69,7 +83,7 @@ export function moneyResultLines(text, k) {
69
83
  * The fact as a run would sign it, for the dry read: the primary label is the anchor (none when U1 moved every label), then the kept units and currencies.
70
84
  * @param {KeptFact} k @returns {Record<string, any>}
71
85
  */
72
- const signedOf = (k) => ({ name: CLAIM, shape: 'money', ...(k.words.length ? { anchor: k.words[0] } : {}), ...(k.currency ? { currency: k.currency } : {}), ...(k.unit ? { unit: k.unit } : {}) });
86
+ const signedOf = (k) => ({ name: CLAIM, shape: 'money', ...(k.words.length ? { anchor: k.words[0] } : {}), ...(k.currency ? { currency: k.currency } : {}), ...(k.unit ? { unit: k.unit } : {}), ...(k.decimal === undefined ? {} : { decimal: k.decimal }) });
73
87
 
74
88
  /**
75
89
  * Dry-read every kept money fact; drop (as `reads-no-result`) the ones whose forms read no result line. Other facts pass through untouched.
package/src/pagefetch.js CHANGED
@@ -9,7 +9,7 @@
9
9
  // proven in step 0 ('read' and 'full' returned almost nothing from a results page). The text that comes back is the input. An empty
10
10
  // fetch is a plain-words stop, never a silent empty input. Nothing here touches a model.
11
11
 
12
- import { SECRET_PATTERNS } from './validate.js';
12
+ import { SECRET_PATTERNS, redactSecrets } from './validate.js';
13
13
  import { TAG } from './labelscan.js';
14
14
 
15
15
  /** the session options (incognito: no auth ever enters; blockPrivateNetwork: SSRF guard); ONE frozen object so a test can pin them */
@@ -30,7 +30,18 @@ export const MIN_PAGE_CHARS = 200;
30
30
  export const MIN_CONTENT_CHARS = 100;
31
31
  /** the page reader's landmark roles for a site's own chrome (the ARIA names of header, menu and footer); everything under one is not the page's content */
32
32
  const CHROME_ROLES = new Set(['banner', 'navigation', 'contentinfo']);
33
- /** our own deadline around one whole fetch (a hung browser leaves no trace; this is the only thing that fires on the absence of an answer) */
33
+ /** how many times one fetch is tried when it comes back header-only or times out (a real Dell page was header-only on one try and whole on the next) */
34
+ export const FETCH_ATTEMPTS = 3;
35
+ /** the pause between attempts (ms) */
36
+ export const RETRY_WAIT_MS = 3000;
37
+ /** a header-only snapshot is re-taken (same open page) every this many ms until content shows up (live: Dell de-de gave header-only 3 times in ~5 s
38
+ * each, while its content arrived ~18-35 s after load; a fast refetch hits the same unsettled page) */
39
+ export const SETTLE_INTERVAL_MS = 2000;
40
+ /** the most one attempt keeps re-reading a header-only page (ms) before it counts as header-only; the per-attempt deadline below still bounds the whole attempt */
41
+ export const SETTLE_BUDGET_MS = 20_000;
42
+ /** how many characters of a header-only page's content (after the snapshot header) the failure record keeps */
43
+ export const HEAD_KEEP_CHARS = 300;
44
+ /** our own deadline around ONE attempt (a fetch that retries can take up to FETCH_ATTEMPTS of these, plus the pauses); a hung browser leaves no trace, this is the only thing that fires on the absence of an answer */
34
45
  export const PAGE_FETCH_TIMEOUT_MS = 90_000;
35
46
 
36
47
  /** @param {unknown} v @returns {boolean} an http(s) address (the one spelling of "this Inputs value is a web address") */
@@ -70,13 +81,18 @@ export function pageContentChars(text) {
70
81
  * snapshot: (o?: {mode?: string}) => Promise<string>, close: () => Promise<void>}} FetchPage the part of barebrowse's page handle the fetch uses
71
82
  */
72
83
 
84
+ /** @typedef {{chars: number, ms: number, head: string}} AttemptRecord what a header-only attempt saw: content characters, elapsed ms, the first characters of the content (secret-masked) */
85
+
73
86
  /**
74
87
  * Fetch one page's text. `connect` is injected (a test never reaches the network); the default is barebrowse's own `connect`.
88
+ * The whole fetch (own browser, own page, closed each time) is tried up to FETCH_ATTEMPTS times, but ONLY when an attempt ends header-only (the page
89
+ * loaded its menus and not its content) or times out (our deadline, or the page load's own timeout). A bot challenge, a refused or private address, any
90
+ * other load error, a near-empty page and a non-text answer are final on the first attempt.
75
91
  * @param {string} url
76
- * @param {{connect?: (opts: Record<string, any>) => Promise<FetchPage>, timeoutMs?: number, idleMs?: number, idleTimeoutMs?: number, closeWaitMs?: number}} [o]
77
- * @returns {Promise<{ok: true, text: string, url: string}|{ok: false, error: string}>} `error` is plain words, without a spend line
92
+ * @param {{connect?: (opts: Record<string, any>) => Promise<FetchPage>, timeoutMs?: number, idleMs?: number, idleTimeoutMs?: number, closeWaitMs?: number, retryWaitMs?: number, settleMs?: number, settleIntervalMs?: number}} [o] `retryWaitMs` - the pause between attempts (a test passes 0); `settleMs`/`settleIntervalMs` - the in-attempt re-read budget and pace for a header-only page
93
+ * @returns {Promise<{ok: true, text: string, url: string}|{ok: false, error: string, attempts?: AttemptRecord[]}>} `error` is plain words, without a spend line
78
94
  */
79
- export async function fetchPageText(url, { connect, timeoutMs = PAGE_FETCH_TIMEOUT_MS, idleMs = NETWORK_IDLE_MS, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, closeWaitMs = 3000 } = {}) {
95
+ export async function fetchPageText(url, { connect, timeoutMs = PAGE_FETCH_TIMEOUT_MS, idleMs = NETWORK_IDLE_MS, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, closeWaitMs = 3000, retryWaitMs = RETRY_WAIT_MS, settleMs = SETTLE_BUDGET_MS, settleIntervalMs = SETTLE_INTERVAL_MS } = {}) {
80
96
  let u;
81
97
  try { u = new URL(url.trim()); } catch { return { ok: false, error: `${url} is not a web address bareloop can read` }; }
82
98
  if (u.protocol !== 'http:' && u.protocol !== 'https:') return { ok: false, error: `${url} is not an http or https address` };
@@ -88,6 +104,30 @@ export async function fetchPageText(url, { connect, timeoutMs = PAGE_FETCH_TIMEO
88
104
  else {
89
105
  try { open = /** @type {any} */ ((await import('barebrowse')).connect); } catch (e) { return { ok: false, error: `the page reader (barebrowse) could not be loaded: ${/** @type {Error} */ (e).message}` }; }
90
106
  }
107
+ /** @type {{ok: false, error: string}} */
108
+ let last = { ok: false, error: '' };
109
+ /** @type {AttemptRecord[]} */
110
+ const attempts = [];
111
+ for (let attempt = 1; attempt <= FETCH_ATTEMPTS; attempt += 1) {
112
+ const r = await attemptOnce(u, open, { timeoutMs, idleMs, idleTimeoutMs, closeWaitMs, settleMs, settleIntervalMs });
113
+ if (r.ok) return r;
114
+ if (!r.retry) return { ok: false, error: r.error, ...(attempts.length || r.record ? { attempts: r.record ? [...attempts, r.record] : attempts } : {}) };
115
+ if (r.record) attempts.push(r.record);
116
+ last = { ok: false, error: r.error };
117
+ if (attempt < FETCH_ATTEMPTS && retryWaitMs > 0) await new Promise((res) => setTimeout(res, retryWaitMs));
118
+ }
119
+ return { ok: false, error: `${last.error} (tried ${FETCH_ATTEMPTS} times)`, ...(attempts.length ? { attempts } : {}) };
120
+ }
121
+
122
+ /**
123
+ * ONE attempt: opens its own browser and page and always closes them. `retry: true` marks the two endings worth another go (header-only, timeout).
124
+ * @param {URL} u @param {(opts: Record<string, any>) => Promise<FetchPage>} open
125
+ * @param {{timeoutMs: number, idleMs: number, idleTimeoutMs: number, closeWaitMs: number, settleMs: number, settleIntervalMs: number}} o
126
+ * @returns {Promise<{ok: true, text: string, url: string}|{ok: false, error: string, retry: boolean, record?: AttemptRecord}>}
127
+ */
128
+ async function attemptOnce(u, open, { timeoutMs, idleMs, idleTimeoutMs, closeWaitMs, settleMs, settleIntervalMs }) {
129
+ const t0 = Date.now();
130
+ let stopped = false;
91
131
  /** @type {NodeJS.Timeout|undefined} */
92
132
  let timer;
93
133
  /** @type {Promise<FetchPage>|undefined} */
@@ -101,19 +141,31 @@ export async function fetchPageText(url, { connect, timeoutMs = PAGE_FETCH_TIMEO
101
141
  await p.goto(u.href, GOTO_TIMEOUT_MS);
102
142
  // bounded: a page that never goes quiet (polling, streaming) is read as it stands - the wait is for a page that is still filling
103
143
  try { await p.waitForNetworkIdle({ idle: idleMs, timeout: idleTimeoutMs }); } catch { /* read what is there */ }
104
- return p.snapshot({ ...SNAPSHOT_OPTIONS });
144
+ let snap = await p.snapshot({ ...SNAPSHOT_OPTIONS });
145
+ // header-only: the page's scripts may still be filling it (network idle came early). Keep the page open and re-read it, up to settleMs;
146
+ // only then does this attempt count as header-only. The attempt's own deadline still bounds all of it.
147
+ const settleEnd = Date.now() + settleMs;
148
+ while (!stopped && Date.now() < settleEnd && isHeaderOnly(snap)) {
149
+ await new Promise((res) => setTimeout(res, Math.min(settleIntervalMs, Math.max(0, settleEnd - Date.now()))));
150
+ if (stopped) break;
151
+ snap = await p.snapshot({ ...SNAPSHOT_OPTIONS });
152
+ }
153
+ return snap;
105
154
  })(),
106
- new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`no answer inside ${Math.round(timeoutMs / 1000)} seconds`)), timeoutMs); }),
155
+ new Promise((_, reject) => { timer = setTimeout(() => reject(Object.assign(new Error(`no answer inside ${Math.round(timeoutMs / 1000)} seconds`), { deadline: true })), timeoutMs); }),
107
156
  ]);
108
- if (typeof text !== 'string') return { ok: false, error: `${u.href} did not come back as text` };
109
- if (/\[BOT CHALLENGE DETECTED\]/.test(text)) return { ok: false, error: `${u.href} asked for a human check (a bot challenge), so the page could not be read` };
157
+ if (typeof text !== 'string') return { ok: false, retry: false, error: `${u.href} did not come back as text` };
158
+ if (/\[BOT CHALLENGE DETECTED\]/.test(text)) return { ok: false, retry: false, error: `${u.href} asked for a human check (a bot challenge), so the page could not be read` };
110
159
  const size = text.trim().length;
111
- if (size < MIN_PAGE_CHARS) return { ok: false, error: `${u.href} came back with ${size === 0 ? 'no text' : `only ${size} characters of text`} (less than ${MIN_PAGE_CHARS}) - the page did not load, or holds nothing to read` };
112
- if (pageContentChars(text) < MIN_CONTENT_CHARS) return { ok: false, error: `${u.href} came back with only the site's own header and menus - the page did not load its content, so there is nothing to read` };
160
+ if (size < MIN_PAGE_CHARS) return { ok: false, retry: false, error: `${u.href} came back with ${size === 0 ? 'no text' : `only ${size} characters of text`} (less than ${MIN_PAGE_CHARS}) - the page did not load, or holds nothing to read` };
161
+ if (pageContentChars(text) < MIN_CONTENT_CHARS) return { ok: false, retry: true, error: `${u.href} came back with only the site's own header and menus - the page did not load its content, so there is nothing to read`, record: headerRecord(text, Date.now() - t0) };
113
162
  return { ok: true, text, url: u.href };
114
163
  } catch (e) {
115
- return { ok: false, error: `${u.href} could not be read: ${/** @type {Error} */ (e)?.message ?? String(e)}` };
164
+ const msg = /** @type {Error} */ (e)?.message ?? String(e);
165
+ const timedOut = /** @type {any} */ (e)?.deadline === true || /time[d ]?-?out/i.test(msg);
166
+ return { ok: false, retry: timedOut, error: `${u.href} could not be read: ${msg}` };
116
167
  } finally {
168
+ stopped = true;
117
169
  if (timer !== undefined) clearTimeout(timer);
118
170
  // always close the browser, also when the deadline fired while it was still starting or loading
119
171
  if (opening) {
@@ -124,3 +176,14 @@ export async function fetchPageText(url, { connect, timeoutMs = PAGE_FETCH_TIMEO
124
176
  }
125
177
  }
126
178
  }
179
+
180
+ /** @param {unknown} text @returns {boolean} text long enough to pass the near-empty floor, yet with no content outside the site's own chrome (the one case worth re-reading) */
181
+ function isHeaderOnly(text) {
182
+ return typeof text === 'string' && !/\[BOT CHALLENGE DETECTED\]/.test(text) && text.trim().length >= MIN_PAGE_CHARS && pageContentChars(text) < MIN_CONTENT_CHARS;
183
+ }
184
+
185
+ /** the short failure record of a header-only attempt; the head is secret-masked (the log is append-only) @param {string} text @param {number} ms @returns {AttemptRecord} */
186
+ function headerRecord(text, ms) {
187
+ const body = text.split('\n').filter((l) => !/^url: /.test(l) && !/^[\d,]+ chars → /.test(l) && !/^hint: /.test(l)).join(' ').replace(/\s+/g, ' ').trim();
188
+ return { chars: pageContentChars(text), ms, head: redactSecrets(body.slice(0, HEAD_KEEP_CHARS)) };
189
+ }
package/src/pagemap.js CHANGED
@@ -16,7 +16,7 @@ import { AUTHOR_CALL_TIMEOUT_MS, AUTHOR_SCOUT_MAX_TOKENS, SURVEY_CAUSES } from '
16
16
  import { lineRules, boundInWords, wordsSaidOn } from './rubricstep.js';
17
17
  import { applyPageRules, validReading } from './pagerules.js';
18
18
  import { dryReadMoney, dryReadTimes, dryReadPhrases } from './pagedry.js';
19
- import { ruleWords } from './facts.js';
19
+ import { ruleWords, FACT_LANGS } from './facts.js';
20
20
  import { lookFactOf } from './inputlook.js';
21
21
 
22
22
  const require = createRequire(import.meta.url);
@@ -25,7 +25,9 @@ const { Loop } = require('bare-agent');
25
25
  /** the label of the one paid call, as the drafting book and the progress line see it */
26
26
  export const MAP_CALL_LABEL = 'page-map';
27
27
  /** the shapes the map may name; a `place` is read by code as a phrase */
28
- export const MAP_SHAPES = Object.freeze(['place', 'money', 'time', 'phrase']);
28
+ export const MAP_SHAPES = Object.freeze(['place', 'money', 'time', 'date', 'number', 'phrase']);
29
+ /** the languages a date's month names may be signed in (one list with src/facts.js) */
30
+ const MAP_LANGS = Object.keys(FACT_LANGS);
29
31
 
30
32
  export const MAP_SYSTEM = 'You map a person\'s job to the exact words of ONE saved web page. You never grade and never invent words: every string you return must be copied from the page text, character for character.';
31
33
 
@@ -40,12 +42,15 @@ export function mapPrompt({ jobLines, label, text }) {
40
42
  return `THE JOB (each ~ rule applies to the line above it):\n${job}\n\n`
41
43
  + 'A "result" is one item the page lists (one flight, one stay, one row). For each job line and for each rule, say which fact(s) every result must carry, and give the PAGE\'S OWN WORDS for it.\n\n'
42
44
  + 'Return ONLY a JSON object, no prose, no code fence:\n'
43
- + '{"facts":[{"name":"<letters and spaces only>","from":"<line N | line N rule M>","shape":"place|money|time|phrase","words":["<page words>", ...],"currency":["<symbol>"],"unit":["<words after the amount>"],"reading":<optional, see below>}]}\n\n'
45
+ + `{"facts":[{"name":"<letters and spaces only>","from":"<line N | line N rule M>","shape":"place|money|time|date|number|phrase","words":["<page words>", ...],"currency":["<symbol>"],"unit":["<words after the amount>"],"lang":"<${MAP_LANGS.join("|")}>","decimal":"<. or ,>","reading":<optional, see below>}]}\n\n`
44
46
  + 'shape meanings:\n'
45
47
  + '- place: words = ways the page writes the place (a city, an airport name), copied from the page.\n'
46
48
  + '- money: words = the label(s) written right BEFORE the amount, copied from the page. currency / unit = how the page writes the currency ("$") and/or the words after the amount ("US dollars"); give every spelling the page uses, as lists.\n'
47
49
  + '- time: words = the label(s) written right BEFORE the time that is the one the rule is about, copied from the page.\n'
50
+ + `- date: words = the label(s) written right BEFORE the date that is the one the rule is about, copied from the page. lang = the language the page writes the month name in (${MAP_LANGS.join(' ')}); leave lang out when the date is written 2026-10-26.\n`
51
+ + '- number: words = the label(s) written right BEFORE a plain number (a rating, a count), copied from the page. decimal = the mark the page writes before the decimals: "." or ",". Give decimal on a money fact too when the page writes a comma before the decimals.\n'
48
52
  + '- phrase: words = fixed words a result\'s text must contain, copied from the page.\n'
53
+ + 'A label must appear ONCE on its page line and sit right before the value; when the value stands alone on its line, give no label.\n'
49
54
  + 'Several words in one list are alternatives for the same fact. Put the best one first.\n'
50
55
  + 'reading (optional, only on a time or money fact): when a ~ rule is a limit on that fact that the page cannot hold in words (for example "not a red-eye" is a limit on the departure time), give your own best reading of the rule as that limit: {"op":"between","lo":<bound>,"hi":<bound>} or {"op":"lt|lte|gt|gte","value":<bound>}. A time bound is a "HH:MM" string on a 24-hour clock; a money bound is a number. Leave reading out when a page word already carries the rule.\n\n'
51
56
  + `PAGE TEXT (the input "${label}"):\n${text}`;
@@ -60,6 +65,7 @@ export function parseMapAnswer(text) {
60
65
  /** @type {any} */
61
66
  let m;
62
67
  try { m = JSON.parse(String(text ?? '').trim()); } catch { return { ok: false }; }
68
+ const notBlank = (/** @type {string} */ w) => w.trim() !== '';
63
69
  const strs = (/** @type {unknown} */ v) => Array.isArray(v) && v.every((x) => typeof x === 'string');
64
70
  if (m === null || typeof m !== 'object' || Array.isArray(m) || !Array.isArray(m.facts)) return { ok: false };
65
71
  for (const f of m.facts) {
@@ -67,6 +73,12 @@ export function parseMapAnswer(text) {
67
73
  if (typeof f.name !== 'string' || !MAP_SHAPES.includes(f.shape) || !strs(f.words)) return { ok: false };
68
74
  if (f.from !== undefined && typeof f.from !== 'string') return { ok: false };
69
75
  for (const k of ['currency', 'unit']) if (f[k] !== undefined && !strs(f[k])) return { ok: false };
76
+ if (f.lang !== undefined && !(typeof f.lang === 'string' && MAP_LANGS.includes(f.lang))) return { ok: false };
77
+ if (f.decimal !== undefined && !(f.decimal === '.' || f.decimal === ',')) return { ok: false };
78
+ // a blank entry (empty or only spaces) names nothing: it is left out here, with no note, and the rest of the fact stands. This is not a JSON
79
+ // repair - the answer was well-formed; a list left empty is absent (words stays a list, `[]`: the fact then reads by its signs alone)
80
+ f.words = f.words.filter(notBlank);
81
+ for (const k of ['currency', 'unit']) if (f[k] !== undefined) { f[k] = f[k].filter(notBlank); if (f[k].length === 0) delete f[k]; }
70
82
  // PRD v1.107 Part 5b: a `reading` is a limit of the signed-rule shape on a time or money fact; any other is a malformed answer (no repair)
71
83
  if (f.reading !== undefined && !validReading(f.shape, f.reading)) return { ok: false };
72
84
  }