@panphora/clay-agent 0.0.0-stage → 0.1.0

package/src/html/find.js

@@ -0,0 +1,157 @@
+import { Worker } from 'node:worker_threads';
+
+import { ClayError } from '../errors.js';
+import { located, parse, pathOf } from './source.js';
+
+const SNIPPET_MAX = 80;
+
+function lineStarts(html) {
+  const starts = [];
+  let index = html.indexOf('\n');
+  while (index !== -1) {
+    starts.push(index);
+    index = html.indexOf('\n', index + 1);
+  }
+  return starts;
+}
+
+function lineAt(starts, offset) {
+  let lo = 0;
+  let hi = starts.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1;
+    if (starts[mid] < offset) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo + 1;
+}
+
+function elementIndex($) {
+  const items = [];
+  for (const el of $('*').toArray()) {
+    const loc = located(el);
+    if (loc) items.push({ ...loc, el });
+  }
+  return items;
+}
+
+function enclosing(items, offset) {
+  let lo = 0;
+  let hi = items.length - 1;
+  let found = -1;
+  while (lo <= hi) {
+    const mid = (lo + hi) >> 1;
+    if (items[mid].start <= offset) {
+      found = mid;
+      lo = mid + 1;
+    } else {
+      hi = mid - 1;
+    }
+  }
+  for (let i = found; i >= 0; i -= 1) {
+    if (items[i].end > offset) return items[i].el;
+  }
+  return null;
+}
+
+function snippet(html, start, end) {
+  const matchLength = end - start;
+  if (matchLength >= SNIPPET_MAX) return html.slice(start, start + SNIPPET_MAX);
+  const budget = SNIPPET_MAX - matchLength;
+  const before = Math.min(Math.floor(budget / 2), start);
+  const after = Math.min(budget - before, html.length - end);
+  return html.slice(start - before, end + after);
+}
+
+function literalMatches(html, text, onMatch) {
+  let from = 0;
+  for (;;) {
+    const start = html.indexOf(text, from);
+    if (start === -1) return;
+    onMatch(start, start + text.length);
+    from = start + text.length;
+  }
+}
+
+const REGEX_DEADLINE_MS = 2000;
+
+function regexRanges(html, source, max) {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(new URL('./regex-worker.js', import.meta.url), {
+      workerData: { html, source, max },
+    });
+    const timer = setTimeout(() => {
+      worker.terminate();
+      reject(
+        new ClayError('find-timeout', 'That regular expression took too long on this document.', {
+          hint: 'Use a simpler pattern, or pass `text` for a literal search.',
+        }),
+      );
+    }, REGEX_DEADLINE_MS);
+    worker.once('message', (result) => {
+      clearTimeout(timer);
+      worker.terminate();
+      resolve(result);
+    });
+    worker.once('error', () => {
+      clearTimeout(timer);
+      reject(new ClayError('internal', 'The regular expression search failed.'));
+    });
+  });
+}
+
+export async function find(html, { text, regex, limit = 20 } = {}) {
+  const hasText = text !== undefined && text !== null;
+  const hasRegex = regex !== undefined && regex !== null;
+  if (hasText === hasRegex) {
+    throw new ClayError('invalid-find', 'Pass exactly one of `text` or `regex`.', {
+      hint: 'Give a literal string as `text`, or a regular expression source as `regex`.',
+    });
+  }
+  if (hasText && (typeof text !== 'string' || text.length === 0)) {
+    throw new ClayError('invalid-find', '`text` must be a non-empty string.', {
+      hint: 'Pass the exact characters to look for.',
+    });
+  }
+  if (hasRegex && typeof regex !== 'string') {
+    throw new ClayError('invalid-find', '`regex` must be a string.', {
+      hint: 'Pass a JavaScript regular expression source string.',
+    });
+  }
+  const max = Number.isInteger(limit) && limit > 0 ? limit : 0;
+  let count = 0;
+  let ranges = [];
+  if (hasText) {
+    const onMatch = (start, end) => {
+      count += 1;
+      if (ranges.length < max) ranges.push({ start, end });
+    };
+    literalMatches(html, text, onMatch);
+  } else {
+    try {
+      new RegExp(regex, 'g');
+    } catch {
+      throw new ClayError('invalid-find', `That is not a valid regular expression: ${regex}`, {
+        hint: 'Pass a JavaScript regular expression source string, such as `id="[0-9]+"`.',
+      });
+    }
+    const result = await regexRanges(html, regex, max);
+    count = result.count;
+    ranges = result.ranges;
+  }
+  if (count === 0) return { count, matches: [] };
+  const { $ } = parse(html);
+  const starts = lineStarts(html);
+  const items = elementIndex($);
+  const matches = ranges.map(({ start, end }) => {
+    const el = enclosing(items, start);
+    return {
+      start,
+      end,
+      line: lineAt(starts, start),
+      path: el ? pathOf(el) : null,
+      snippet: snippet(html, start, end),
+    };
+  });
+  return { count, matches };
+}

package/src/html/outline.js

@@ -0,0 +1,154 @@
+import { ClayError } from '../errors.js';
+import { isElement, located, parse, pathOf } from './source.js';
+
+const NO_TEXT = new Set(['script', 'style']);
+const TEXT_LIMIT = 60;
+const MAX_CHILDREN = 40;
+const OUTLINE_BUDGET = 8000;
+const CAPPED_HINT =
+  'Part of the outline is hidden. Outline one part with select (a path from this outline), or use find_in_document.';
+
+function collapseWhitespace(value) {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function directText(el) {
+  let text = '';
+  for (const child of el.children) {
+    if (child.type === 'text') text += child.data;
+  }
+  return collapseWhitespace(text).slice(0, TEXT_LIMIT);
+}
+
+function classOf(el) {
+  const value = el.attribs?.class;
+  return typeof value === 'string' ? value : null;
+}
+
+function runKey(el) {
+  const className = classOf(el);
+  return `${el.name}\u0000${className === null ? '\u0000' : className}`;
+}
+
+function expand(children) {
+  const out = [];
+  for (const child of children) {
+    if (!isElement(child)) continue;
+    if (!located(child)) {
+      out.push(...expand(child.children));
+      continue;
+    }
+    out.push(child);
+  }
+  return out;
+}
+
+function entryFor(el, level, depth, last, count, state) {
+  const loc = located(el);
+  const lastLoc = last === el ? loc : located(last);
+  const entry = { path: pathOf(el), tag: el.name };
+  const id = el.attribs?.id;
+  if (id) entry.id = id;
+  const className = classOf(el);
+  if (className !== null) entry.class = className;
+  entry.start = loc.start;
+  entry.end = lastLoc.end;
+  entry.chars = lastLoc.end - loc.start;
+  if (count > 1) {
+    entry.repeated = count;
+    entry.lastPath = pathOf(last);
+  }
+  if (NO_TEXT.has(el.name)) {
+    const type = el.attribs?.type;
+    if (type) entry.type = type;
+  } else {
+    const text = directText(el);
+    if (text) entry.text = text;
+    if (level < depth) {
+      const children = buildEntries(expand(el.children), level + 1, depth, state);
+      if (children.length > 0) entry.children = children;
+    }
+  }
+  return entry;
+}
+
+function buildEntries(children, level, depth, state) {
+  const runs = [];
+  let i = 0;
+  while (i < children.length) {
+    const el = children[i];
+    let j = i;
+    while (j + 1 < children.length && runKey(children[j + 1]) === runKey(el)) j += 1;
+    runs.push({ el, last: children[j], count: j - i + 1 });
+    i = j + 1;
+  }
+  const shown = runs.slice(0, MAX_CHILDREN);
+  const entries = shown.map((run) => entryFor(run.el, level, depth, run.last, run.count, state));
+  if (runs.length > shown.length) {
+    const hidden = runs.slice(shown.length);
+    entries.push({
+      more: hidden.length,
+      start: located(hidden[0].el).start,
+      end: located(hidden[hidden.length - 1].last).end,
+    });
+    state.capped = true;
+  }
+  return entries;
+}
+
+function readApiRules($) {
+  const tag = $('script[data-rules-name~="api"]').first();
+  if (tag.length === 0) return { hasApiRules: false, dataKeys: [] };
+  let dataKeys = [];
+  try {
+    const data = JSON.parse(tag.text());
+    if (data && typeof data === 'object') dataKeys = Object.keys(data);
+  } catch {
+    dataKeys = [];
+  }
+  return { hasApiRules: true, dataKeys };
+}
+
+function trimToBudget(entries, maxChars, state) {
+  const popped = [];
+  while (entries.length > 1 && JSON.stringify(entries).length > maxChars) {
+    popped.unshift(entries.pop());
+  }
+  if (popped.length === 0) return;
+  const more = popped.reduce((sum, entry) => sum + (entry.repeated ?? 1) + (entry.more ?? 0), 0);
+  entries.push({ more, start: popped[0].start, end: popped[popped.length - 1].end });
+  state.capped = true;
+}
+
+export function outline(html, { depth = 4, select = null, maxChars = OUTLINE_BUDGET } = {}) {
+  const { $ } = parse(html);
+  const root = select === null ? $('html').first()[0] : null;
+  let start = root;
+  if (select !== null) {
+    try {
+      start = $(select).first()[0] ?? null;
+    } catch {
+      throw new ClayError('invalid-selector', `That is not a valid CSS selector: ${select}`, {
+        hint: 'Pass a selector such as `main#board` or `article.card`.',
+      });
+    }
+  }
+  const requested = Number.isInteger(depth) && depth > 0 ? depth : 0;
+  const state = { capped: false };
+  let used = requested;
+  let entries = [];
+  if (start && requested > 0) {
+    const children = expand(start.children);
+    entries = buildEntries(children, 1, used, state);
+    while (JSON.stringify(entries).length > maxChars && used > 1) {
+      used -= 1;
+      state.capped = false;
+      entries = buildEntries(children, 1, used, state);
+    }
+    if (JSON.stringify(entries).length > maxChars) trimToBudget(entries, maxChars, state);
+  }
+  const truncated = used < requested || state.capped;
+  const result = { size: html.length, ...readApiRules($), depth: used, truncated, outline: entries };
+  if (truncated) result.hint = CAPPED_HINT;
+  return result;
+}

package/src/html/regex-worker.js

@@ -0,0 +1,17 @@
+import { parentPort, workerData } from 'node:worker_threads';
+
+const { html, source, max } = workerData;
+const pattern = new RegExp(source, 'g');
+let count = 0;
+const ranges = [];
+for (;;) {
+  const match = pattern.exec(html);
+  if (!match) break;
+  count += 1;
+  if (ranges.length < max) ranges.push({ start: match.index, end: match.index + match[0].length });
+  if (match[0].length === 0) {
+    pattern.lastIndex = match.index + 1;
+    if (pattern.lastIndex > html.length) break;
+  }
+}
+parentPort.postMessage({ count, ranges });

package/src/html/select.js

@@ -0,0 +1,54 @@
+import { ClayError } from '../errors.js';
+import { located, parse, pathOf } from './source.js';
+
+export function select(html, css, { limit = 20, maxChars = 40000 } = {}) {
+  const { $ } = parse(html);
+  let found;
+  try {
+    if (typeof css !== 'string' || !css.trim()) throw new Error('empty selector');
+    found = $(css).toArray();
+  } catch {
+    throw new ClayError('invalid-selector', `That is not a valid CSS selector: ${css}`, {
+      hint: 'Pass a selector such as `article.card` or `main#board > section`.',
+    });
+  }
+  const locatedMatches = [];
+  for (const el of found) {
+    const loc = located(el);
+    if (!loc) continue;
+    locatedMatches.push({ el, ...loc });
+  }
+  let remaining = maxChars;
+  const matches = locatedMatches.slice(0, Math.max(0, limit)).map(({ el, start, end }) => {
+    const match = { path: pathOf(el), start, end };
+    if (remaining <= 0) {
+      match.truncated = true;
+      return match;
+    }
+    const take = Math.min(end - start, remaining);
+    match.html = html.slice(start, start + take);
+    if (take < end - start) match.truncated = true;
+    remaining -= take;
+    return match;
+  });
+  return { count: locatedMatches.length, matches };
+}
+
+export function range(html, start, length, { maxChars = Infinity } = {}) {
+  if (!Number.isInteger(start) || start < 0) {
+    throw new ClayError('invalid-range', 'The range start must be a non-negative whole number.', {
+      hint: 'Pass a start offset from a read, such as 0.',
+    });
+  }
+  if (!Number.isInteger(length) || length < 0) {
+    throw new ClayError('invalid-range', 'The range length must be a non-negative whole number.', {
+      hint: 'Pass the number of characters to read.',
+    });
+  }
+  const from = Math.min(start, html.length);
+  const full = Math.min(from + length, html.length);
+  const to = Math.min(from + length, html.length, from + maxChars);
+  const result = { start: from, end: to, html: html.slice(from, to) };
+  if (to < full) result.truncated = true;
+  return result;
+}

package/src/html/source.js

@@ -0,0 +1,73 @@
+import { load } from 'cheerio';
+
+const ELEMENT_TYPES = new Set(['tag', 'script', 'style']);
+
+export function isElement(node) {
+  return Boolean(node && ELEMENT_TYPES.has(node.type) && node.name);
+}
+
+export function parse(html) {
+  const $ = load(html, { sourceCodeLocationInfo: true });
+  return { html, $ };
+}
+
+export function located(el) {
+  const loc = el?.sourceCodeLocation;
+  if (!loc || typeof loc.startOffset !== 'number' || typeof loc.endOffset !== 'number') return null;
+  return { start: loc.startOffset, end: loc.endOffset };
+}
+
+function escapeIdent(value) {
+  return String(value).replace(/([^\w-])/g, '\\$1');
+}
+
+const typePositions = new WeakMap();
+
+function positionsUnder(parent) {
+  let entry = typePositions.get(parent);
+  if (!entry) {
+    const position = new Map();
+    const total = new Map();
+    for (const child of parent.children) {
+      if (!isElement(child)) continue;
+      const n = (total.get(child.name) ?? 0) + 1;
+      total.set(child.name, n);
+      position.set(child, n);
+    }
+    entry = { position, total };
+    typePositions.set(parent, entry);
+  }
+  return entry;
+}
+
+function stepFor(el) {
+  const id = el.attribs?.id;
+  if (id) return `${el.name}#${escapeIdent(id)}`;
+  let step = el.name;
+  const className = firstClass(el);
+  if (className) step += `.${escapeIdent(className)}`;
+  const parent = el.parent;
+  if (isElement(parent)) {
+    const { position, total } = positionsUnder(parent);
+    if (total.get(el.name) > 1) step += `:nth-of-type(${position.get(el)})`;
+  }
+  return step;
+}
+
+function firstClass(el) {
+  const value = el.attribs?.class;
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  return trimmed ? trimmed.split(/\s+/)[0] : null;
+}
+
+export function pathOf(el) {
+  const chain = [];
+  let node = el;
+  while (isElement(node)) {
+    chain.unshift(node);
+    if (node.attribs?.id || node.name === 'html') break;
+    node = node.parent;
+  }
+  return chain.map(stepFor).join(' > ');
+}