explorbot 0.4.9 → 0.5.0

package/dist/src/utils/mdq/query.js

@@ -0,0 +1,451 @@
+import { marked } from 'marked';
+import { MarkdownEditor } from "./edit.js";
+export { MarkdownEditor };
+export class MarkdownQuery {
+    source;
+    matches;
+    constructor(source, matches) {
+        this.source = source;
+        this.matches = matches || buildTokenIndex(source);
+    }
+    query(selector, matcher) {
+        const segments = parseQuery(selector);
+        if (matcher !== undefined && segments.length > 0)
+            segments[segments.length - 1].textMatch = this.valueMatcher(matcher);
+        return new MarkdownQuery(this.source, this.run(this.expandSections(this.matches), segments));
+    }
+    section(matcher, options) {
+        return this.query(`section${options?.depth || ''}`, matcher);
+    }
+    heading(matcher, options) {
+        if (options?.depth)
+            return this.query(`h${options.depth}`, matcher);
+        return this.query('heading', matcher);
+    }
+    paragraph(matcher) {
+        return this.query('paragraph', matcher);
+    }
+    table(matcher) {
+        return this.query('table', matcher);
+    }
+    list(matcher) {
+        return this.query('list', matcher);
+    }
+    item(matcher) {
+        return this.query('item', matcher);
+    }
+    code(matcher) {
+        return this.query('code', matcher);
+    }
+    blockquote(matcher) {
+        return this.query('blockquote', matcher);
+    }
+    comment(matcher) {
+        return this.query('comment', matcher);
+    }
+    html(matcher) {
+        return this.query('html', matcher);
+    }
+    hr() {
+        return this.query('hr');
+    }
+    text() {
+        return this.matches.map((range) => this.source.slice(range.start, range.start + range.length)).join('');
+    }
+    nodes() {
+        return this.matches.map((range) => {
+            const token = range.token;
+            if (token.type !== 'heading')
+                return { type: token.type, depth: null, text: this.tokenText(range.token) };
+            return { type: token.type, depth: token.depth, text: this.tokenText(range.token) };
+        });
+    }
+    rows() {
+        const results = [];
+        for (const range of this.matches) {
+            if (range.token.type !== 'table')
+                continue;
+            const table = range.token;
+            const headers = table.header.map((cell) => cell.text);
+            for (const row of table.rows) {
+                const entry = {};
+                for (let i = 0; i < headers.length; i++) {
+                    entry[headers[i]] = row[i]?.text ?? '';
+                }
+                results.push(entry);
+            }
+        }
+        return results;
+    }
+    entries() {
+        const entries = {};
+        for (const range of this.matches) {
+            for (const line of this.tokenText(range.token).split('\n')) {
+                const separator = line.indexOf(':');
+                if (separator < 1)
+                    continue;
+                const value = line.slice(separator + 1).trim();
+                if (value)
+                    entries[line.slice(0, separator).trim().toLowerCase()] = value;
+            }
+        }
+        return entries;
+    }
+    frontmatter() {
+        return this.editor().frontmatter();
+    }
+    count() {
+        return this.matches.length;
+    }
+    exists() {
+        return this.matches.length > 0;
+    }
+    at(index) {
+        let resolved = index;
+        if (resolved < 0)
+            resolved = this.matches.length + resolved;
+        if (resolved < 0 || resolved >= this.matches.length)
+            return new MarkdownQuery(this.source, []);
+        return new MarkdownQuery(this.source, [this.matches[resolved]]);
+    }
+    slice(from, to) {
+        return new MarkdownQuery(this.source, this.matches.slice(from, to));
+    }
+    first() {
+        return this.slice(0, 1);
+    }
+    last() {
+        return this.slice(-1);
+    }
+    each() {
+        return this.matches.map((range) => new MarkdownQuery(this.source, [range]));
+    }
+    preceding() {
+        if (this.matches.length === 0)
+            return new MarkdownQuery(this.source, []);
+        const cutoff = this.matches[0].start;
+        return new MarkdownQuery(this.source, buildTokenIndex(this.source).filter((range) => range.start + range.length <= cutoff));
+    }
+    following() {
+        if (this.matches.length === 0)
+            return new MarkdownQuery(this.source, []);
+        const last = this.matches[this.matches.length - 1];
+        const cutoff = last.start + last.length;
+        return new MarkdownQuery(this.source, buildTokenIndex(this.source).filter((range) => range.start >= cutoff));
+    }
+    replace(content) {
+        return this.replaceEach(() => content);
+    }
+    replaceEach(replacer) {
+        return new MarkdownQuery(this.editor().replace(this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index))));
+    }
+    remove() {
+        return new MarkdownQuery(this.editor().remove(this.matches));
+    }
+    insertBefore(markdown) {
+        return this.insertAll((range) => range.start, markdown);
+    }
+    insertAfter(markdown) {
+        return this.insertAll((range) => MarkdownEditor.blockEnd(range), markdown);
+    }
+    prepend(markdown) {
+        return this.insertAll((range) => this.containerStart(range), markdown);
+    }
+    append(markdown) {
+        return this.insertAll((range) => this.containerEnd(range), markdown);
+    }
+    addRow(row) {
+        return new MarkdownQuery(this.editor().replace(this.matches, (range) => {
+            if (range.token.type !== 'table')
+                throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`);
+            const table = range.token;
+            const headers = table.header.map((cell) => cell.text);
+            const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || ''));
+            return MarkdownEditor.table(headers, [...existing, headers.map((header) => row[header] || '')], table.align);
+        }));
+    }
+    addItem(text) {
+        return new MarkdownQuery(this.editor().replace(this.matches, (range) => {
+            if (range.token.type !== 'list')
+                throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`);
+            const raw = (range.token.raw || '').trimEnd();
+            return `${raw}\n${MarkdownEditor.item(raw, text)}\n`;
+        }));
+    }
+    setEntry(key, value) {
+        return this.replaceEach((match) => {
+            const token = match.matches[0].token;
+            return MarkdownEditor.entries(this.tokenText(token), key, value, token.type === 'blockquote');
+        });
+    }
+    setFrontmatter(key, value) {
+        return new MarkdownQuery(this.editor().setFrontmatter(key, value));
+    }
+    toString() {
+        return this.source;
+    }
+    valueOf() {
+        return this.source;
+    }
+    /** Compatibility aliases. Each delegates to the canonical name above; prefer those. */
+    get() {
+        return this.text();
+    }
+    meta() {
+        return this.nodes();
+    }
+    toJson() {
+        return this.rows();
+    }
+    keyValue() {
+        return this.entries();
+    }
+    setKeyValue(key, value) {
+        return this.setEntry(key, value);
+    }
+    before() {
+        return this.preceding();
+    }
+    after() {
+        return this.following();
+    }
+    editor() {
+        return new MarkdownEditor(this.source);
+    }
+    run(candidates, segments) {
+        if (segments.length === 0)
+            return candidates;
+        const segment = segments[0];
+        const remaining = segments.slice(1);
+        if (/^section[1-6]?$/.test(segment.selector)) {
+            const sections = this.narrow(this.sectionsOf(candidates, segment), segment);
+            if (remaining.length === 0)
+                return sections;
+            const results = [];
+            for (const section of sections)
+                results.push(...this.run(section.innerTokens || [], remaining));
+            return results;
+        }
+        if (segment.selector === 'comment') {
+            const comments = candidates.filter((range) => this.isComment(range.token));
+            return this.run(this.narrow(this.filterByText(comments, segment), segment), remaining);
+        }
+        if (segment.selector === 'item') {
+            return this.run(this.narrow(this.filterByText(this.itemsOf(candidates), segment), segment), remaining);
+        }
+        const depth = segment.selector.match(/^h([1-6])$/);
+        let type = TOKEN_ALIASES[segment.selector] || segment.selector;
+        if (depth)
+            type = 'heading';
+        let matches = candidates.filter((range) => range.token.type === type);
+        if (depth)
+            matches = matches.filter((range) => range.token.depth === Number.parseInt(depth[1], 10));
+        return this.run(this.narrow(this.filterByText(matches, segment), segment), remaining);
+    }
+    sectionsOf(candidates, segment) {
+        const wanted = segment.selector.match(/^section([1-6])$/);
+        const sections = [];
+        for (let i = 0; i < candidates.length; i++) {
+            const range = candidates[i];
+            if (range.token.type !== 'heading')
+                continue;
+            const heading = range.token;
+            if (wanted && heading.depth !== Number.parseInt(wanted[1], 10))
+                continue;
+            if (segment.textMatch && !this.matchText(heading.text, segment.textMatch))
+                continue;
+            const innerTokens = [];
+            let end = range.start + range.length;
+            for (let j = i + 1; j < candidates.length; j++) {
+                const next = candidates[j];
+                if (next.token.type === 'heading' && next.token.depth <= heading.depth)
+                    break;
+                innerTokens.push(next);
+                end = MarkdownEditor.blockEnd(next);
+            }
+            sections.push({ token: range.token, start: range.start, length: end - range.start, innerTokens });
+        }
+        return sections;
+    }
+    itemsOf(candidates) {
+        const items = [];
+        for (const range of candidates) {
+            if (range.token.type !== 'list')
+                continue;
+            const raw = range.token.raw;
+            let cursor = 0;
+            for (const item of range.token.items) {
+                const itemRaw = item.raw;
+                const at = raw.indexOf(itemRaw, cursor);
+                if (at === -1)
+                    continue;
+                items.push({ token: item, start: range.start + at, length: itemRaw.length });
+                cursor = at + itemRaw.length;
+            }
+        }
+        return items;
+    }
+    expandSections(matches) {
+        if (!matches.some((range) => range.innerTokens))
+            return matches;
+        const expanded = [];
+        for (const range of matches) {
+            if (!range.innerTokens) {
+                expanded.push(range);
+                continue;
+            }
+            expanded.push({ token: range.token, start: range.start, length: (range.token.raw || '').length });
+            expanded.push(...range.innerTokens);
+        }
+        return expanded;
+    }
+    narrow(matches, segment) {
+        if (segment.index !== null) {
+            let index = segment.index;
+            if (index < 0)
+                index = matches.length + index;
+            if (index < 0 || index >= matches.length)
+                return [];
+            return [matches[index]];
+        }
+        if (segment.slice)
+            return matches.slice(segment.slice.from, segment.slice.to);
+        return matches;
+    }
+    filterByText(matches, segment) {
+        if (!segment.textMatch)
+            return matches;
+        return matches.filter((range) => this.matchText(this.tokenText(range.token), segment.textMatch));
+    }
+    matchText(text, matcher) {
+        let result = false;
+        if (matcher.mode === 'exact')
+            result = text === matcher.value;
+        if (matcher.mode === 'contains')
+            result = text.includes(matcher.value);
+        if (matcher.mode === 'regex')
+            result = new RegExp(matcher.value, matcher.flags || '').test(text);
+        if (matcher.mode === 'predicate')
+            result = matcher.predicate(text);
+        if (matcher.negated)
+            return !result;
+        return result;
+    }
+    valueMatcher(matcher) {
+        if (typeof matcher === 'function')
+            return { mode: 'predicate', value: '', negated: false, predicate: matcher };
+        if (matcher instanceof RegExp)
+            return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags };
+        return { mode: 'exact', value: matcher, negated: false };
+    }
+    tokenText(token) {
+        const value = token;
+        if (this.isComment(token))
+            return value.raw.trim().replace(/^<!--/, '').replace(/-->$/, '').trim();
+        if (token.type === 'html')
+            return value.raw || '';
+        if (token.type === 'table')
+            return [...(value.header || []).map((cell) => cell.text), ...(value.rows || []).flatMap((row) => row.map((cell) => cell.text))].join(', ');
+        return value.text || '';
+    }
+    isComment(token) {
+        if (token.type !== 'html')
+            return false;
+        return (token.raw || '').trimStart().startsWith('<!--');
+    }
+    insertAll(offsetOf, markdown) {
+        return new MarkdownQuery(this.editor().insert(this.matches.map(offsetOf), String(markdown)));
+    }
+    containerStart(range) {
+        if (!range.innerTokens)
+            throw new MdqOperationError(`prepend needs a section or list, got ${range.token.type}`);
+        return range.start + (range.token.raw || '').length;
+    }
+    containerEnd(range) {
+        if (!range.innerTokens)
+            throw new MdqOperationError(`append needs a section or list, got ${range.token.type}`);
+        const last = range.innerTokens[range.innerTokens.length - 1];
+        if (!last)
+            return this.containerStart(range);
+        return MarkdownEditor.blockEnd(last);
+    }
+}
+export class MdqError extends Error {
+}
+export class MdqSelectorError extends MdqError {
+    index;
+    constructor(message, index) {
+        super(message);
+        this.name = 'MdqSelectorError';
+        this.index = index;
+    }
+}
+export class MdqOperationError extends MdqError {
+    constructor(message) {
+        super(message);
+        this.name = 'MdqOperationError';
+    }
+}
+export function mdq(source) {
+    return new MarkdownQuery(String(source));
+}
+export function parseQuery(input) {
+    const segments = [];
+    SEGMENT.lastIndex = 0;
+    while (SEGMENT.lastIndex < input.length) {
+        const at = SEGMENT.lastIndex;
+        const match = SEGMENT.exec(input);
+        if (!match)
+            throw new MdqSelectorError(`Unexpected character "${input[at]}" in selector`, at);
+        const [, prefix, name, negated, contains, quoted, pattern, flags, brackets] = match;
+        if (!SELECTOR_NAMES.has(name) && !/^(?:h|section)[1-6]$/.test(name))
+            throw new MdqSelectorError(`Unknown selector "${name}"`, at + prefix.length);
+        const segment = { selector: name, index: null, slice: null };
+        if (quoted !== undefined) {
+            let mode = 'exact';
+            if (contains)
+                mode = 'contains';
+            segment.textMatch = { mode, value: quoted.replace(/\\(.)/g, '$1'), negated: Boolean(negated) };
+        }
+        if (pattern !== undefined) {
+            segment.textMatch = { mode: 'regex', value: pattern, negated: Boolean(negated), flags };
+        }
+        for (const [, bounds] of brackets.matchAll(/\[([^\]]*)\]/g)) {
+            if (!bounds || !/^-?\d*(?::-?\d*)?$/.test(bounds))
+                continue;
+            if (!bounds.includes(':')) {
+                segment.index = Number.parseInt(bounds, 10);
+                continue;
+            }
+            const [from, to] = bounds.split(':');
+            segment.slice = {};
+            if (from)
+                segment.slice.from = Number.parseInt(from, 10);
+            if (to)
+                segment.slice.to = Number.parseInt(to, 10);
+        }
+        segments.push(segment);
+    }
+    return segments;
+}
+export function buildTokenIndex(source) {
+    const { body, offset } = MarkdownEditor.splitFrontmatter(source);
+    const ranges = [];
+    let cursor = offset;
+    for (const token of marked.lexer(body)) {
+        const raw = token.raw || '';
+        if (token.type === 'space') {
+            const previous = ranges[ranges.length - 1];
+            if (previous)
+                previous.trailing = { start: cursor, length: raw.length };
+            cursor += raw.length;
+            continue;
+        }
+        ranges.push({ token, start: cursor, length: raw.length });
+        cursor += raw.length;
+    }
+    return ranges;
+}
+const SELECTOR_NAMES = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']);
+const TOKEN_ALIASES = { item: 'list_item' };
+const SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y;

package/dist/src/utils/step-analyzer.d.ts

@@ -9,3 +9,6 @@ export declare function stripComments(code: string): string;
 export declare function isNonReusableCode(code: string): boolean;
 export declare function toReusableSessionStep(step: StepData): SessionStep | null;
 export declare function mergeUniqueStepsByCode(primary: SessionStep[], secondary: SessionStep[]): SessionStep[];
+export declare function isInternalStep(step: {
+    title?: string;
+}): boolean;

package/dist/src/utils/step-analyzer.js

@@ -1,5 +1,6 @@
 import { isDynamicId } from "./xpath.js";
 export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'];
+const INTERNAL_STEP_PREFIXES = ['grab', 'save'];
 const CODECEPT_FORM_COMMANDS = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption'];
 export function isCodeceptToolName(toolName) {
     return CODECEPT_TOOLS.includes(toolName);
@@ -66,3 +67,9 @@ export function mergeUniqueStepsByCode(primary, secondary) {
     }
     return merged;
 }
+export function isInternalStep(step) {
+    const title = step?.title;
+    if (!title)
+        return false;
+    return INTERNAL_STEP_PREFIXES.some((prefix) => title.startsWith(prefix));
+}

package/dist/src/utils/strings.d.ts

@@ -4,4 +4,5 @@ export declare function normalizeInlineText(text: string): string;
 export declare function sanitizeFilename(name: string): string;
 export declare function safeFilename(name: string, ext?: string, maxBytes?: number): string;
 export declare function truncate(text: string, max: number): string;
+export declare function truncateMiddle(text: string, max: number): string;
 export declare function compactErrorMessage(error: unknown): string;

package/dist/src/utils/strings.js

@@ -42,6 +42,13 @@ export function truncate(text, max) {
         return text;
     return `${text.slice(0, max - 3)}...`;
 }
+export function truncateMiddle(text, max) {
+    if (text.length <= max)
+        return text;
+    const head = Math.ceil((max - 3) / 2);
+    const tail = max - 3 - head;
+    return `${text.slice(0, head)}...${text.slice(text.length - tail)}`;
+}
 const MAX_COMPACT_ERROR = 400;
 export function compactErrorMessage(error) {
     let text = stripAnsi(String(error));

package/dist/src/utils/test-files.d.ts

@@ -1,3 +1,4 @@
+export declare function listTestFiles(testsDir: string): string[];
 export declare function loadTestSuites(testsDir: string): any[];
 export declare function printTestList(suites: any[]): void;
 export declare function dryRunTestFile(filePath: string): Promise<void>;

package/dist/src/utils/test-files.js

@@ -8,12 +8,15 @@ import storeListener from 'codeceptjs/lib/listener/store';
 import store from 'codeceptjs/lib/store';
 import figureSet from 'figures';
 import { ConfigParser } from "../config.js";
-export function loadTestSuites(testsDir) {
+export function listTestFiles(testsDir) {
     if (!existsSync(testsDir))
         return [];
-    const jsFiles = readdirSync(testsDir)
+    return readdirSync(testsDir)
         .filter((f) => f.endsWith('.js'))
         .map((f) => path.resolve(testsDir, f));
+}
+export function loadTestSuites(testsDir) {
+    const jsFiles = listTestFiles(testsDir);
     if (jsFiles.length === 0)
         return [];
     codeceptjs.container.createMocha();

package/dist/src/utils/url-matcher.d.ts

@@ -6,3 +6,4 @@ export declare function generalizeUrl(url: string, replaceSegment?: (segment: st
 export declare function matchesUrl(pattern: string, path: string): boolean;
 export declare function extractStatePath(url: string): string;
 export declare function matchesNavigationUrl(expected: string, current: string): boolean;
+export declare function isSameHostFamily(urlA: string, urlB: string): boolean;

package/dist/src/utils/url-matcher.js

@@ -145,3 +145,10 @@ export function matchesNavigationUrl(expected, current) {
         .filter(Boolean);
     return recordSegments.length > 0 && recordSegments.every(isDynamicSegment);
 }
+export function isSameHostFamily(urlA, urlB) {
+    const hostA = URL.parse(urlA)?.host.toLowerCase();
+    const hostB = URL.parse(urlB)?.host.toLowerCase();
+    if (!hostA || !hostB)
+        return false;
+    return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`);
+}

package/docs/api-testing/planning.md

@@ -21,7 +21,7 @@ endpoint: "/users"
 CRUD for users. Admin role required for writes. IDs are UUIDs.
 ```
 
-Chief loads knowledge matching the endpoint it's planning, and Curler loads it again for the endpoint it's testing, so auth headers and payload rules reach the requests themselves. Running `know` again on the same endpoint appends to the file. See [knowledge](../workflow/knowledge.md) for how matching and files work.
+Chief loads knowledge matching the endpoint it's planning, and Curler loads it again for the endpoint it's testing. When `plan` or `explore` runs, Chief first picks out the headers, query parameters and body fields the knowledge says to send, and every request whose path matches the file's `endpoint:` pattern carries them, the sample data fetch included. Headers from `api.headers` or `-H` take precedence over knowledge. Running `know` again on the same endpoint appends to the file. See [knowledge](../workflow/knowledge.md) for how matching and files work.
 
 For a fact that should not be stored — a token, a one-off fixture — pass `--knowledge` instead. It applies to the run only: