explorbot 0.4.10 → 0.5.0

package/dist/src/utils/markdown-query.js

@@ -1,444 +1 @@
-import { marked } from 'marked';
-export function parseQuery(input) {
-    const segments = [];
-    let pos = 0;
-    function peek() {
-        return pos < input.length ? input[pos] : '';
-    }
-    function advance() {
-        return input[pos++] || '';
-    }
-    function skipWhitespace() {
-        while (pos < input.length && /\s/.test(input[pos]))
-            pos++;
-    }
-    function readIdentifier() {
-        const start = pos;
-        while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos]))
-            pos++;
-        return input.slice(start, pos);
-    }
-    function readQuotedString() {
-        const quote = advance();
-        let result = '';
-        while (pos < input.length && input[pos] !== quote) {
-            if (input[pos] === '\\') {
-                pos++;
-                result += input[pos] || '';
-            }
-            else {
-                result += input[pos];
-            }
-            pos++;
-        }
-        if (pos < input.length)
-            pos++;
-        return result;
-    }
-    function readUntilAny(chars) {
-        const start = pos;
-        while (pos < input.length && !chars.includes(input[pos]))
-            pos++;
-        return input.slice(start, pos);
-    }
-    function parseTextMatcher() {
-        let negated = false;
-        if (peek() === '!') {
-            negated = true;
-            advance();
-        }
-        if (peek() === '~') {
-            advance();
-            const value = readQuotedString();
-            return { mode: 'contains', value, negated };
-        }
-        if (peek() === '/') {
-            advance();
-            let value = '';
-            while (pos < input.length && input[pos] !== '/') {
-                value += input[pos];
-                pos++;
-            }
-            if (pos < input.length)
-                pos++;
-            while (pos < input.length && /[gimsuy]/.test(input[pos]))
-                pos++;
-            return { mode: 'regex', value, negated };
-        }
-        const value = readQuotedString();
-        return { mode: 'exact', value, negated };
-    }
-    while (pos < input.length) {
-        skipWhitespace();
-        if (pos >= input.length)
-            break;
-        const selector = readIdentifier();
-        if (!selector) {
-            pos++;
-            continue;
-        }
-        const segment = {
-            selector: selector,
-            index: null,
-            slice: null,
-        };
-        if (peek() === '(') {
-            advance();
-            segment.textMatch = parseTextMatcher();
-            if (peek() === ')')
-                advance();
-        }
-        while (peek() === '[') {
-            advance();
-            const content = readUntilAny(']');
-            if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') {
-                if (content.includes(':')) {
-                    const colonIdx = content.indexOf(':');
-                    const fromStr = content.slice(0, colonIdx);
-                    const toStr = content.slice(colonIdx + 1);
-                    segment.slice = {
-                        from: fromStr ? Number.parseInt(fromStr, 10) : undefined,
-                        to: toStr ? Number.parseInt(toStr, 10) : undefined,
-                    };
-                }
-                else {
-                    segment.index = Number.parseInt(content, 10);
-                }
-            }
-            if (peek() === ']')
-                advance();
-        }
-        segments.push(segment);
-    }
-    return segments;
-}
-function matchText(text, matcher) {
-    let result;
-    switch (matcher.mode) {
-        case 'exact':
-            result = text === matcher.value;
-            break;
-        case 'contains':
-            result = text.includes(matcher.value);
-            break;
-        case 'regex':
-            result = new RegExp(matcher.value, 'i').test(text);
-            break;
-        default:
-            result = false;
-    }
-    return matcher.negated ? !result : result;
-}
-function entryKey(line) {
-    const separator = line.indexOf(':');
-    if (separator < 1)
-        return null;
-    return line.slice(0, separator).trim().toLowerCase();
-}
-function getTokenText(token) {
-    const t = token;
-    switch (token.type) {
-        case 'heading':
-        case 'paragraph':
-        case 'code':
-        case 'blockquote':
-        case 'list_item':
-            return t.text || '';
-        case 'table':
-            return (t.header || []).map((h) => h.text).join(', ');
-        default:
-            return '';
-    }
-}
-function getHeadingDepth(selector) {
-    const match = selector.match(/^h([1-6])$/);
-    return match ? Number.parseInt(match[1], 10) : null;
-}
-function isSectionSelector(selector) {
-    return /^section\d?$/.test(selector);
-}
-function getSectionDepth(selector) {
-    const match = selector.match(/^section([1-6])$/);
-    return match ? Number.parseInt(match[1], 10) : null;
-}
-function selectorToTokenType(selector) {
-    if (/^h[1-6]$/.test(selector))
-        return 'heading';
-    const map = {
-        heading: 'heading',
-        paragraph: 'paragraph',
-        table: 'table',
-        code: 'code',
-        list: 'list',
-        blockquote: 'blockquote',
-        hr: 'hr',
-        item: 'list_item',
-    };
-    return map[selector] || null;
-}
-function buildTokenIndex(source) {
-    const tokens = marked.lexer(source);
-    const ranges = [];
-    let offset = 0;
-    for (const token of tokens) {
-        const raw = token.raw || '';
-        ranges.push({ token, start: offset, length: raw.length });
-        offset += raw.length;
-    }
-    return ranges;
-}
-function computeSections(candidates, segment) {
-    const sectionDepth = getSectionDepth(segment.selector);
-    const sections = [];
-    for (let i = 0; i < candidates.length; i++) {
-        const range = candidates[i];
-        if (range.token.type !== 'heading')
-            continue;
-        const heading = range.token;
-        if (sectionDepth !== null && heading.depth !== sectionDepth)
-            continue;
-        if (segment.textMatch && !matchText(heading.text, segment.textMatch))
-            continue;
-        const depth = heading.depth;
-        const innerTokens = [];
-        let endOffset = range.start + range.length;
-        for (let j = i + 1; j < candidates.length; j++) {
-            const nextRange = candidates[j];
-            if (nextRange.token.type === 'heading' && nextRange.token.depth <= depth)
-                break;
-            innerTokens.push(nextRange);
-            endOffset = nextRange.start + nextRange.length;
-        }
-        sections.push({
-            token: range.token,
-            start: range.start,
-            length: endOffset - range.start,
-            innerTokens,
-        });
-    }
-    return sections;
-}
-function extractListItems(candidates) {
-    const items = [];
-    for (const range of candidates) {
-        if (range.token.type !== 'list')
-            continue;
-        const list = range.token;
-        const listRaw = range.token.raw;
-        let searchFrom = 0;
-        for (const item of list.items) {
-            const itemRaw = item.raw;
-            const idx = listRaw.indexOf(itemRaw, searchFrom);
-            if (idx === -1)
-                continue;
-            items.push({
-                token: item,
-                start: range.start + idx,
-                length: itemRaw.length,
-            });
-            searchFrom = idx + itemRaw.length;
-        }
-    }
-    return items;
-}
-function applyIndexSlice(matches, segment) {
-    if (segment.index !== null) {
-        const idx = segment.index < 0 ? matches.length + segment.index : segment.index;
-        return idx >= 0 && idx < matches.length ? [matches[idx]] : [];
-    }
-    if (segment.slice) {
-        const { from, to } = segment.slice;
-        return matches.slice(from, to);
-    }
-    return matches;
-}
-function expandSectionRanges(matches) {
-    let hasSection = false;
-    for (const m of matches) {
-        if (m.innerTokens) {
-            hasSection = true;
-            break;
-        }
-    }
-    if (!hasSection)
-        return matches;
-    const expanded = [];
-    for (const m of matches) {
-        if (m.innerTokens) {
-            expanded.push({ token: m.token, start: m.start, length: (m.token.raw || '').length });
-            expanded.push(...m.innerTokens);
-        }
-        else {
-            expanded.push(m);
-        }
-    }
-    return expanded;
-}
-function executeSegments(candidates, segments) {
-    if (segments.length === 0)
-        return candidates;
-    const segment = segments[0];
-    const remaining = segments.slice(1);
-    if (isSectionSelector(segment.selector)) {
-        const sections = computeSections(candidates, segment);
-        const indexed = applyIndexSlice(sections, segment);
-        if (remaining.length === 0)
-            return indexed;
-        const results = [];
-        for (const section of indexed) {
-            results.push(...executeSegments(section.innerTokens || [], remaining));
-        }
-        return results;
-    }
-    if (segment.selector === 'item') {
-        let items = extractListItems(candidates);
-        if (segment.textMatch) {
-            items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch));
-        }
-        return executeSegments(applyIndexSlice(items, segment), remaining);
-    }
-    const tokenType = selectorToTokenType(segment.selector);
-    if (!tokenType)
-        return [];
-    let matches = candidates.filter((r) => r.token.type === tokenType);
-    const depth = getHeadingDepth(segment.selector);
-    if (depth !== null) {
-        matches = matches.filter((r) => r.token.depth === depth);
-    }
-    if (segment.textMatch) {
-        matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch));
-    }
-    return executeSegments(applyIndexSlice(matches, segment), remaining);
-}
-export class MarkdownQuery {
-    source;
-    matches;
-    constructor(source, matches) {
-        this.source = source;
-        this.matches = matches || buildTokenIndex(source);
-    }
-    query(selector) {
-        const segments = parseQuery(selector);
-        const candidates = expandSectionRanges(this.matches);
-        const results = executeSegments(candidates, segments);
-        return new MarkdownQuery(this.source, results);
-    }
-    text() {
-        return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join('');
-    }
-    get() {
-        return this.text();
-    }
-    toJson() {
-        const results = [];
-        for (const range of this.matches) {
-            if (range.token.type !== 'table')
-                continue;
-            const table = range.token;
-            const headers = table.header.map((h) => h.text);
-            for (const row of table.rows) {
-                const obj = {};
-                for (let i = 0; i < headers.length; i++) {
-                    obj[headers[i]] = row[i]?.text ?? '';
-                }
-                results.push(obj);
-            }
-        }
-        return results;
-    }
-    keyValue() {
-        const entries = {};
-        for (const range of this.matches) {
-            for (const line of getTokenText(range.token).split('\n')) {
-                const key = entryKey(line);
-                if (!key)
-                    continue;
-                const value = line.slice(line.indexOf(':') + 1).trim();
-                if (value)
-                    entries[key] = value;
-            }
-        }
-        return entries;
-    }
-    setKeyValue(key, value) {
-        return this.replaceEach((match) => {
-            const token = match.matches[0].token;
-            const lines = getTokenText(token)
-                .split('\n')
-                .map((line) => line.trim())
-                .filter(Boolean);
-            const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase());
-            if (index < 0 && value)
-                lines.push(`${key}: ${value}`);
-            if (index >= 0 && value)
-                lines[index] = `${key}: ${value}`;
-            if (index >= 0 && !value)
-                lines.splice(index, 1);
-            if (token.type !== 'blockquote')
-                return lines.join('\n');
-            return lines.map((line) => `> ${line}`).join('\n');
-        });
-    }
-    replace(content) {
-        return this.replaceEach(() => content);
-    }
-    replaceEach(replacer) {
-        const sorted = [...this.matches].sort((a, b) => a.start - b.start);
-        const kept = [];
-        let lastEnd = -1;
-        for (const range of sorted) {
-            if (range.start < lastEnd)
-                continue;
-            kept.push(range);
-            lastEnd = range.start + range.length;
-        }
-        const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index));
-        let result = this.source;
-        for (let i = kept.length - 1; i >= 0; i--) {
-            const range = kept[i];
-            result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length);
-        }
-        return result;
-    }
-    count() {
-        return this.matches.length;
-    }
-    first() {
-        return new MarkdownQuery(this.source, this.matches.slice(0, 1));
-    }
-    last() {
-        return new MarkdownQuery(this.source, this.matches.slice(-1));
-    }
-    before() {
-        if (this.matches.length === 0)
-            return new MarkdownQuery(this.source, []);
-        const cutoff = this.matches[0].start;
-        const allTokens = buildTokenIndex(this.source);
-        const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff);
-        return new MarkdownQuery(this.source, beforeTokens);
-    }
-    after() {
-        if (this.matches.length === 0)
-            return new MarkdownQuery(this.source, []);
-        const lastMatch = this.matches[this.matches.length - 1];
-        const cutoff = lastMatch.start + lastMatch.length;
-        const allTokens = buildTokenIndex(this.source);
-        const afterTokens = allTokens.filter((r) => r.start >= cutoff);
-        return new MarkdownQuery(this.source, afterTokens);
-    }
-    each() {
-        return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
-    }
-    meta() {
-        return this.matches.map((range) => {
-            const token = range.token;
-            let depth = null;
-            if (token.type === 'heading')
-                depth = token.depth;
-            return { type: token.type, depth, text: getTokenText(range.token) };
-        });
-    }
-}
-export function mdq(source) {
-    return new MarkdownQuery(source);
-}
+export * from "./mdq/query.js";

package/dist/src/utils/mdq/cli.d.ts

@@ -0,0 +1,6 @@
+export declare function runMdq(argv: string[], readStdin: StdinReader): Promise<CliResult>;
+export type StdinReader = () => Promise<string>;
+export interface CliResult {
+    output: string;
+    code: number;
+}

package/dist/src/utils/mdq/cli.js

@@ -0,0 +1,122 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { Command } from 'commander';
+import { MdqError, mdq } from "./query.js";
+const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'];
+export async function runMdq(argv, readStdin) {
+    const program = new Command();
+    program
+        .name('mdq')
+        .description('query and edit markdown')
+        .argument('[selector]', 'markdown selector')
+        .argument('[file]', 'file to read; stdin when omitted')
+        .option('-j, --json', 'output rows as JSON')
+        .option('-c, --count', 'print the number of matches')
+        .option('-t, --text', 'print unwrapped text')
+        .option('--frontmatter', 'print frontmatter as JSON')
+        .option('-i, --in-place', 'write the result back to the file')
+        .option('--remove', 'delete matched blocks')
+        .option('--replace <markdown>', 'replace matched blocks')
+        .option('--insert-before <markdown>', 'insert before each match')
+        .option('--insert-after <markdown>', 'insert after each match')
+        .option('--prepend <markdown>', 'insert at the start of each match')
+        .option('--append <markdown>', 'insert at the end of each match')
+        .option('--add-row <json>', 'append a table row')
+        .option('--add-item <text>', 'append a list item')
+        .option('--set <key=value>', 'set an entry; omit the value to delete it')
+        .exitOverride()
+        .configureOutput({ writeOut: () => { }, writeErr: () => { } });
+    try {
+        program.parse(argv, { from: 'user' });
+    }
+    catch (error) {
+        return { output: String(error.message), code: 2 };
+    }
+    const options = program.opts();
+    let [selector, file] = program.args;
+    if (options.frontmatter && selector && !file) {
+        file = selector;
+        selector = '';
+    }
+    let source = '';
+    if (!file)
+        source = await readStdin();
+    if (file) {
+        try {
+            source = readFileSync(file, 'utf8');
+        }
+        catch {
+            return { output: `Cannot read ${file}`, code: 2 };
+        }
+    }
+    try {
+        return await apply(mdq(source), selector, options, file);
+    }
+    catch (error) {
+        if (error instanceof MdqError)
+            return { output: error.message, code: 2 };
+        throw error;
+    }
+}
+async function apply(doc, selector, options, file) {
+    if (options.frontmatter)
+        return { output: `${JSON.stringify(doc.frontmatter(), null, 2)}\n`, code: 0 };
+    if (!selector)
+        return { output: 'A selector is required', code: 2 };
+    const chosen = EDIT_FLAGS.filter((flag) => options[flag] !== undefined);
+    if (chosen.length > 1)
+        return { output: 'Only one edit at a time', code: 2 };
+    const selection = doc.query(selector);
+    if (chosen.length === 0)
+        return read(selection, options);
+    if (!selection.exists())
+        return finish(String(doc), 1, options, file);
+    return finish(String(edit(selection, chosen[0], options)), 0, options, file);
+}
+function read(selection, options) {
+    let code = 1;
+    if (selection.exists())
+        code = 0;
+    if (options.count)
+        return { output: `${selection.count()}\n`, code: 0 };
+    if (options.json)
+        return { output: `${JSON.stringify(selection.rows(), null, 2)}\n`, code };
+    if (options.text)
+        return {
+            output: selection
+                .nodes()
+                .map((node) => node.text)
+                .join('\n\n'),
+            code,
+        };
+    return { output: selection.text(), code };
+}
+function edit(selection, flag, options) {
+    if (flag === 'remove')
+        return selection.remove();
+    if (flag === 'replace')
+        return selection.replace(options.replace);
+    if (flag === 'insertBefore')
+        return selection.insertBefore(options.insertBefore);
+    if (flag === 'insertAfter')
+        return selection.insertAfter(options.insertAfter);
+    if (flag === 'prepend')
+        return selection.prepend(options.prepend);
+    if (flag === 'append')
+        return selection.append(options.append);
+    if (flag === 'addItem')
+        return selection.addItem(options.addItem);
+    if (flag === 'addRow')
+        return selection.addRow(JSON.parse(options.addRow));
+    const separator = options.set.indexOf('=');
+    if (separator < 0)
+        return selection.setEntry(options.set, null);
+    return selection.setEntry(options.set.slice(0, separator), options.set.slice(separator + 1) || null);
+}
+function finish(output, code, options, file) {
+    if (!options.inPlace)
+        return { output, code };
+    if (!file)
+        return { output: '--in-place needs a file', code: 2 };
+    writeFileSync(file, output);
+    return { output: '', code };
+}

package/dist/src/utils/mdq/edit.d.ts

@@ -0,0 +1,24 @@
+import type { MatchedRange } from './query.js';
+export declare class MarkdownEditor {
+    source: string;
+    constructor(source: string);
+    static splitFrontmatter(source: string): FrontmatterSplit;
+    static blockEnd(range: MatchedRange): number;
+    static table(headers: string[], rows: string[][], align: (string | null)[]): string;
+    static item(listRaw: string, text: string): string;
+    static entries(text: string, key: string, value: string | null, isBlockquote: boolean): string;
+    remove(ranges: MatchedRange[]): string;
+    replace(ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string;
+    insert(offsets: number[], markdown: string): string;
+    frontmatter(): Record<string, unknown>;
+    setFrontmatter(key: string, value: unknown): string;
+    static spliceIn(source: string, offset: number, markdown: string): string;
+    static dashes(alignment: string | null, width: number): string;
+    static entryKey(line: string): string | null;
+    dedupe(ranges: MatchedRange[]): MatchedRange[];
+}
+export interface FrontmatterSplit {
+    raw: string;
+    body: string;
+    offset: number;
+}