explorbot 0.0.1 → 0.0.5

package/src/utils/markdown-parser.ts

@@ -0,0 +1,127 @@
+import { marked } from 'marked';
+
+export interface MarkdownSection {
+  name: string;
+  depth: number;
+  rawMarkdown: string;
+}
+
+export function parseSections(markdown: string, opts?: { minDepth?: number; maxDepth?: number }): MarkdownSection[] {
+  const { minDepth = 2, maxDepth = 3 } = opts || {};
+  const tokens = marked.lexer(markdown);
+  const sections: MarkdownSection[] = [];
+  let current: MarkdownSection | null = null;
+
+  for (const token of tokens) {
+    const depth = (token as any).depth;
+    if (token.type === 'heading' && depth >= minDepth && depth <= maxDepth) {
+      current = { name: (token as any).text, depth, rawMarkdown: (token as any).raw };
+      sections.push(current);
+      continue;
+    }
+    if (current) {
+      current.rawMarkdown += (token as any).raw || '';
+    }
+  }
+
+  return sections;
+}
+
+export function extractLabeledCode(markdown: string, labelPattern: RegExp): string | null {
+  const tokens = marked.lexer(markdown);
+  let expectValueInNext = false;
+
+  for (const token of tokens) {
+    if (token.type === 'heading' && labelPattern.test((token as any).text || '')) {
+      expectValueInNext = true;
+      continue;
+    }
+
+    if (expectValueInNext) {
+      if (token.type === 'code') {
+        const val = ((token as any).text as string).trim();
+        if (val && !val.includes('\n')) return val;
+        expectValueInNext = false;
+        continue;
+      }
+      if (token.type === 'paragraph') {
+        const codespan = ((token as any).tokens || []).find((t: any) => t.type === 'codespan');
+        if (codespan) return codespan.text;
+        expectValueInNext = false;
+      }
+      if (token.type !== 'space') expectValueInNext = false;
+    }
+
+    if (token.type === 'paragraph' || token.type === 'blockquote') {
+      const raw = (token as any).raw || '';
+      if (!labelPattern.test(raw)) continue;
+
+      const codespans = ((token as any).tokens || []).filter((t: any) => t.type === 'codespan');
+      if (codespans.length > 0) return codespans[codespans.length - 1].text;
+
+      const match = raw.match(labelPattern);
+      if (match) {
+        const afterLabel = raw.substring(match.index! + match[0].length);
+        const quotedMatch = afterLabel.match(/['"`]([^'"`]+)['"`]/);
+        if (quotedMatch) return quotedMatch[1];
+      }
+
+      expectValueInNext = true;
+    }
+  }
+
+  return null;
+}
+
+export function tableToJson(markdown: string): Record<string, string>[] {
+  const tokens = marked.lexer(markdown);
+  for (const token of tokens) {
+    if (token.type !== 'table') continue;
+    const tableToken = token as any;
+    const headers: string[] = tableToken.header.map((h: any) => h.text);
+    return tableToken.rows.map((row: any) => {
+      const obj: Record<string, string> = {};
+      for (let i = 0; i < headers.length; i++) {
+        obj[headers[i]] = row[i]?.text ?? '';
+      }
+      return obj;
+    });
+  }
+  return [];
+}
+
+export function jsonToTable(rows: Record<string, string>[], columns?: string[]): string {
+  if (rows.length === 0) return '';
+  const headers = columns || Object.keys(rows[0]);
+  let md = `| ${headers.join(' | ')} |\n`;
+  md += `|${headers.map(() => '------|').join('')}\n`;
+  for (const row of rows) {
+    md += `| ${headers.map((h) => row[h] ?? '-').join(' | ')} |\n`;
+  }
+  return md;
+}
+
+export function findTableRaw(markdown: string): string | null {
+  const tokens = marked.lexer(markdown);
+  for (const token of tokens) {
+    if (token.type === 'table') return (token as any).raw;
+  }
+  return null;
+}
+
+export function findTableLineRange(markdown: string): { start: number; end: number } | null {
+  const lines = markdown.split('\n');
+  let start = -1;
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (trimmed.startsWith('|') && trimmed.endsWith('|')) {
+      if (start === -1) start = i;
+    } else if (start !== -1) {
+      return { start, end: i };
+    }
+  }
+
+  if (start !== -1) return { start, end: lines.length };
+  return null;
+}

package/src/utils/markdown-query.ts

@@ -0,0 +1,466 @@
+import { type Token, type Tokens, marked } from 'marked';
+
+export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6';
+
+export interface TextMatcher {
+  mode: 'exact' | 'contains' | 'regex';
+  value: string;
+  negated: boolean;
+}
+
+export interface QuerySegment {
+  selector: SelectorType;
+  textMatch?: TextMatcher;
+  index: number | null;
+  slice: { from?: number; to?: number } | null;
+}
+
+export interface MatchedRange {
+  token: Token;
+  start: number;
+  length: number;
+  innerTokens?: MatchedRange[];
+}
+
+export function parseQuery(input: string): QuerySegment[] {
+  const segments: QuerySegment[] = [];
+  let pos = 0;
+
+  function peek(): string {
+    return pos < input.length ? input[pos] : '';
+  }
+
+  function advance(): string {
+    return input[pos++] || '';
+  }
+
+  function skipWhitespace() {
+    while (pos < input.length && /\s/.test(input[pos])) pos++;
+  }
+
+  function readIdentifier(): string {
+    const start = pos;
+    while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos])) pos++;
+    return input.slice(start, pos);
+  }
+
+  function readQuotedString(): string {
+    const quote = advance();
+    let result = '';
+    while (pos < input.length && input[pos] !== quote) {
+      if (input[pos] === '\\') {
+        pos++;
+        result += input[pos] || '';
+      } else {
+        result += input[pos];
+      }
+      pos++;
+    }
+    if (pos < input.length) pos++;
+    return result;
+  }
+
+  function readUntilAny(chars: string): string {
+    const start = pos;
+    while (pos < input.length && !chars.includes(input[pos])) pos++;
+    return input.slice(start, pos);
+  }
+
+  function parseTextMatcher(): TextMatcher {
+    let negated = false;
+    if (peek() === '!') {
+      negated = true;
+      advance();
+    }
+
+    if (peek() === '~') {
+      advance();
+      const value = readQuotedString();
+      return { mode: 'contains', value, negated };
+    }
+
+    if (peek() === '/') {
+      advance();
+      let value = '';
+      while (pos < input.length && input[pos] !== '/') {
+        value += input[pos];
+        pos++;
+      }
+      if (pos < input.length) pos++;
+      while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++;
+      return { mode: 'regex', value, negated };
+    }
+
+    const value = readQuotedString();
+    return { mode: 'exact', value, negated };
+  }
+
+  while (pos < input.length) {
+    skipWhitespace();
+    if (pos >= input.length) break;
+
+    const selector = readIdentifier();
+    if (!selector) {
+      pos++;
+      continue;
+    }
+
+    const segment: QuerySegment = {
+      selector: selector as SelectorType,
+      index: null,
+      slice: null,
+    };
+
+    if (peek() === '(') {
+      advance();
+      segment.textMatch = parseTextMatcher();
+      if (peek() === ')') advance();
+    }
+
+    while (peek() === '[') {
+      advance();
+      const content = readUntilAny(']');
+      if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') {
+        if (content.includes(':')) {
+          const colonIdx = content.indexOf(':');
+          const fromStr = content.slice(0, colonIdx);
+          const toStr = content.slice(colonIdx + 1);
+          segment.slice = {
+            from: fromStr ? Number.parseInt(fromStr, 10) : undefined,
+            to: toStr ? Number.parseInt(toStr, 10) : undefined,
+          };
+        } else {
+          segment.index = Number.parseInt(content, 10);
+        }
+      }
+      if (peek() === ']') advance();
+    }
+
+    segments.push(segment);
+  }
+
+  return segments;
+}
+
+function matchText(text: string, matcher: TextMatcher): boolean {
+  let result: boolean;
+
+  switch (matcher.mode) {
+    case 'exact':
+      result = text === matcher.value;
+      break;
+    case 'contains':
+      result = text.includes(matcher.value);
+      break;
+    case 'regex':
+      result = new RegExp(matcher.value, 'i').test(text);
+      break;
+    default:
+      result = false;
+  }
+
+  return matcher.negated ? !result : result;
+}
+
+function getTokenText(token: Token): string {
+  const t = token as any;
+  switch (token.type) {
+    case 'heading':
+    case 'paragraph':
+    case 'code':
+    case 'blockquote':
+    case 'list_item':
+      return t.text || '';
+    case 'table':
+      return (t.header || []).map((h: any) => h.text).join(', ');
+    default:
+      return '';
+  }
+}
+
+function getHeadingDepth(selector: string): number | null {
+  const match = selector.match(/^h([1-6])$/);
+  return match ? Number.parseInt(match[1], 10) : null;
+}
+
+function isSectionSelector(selector: string): boolean {
+  return /^section\d?$/.test(selector);
+}
+
+function getSectionDepth(selector: string): number | null {
+  const match = selector.match(/^section([1-6])$/);
+  return match ? Number.parseInt(match[1], 10) : null;
+}
+
+function selectorToTokenType(selector: string): string | null {
+  if (/^h[1-6]$/.test(selector)) return 'heading';
+  const map: Record<string, string> = {
+    heading: 'heading',
+    paragraph: 'paragraph',
+    table: 'table',
+    code: 'code',
+    list: 'list',
+    blockquote: 'blockquote',
+    hr: 'hr',
+    item: 'list_item',
+  };
+  return map[selector] || null;
+}
+
+function buildTokenIndex(source: string): MatchedRange[] {
+  const tokens = marked.lexer(source);
+  const ranges: MatchedRange[] = [];
+  let offset = 0;
+
+  for (const token of tokens) {
+    const raw = (token as any).raw || '';
+    ranges.push({ token, start: offset, length: raw.length });
+    offset += raw.length;
+  }
+
+  return ranges;
+}
+
+function computeSections(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] {
+  const sectionDepth = getSectionDepth(segment.selector as string);
+  const sections: MatchedRange[] = [];
+
+  for (let i = 0; i < candidates.length; i++) {
+    const range = candidates[i];
+    if (range.token.type !== 'heading') continue;
+
+    const heading = range.token as Tokens.Heading;
+    if (sectionDepth !== null && heading.depth !== sectionDepth) continue;
+    if (segment.textMatch && !matchText(heading.text, segment.textMatch)) continue;
+
+    const depth = heading.depth;
+    const innerTokens: MatchedRange[] = [];
+    let endOffset = range.start + range.length;
+
+    for (let j = i + 1; j < candidates.length; j++) {
+      const nextRange = candidates[j];
+      if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break;
+      innerTokens.push(nextRange);
+      endOffset = nextRange.start + nextRange.length;
+    }
+
+    sections.push({
+      token: range.token,
+      start: range.start,
+      length: endOffset - range.start,
+      innerTokens,
+    });
+  }
+
+  return sections;
+}
+
+function extractListItems(candidates: MatchedRange[]): MatchedRange[] {
+  const items: MatchedRange[] = [];
+
+  for (const range of candidates) {
+    if (range.token.type !== 'list') continue;
+
+    const list = range.token as Tokens.List;
+    const listRaw = (range.token as any).raw as string;
+    let searchFrom = 0;
+
+    for (const item of list.items) {
+      const itemRaw = (item as any).raw as string;
+      const idx = listRaw.indexOf(itemRaw, searchFrom);
+      if (idx === -1) continue;
+
+      items.push({
+        token: item as unknown as Token,
+        start: range.start + idx,
+        length: itemRaw.length,
+      });
+
+      searchFrom = idx + itemRaw.length;
+    }
+  }
+
+  return items;
+}
+
+function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] {
+  if (segment.index !== null) {
+    const idx = segment.index < 0 ? matches.length + segment.index : segment.index;
+    return idx >= 0 && idx < matches.length ? [matches[idx]] : [];
+  }
+
+  if (segment.slice) {
+    const { from, to } = segment.slice;
+    return matches.slice(from, to);
+  }
+
+  return matches;
+}
+
+function expandSectionRanges(matches: MatchedRange[]): MatchedRange[] {
+  let hasSection = false;
+  for (const m of matches) {
+    if (m.innerTokens) {
+      hasSection = true;
+      break;
+    }
+  }
+  if (!hasSection) return matches;
+
+  const expanded: MatchedRange[] = [];
+  for (const m of matches) {
+    if (m.innerTokens) {
+      expanded.push({ token: m.token, start: m.start, length: ((m.token as any).raw || '').length });
+      expanded.push(...m.innerTokens);
+    } else {
+      expanded.push(m);
+    }
+  }
+  return expanded;
+}
+
+function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] {
+  if (segments.length === 0) return candidates;
+
+  const segment = segments[0];
+  const remaining = segments.slice(1);
+
+  if (isSectionSelector(segment.selector as string)) {
+    const sections = computeSections(candidates, segment);
+    const indexed = applyIndexSlice(sections, segment);
+
+    if (remaining.length === 0) return indexed;
+
+    const results: MatchedRange[] = [];
+    for (const section of indexed) {
+      results.push(...executeSegments(section.innerTokens || [], remaining));
+    }
+    return results;
+  }
+
+  if (segment.selector === 'item') {
+    let items = extractListItems(candidates);
+    if (segment.textMatch) {
+      items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch!));
+    }
+    return executeSegments(applyIndexSlice(items, segment), remaining);
+  }
+
+  const tokenType = selectorToTokenType(segment.selector as string);
+  if (!tokenType) return [];
+
+  let matches = candidates.filter((r) => r.token.type === tokenType);
+
+  const depth = getHeadingDepth(segment.selector as string);
+  if (depth !== null) {
+    matches = matches.filter((r) => (r.token as any).depth === depth);
+  }
+
+  if (segment.textMatch) {
+    matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch!));
+  }
+
+  return executeSegments(applyIndexSlice(matches, segment), remaining);
+}
+
+export class MarkdownQuery {
+  private source: string;
+  private matches: MatchedRange[];
+
+  constructor(source: string, matches?: MatchedRange[]) {
+    this.source = source;
+    this.matches = matches || buildTokenIndex(source);
+  }
+
+  query(selector: string): MarkdownQuery {
+    const segments = parseQuery(selector);
+    const candidates = expandSectionRanges(this.matches);
+    const results = executeSegments(candidates, segments);
+    return new MarkdownQuery(this.source, results);
+  }
+
+  text(): string {
+    return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join('');
+  }
+
+  get(): string {
+    return this.text();
+  }
+
+  toJson(): Record<string, string>[] {
+    const results: Record<string, string>[] = [];
+
+    for (const range of this.matches) {
+      if (range.token.type !== 'table') continue;
+
+      const table = range.token as Tokens.Table;
+      const headers = table.header.map((h) => h.text);
+      for (const row of table.rows) {
+        const obj: Record<string, string> = {};
+        for (let i = 0; i < headers.length; i++) {
+          obj[headers[i]] = row[i]?.text ?? '';
+        }
+        results.push(obj);
+      }
+    }
+
+    return results;
+  }
+
+  replace(content: string): string {
+    const sorted = [...this.matches].sort((a, b) => a.start - b.start);
+
+    const kept: MatchedRange[] = [];
+    let lastEnd = -1;
+    for (const range of sorted) {
+      if (range.start < lastEnd) continue;
+      kept.push(range);
+      lastEnd = range.start + range.length;
+    }
+
+    let result = this.source;
+    for (let i = kept.length - 1; i >= 0; i--) {
+      const range = kept[i];
+      result = result.slice(0, range.start) + content + result.slice(range.start + range.length);
+    }
+
+    return result;
+  }
+
+  count(): number {
+    return this.matches.length;
+  }
+
+  first(): MarkdownQuery {
+    return new MarkdownQuery(this.source, this.matches.slice(0, 1));
+  }
+
+  last(): MarkdownQuery {
+    return new MarkdownQuery(this.source, this.matches.slice(-1));
+  }
+
+  before(): MarkdownQuery {
+    if (this.matches.length === 0) return new MarkdownQuery(this.source, []);
+    const cutoff = this.matches[0].start;
+    const allTokens = buildTokenIndex(this.source);
+    const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff);
+    return new MarkdownQuery(this.source, beforeTokens);
+  }
+
+  after(): MarkdownQuery {
+    if (this.matches.length === 0) return new MarkdownQuery(this.source, []);
+    const lastMatch = this.matches[this.matches.length - 1];
+    const cutoff = lastMatch.start + lastMatch.length;
+    const allTokens = buildTokenIndex(this.source);
+    const afterTokens = allTokens.filter((r) => r.start >= cutoff);
+    return new MarkdownQuery(this.source, afterTokens);
+  }
+
+  each(): MarkdownQuery[] {
+    return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
+  }
+}
+
+export function mdq(source: string): MarkdownQuery {
+  return new MarkdownQuery(source);
+}

package/src/utils/markdown-terminal.ts

@@ -0,0 +1,43 @@
+import dedent from 'dedent';
+import { marked } from 'marked';
+import { markedTerminal } from 'marked-terminal';
+
+if (!process.env.FORCE_COLOR) {
+  process.env.FORCE_COLOR = '1';
+}
+
+marked.use(markedTerminal());
+marked.setOptions({ breaks: true });
+
+const renderer = (marked as any).defaults.renderer;
+if (renderer) {
+  if (renderer.text) {
+    const originalText = renderer.text;
+    renderer.text = function (text: any) {
+      if (typeof text === 'object') {
+        text = text.tokens ? this.parser.parseInline(text.tokens) : text.text;
+      }
+      return originalText.call(this, text);
+    };
+  }
+  if (renderer.listitem) {
+    const originalListitem = renderer.listitem;
+    renderer.listitem = function (text: any) {
+      if (typeof text === 'object') {
+        const item = text;
+        const parser = this.parser;
+        const originalParse = parser.parse.bind(parser);
+        parser.parse = (tokens: any, loose?: boolean) => parser.parseInline(tokens);
+        const result = originalListitem.call(this, text);
+        parser.parse = originalParse;
+        return result;
+      }
+      return originalListitem.call(this, text);
+    };
+  }
+}
+
+export function parseMarkdownToTerminal(markdown: string): string {
+  const result = marked.parse(dedent(markdown)) as string;
+  return result.replace(/^ {4}\*/gm, '*');
+}

package/src/utils/research-parser.ts

@@ -0,0 +1,11 @@
+export {
+  RESEARCH_COLUMN_ORDER,
+  type ResearchElement,
+  type ResearchSection,
+  extractContainerFromBlockquote,
+  extractValidContainers,
+  formatResearchSummary,
+  mapRowToElement,
+  parseResearchSections,
+  rebuildSectionMarkdown,
+} from '../ai/researcher/parser.ts';