explorbot 0.0.1 → 0.1.0

package/src/api/api-client.ts

@@ -0,0 +1,109 @@
+import { RequestResult, generateRequestId, resetRequestCounter } from './request-result.ts';
+
+export class ApiClient {
+  private baseEndpoint: string;
+  private defaultHeaders: Record<string, string>;
+
+  constructor(baseEndpoint: string, defaultHeaders: Record<string, string> = {}) {
+    this.baseEndpoint = baseEndpoint.replace(/\/$/, '');
+    this.defaultHeaders = defaultHeaders;
+  }
+
+  setHeaders(headers: Record<string, string>): void {
+    Object.assign(this.defaultHeaders, headers);
+  }
+
+  getHeaders(): Record<string, string> {
+    return { ...this.defaultHeaders };
+  }
+
+  getBaseEndpoint(): string {
+    return this.baseEndpoint;
+  }
+
+  async request(opts: {
+    method: string;
+    path: string;
+    headers?: Record<string, string>;
+    body?: any;
+    queryParams?: Record<string, string>;
+  }): Promise<RequestResult> {
+    const method = opts.method.toUpperCase();
+    const urlPath = opts.path.startsWith('/') ? opts.path : `/${opts.path}`;
+    let fullUrl = `${this.baseEndpoint}${urlPath}`;
+
+    if (opts.queryParams && Object.keys(opts.queryParams).length > 0) {
+      const params = new URLSearchParams(opts.queryParams);
+      fullUrl += `?${params.toString()}`;
+    }
+
+    const headers: Record<string, string> = {
+      ...this.defaultHeaders,
+      ...opts.headers,
+    };
+
+    const fetchOptions: RequestInit = { method, headers };
+
+    if (opts.body && !['GET', 'HEAD'].includes(method)) {
+      if (typeof opts.body === 'string') {
+        fetchOptions.body = opts.body;
+      } else {
+        fetchOptions.body = JSON.stringify(opts.body);
+        if (!headers['Content-Type'] && !headers['content-type']) {
+          headers['Content-Type'] = 'application/json';
+        }
+      }
+    }
+
+    const id = generateRequestId(method, urlPath);
+    const start = performance.now();
+
+    try {
+      const response = await fetch(fullUrl, fetchOptions);
+      const rawBody = await response.text();
+      const timing = Math.round(performance.now() - start);
+
+      const responseHeaders: Record<string, string> = {};
+      response.headers.forEach((value, key) => {
+        responseHeaders[key] = value;
+      });
+
+      const result = new RequestResult({
+        id,
+        method,
+        path: urlPath,
+        fullUrl,
+        requestHeaders: headers,
+        requestBody: opts.body,
+        status: response.status,
+        statusText: response.statusText,
+        responseHeaders,
+        timing,
+        timestamp: new Date(),
+      });
+      result.rawResponseBodyValue = rawBody;
+      return result;
+    } catch (err: any) {
+      const result = new RequestResult({
+        id,
+        method,
+        path: urlPath,
+        fullUrl,
+        requestHeaders: headers,
+        requestBody: opts.body,
+        status: 0,
+        statusText: 'Network Error',
+        responseHeaders: {},
+        timing: Math.round(performance.now() - start),
+        timestamp: new Date(),
+        error: err.message || String(err),
+      });
+      result.rawResponseBodyValue = '';
+      return result;
+    }
+  }
+
+  static resetCounter(): void {
+    resetRequestCounter();
+  }
+}

package/src/api/request-result.ts

@@ -0,0 +1,212 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { sanitizeFilename } from '../utils/strings.ts';
+
+let requestCounter = 0;
+
+export function generateRequestId(method: string, urlPath: string, prefix = ''): string {
+  requestCounter++;
+  const num = String(requestCounter).padStart(3, '0');
+  const sanitized = sanitizeFilename(urlPath.replace(/^\//, ''));
+  return `${prefix}${num}_${method}_${sanitized}`;
+}
+
+export function resetRequestCounter(): void {
+  requestCounter = 0;
+}
+
+export class RequestResult {
+  id: string;
+  method: string;
+  path: string;
+  fullUrl: string;
+  requestHeaders: Record<string, string>;
+  requestBody?: any;
+  status: number;
+  statusText: string;
+  responseHeaders: Record<string, string>;
+  timing: number;
+  timestamp: Date;
+  error?: string;
+
+  requestFile = '';
+  responseFile = '';
+
+  private _rawResponseBody?: string;
+
+  constructor(opts: {
+    id: string;
+    method: string;
+    path: string;
+    fullUrl: string;
+    requestHeaders: Record<string, string>;
+    requestBody?: any;
+    status: number;
+    statusText: string;
+    responseHeaders: Record<string, string>;
+    timing: number;
+    timestamp: Date;
+    error?: string;
+  }) {
+    this.id = opts.id;
+    this.method = opts.method;
+    this.path = opts.path;
+    this.fullUrl = opts.fullUrl;
+    this.requestHeaders = opts.requestHeaders;
+    this.requestBody = opts.requestBody;
+    this.status = opts.status;
+    this.statusText = opts.statusText;
+    this.responseHeaders = opts.responseHeaders;
+    this.timing = opts.timing;
+    this.timestamp = opts.timestamp;
+    this.error = opts.error;
+  }
+
+  get rawResponseBody(): string {
+    if (this._rawResponseBody !== undefined) return this._rawResponseBody;
+    if (this.responseFile && existsSync(this.responseFile)) {
+      this._rawResponseBody = readFileSync(this.responseFile, 'utf8');
+      return this._rawResponseBody;
+    }
+    return '';
+  }
+
+  set rawResponseBodyValue(value: string) {
+    this._rawResponseBody = value;
+  }
+
+  get responseBody(): any {
+    const raw = this.rawResponseBody;
+    if (!raw) return null;
+    try {
+      return JSON.parse(raw);
+    } catch {
+      return raw;
+    }
+  }
+
+  save(outputDir: string): void {
+    const requestsDir = path.join(outputDir, 'requests');
+    if (!existsSync(requestsDir)) {
+      mkdirSync(requestsDir, { recursive: true });
+    }
+
+    this.requestFile = path.join(requestsDir, `${this.id}.request.yaml`);
+    this.responseFile = path.join(requestsDir, `${this.id}.response.json`);
+
+    const headerLines = Object.entries(this.requestHeaders)
+      .map(([k, v]) => `  ${k}: ${v}`)
+      .join('\n');
+
+    const responseHeaderLines = Object.entries(this.responseHeaders)
+      .map(([k, v]) => `  ${k}: ${v}`)
+      .join('\n');
+
+    let yaml = '---\n';
+    yaml += `method: ${this.method}\n`;
+    yaml += `url: ${this.path}\n`;
+    yaml += `fullUrl: ${this.fullUrl}\n`;
+    yaml += `headers:\n${headerLines}\n`;
+    yaml += `status: ${this.status}\n`;
+    yaml += `statusText: ${this.statusText}\n`;
+    yaml += `responseHeaders:\n${responseHeaderLines}\n`;
+    yaml += `timing: ${this.timing}\n`;
+    yaml += `timestamp: ${this.timestamp.toISOString()}\n`;
+    yaml += '---\n';
+
+    if (this.requestBody) {
+      const body = typeof this.requestBody === 'string' ? this.requestBody : JSON.stringify(this.requestBody, null, 2);
+      yaml += body;
+    }
+
+    writeFileSync(this.requestFile, yaml, 'utf8');
+    writeFileSync(this.responseFile, this._rawResponseBody || '', 'utf8');
+  }
+
+  static load(requestFile: string): RequestResult {
+    const content = readFileSync(requestFile, 'utf8');
+    const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+    if (!frontmatterMatch) throw new Error(`Invalid request file: ${requestFile}`);
+
+    const meta: Record<string, any> = {};
+    const lines = frontmatterMatch[1].split('\n');
+    let currentKey = '';
+    let currentObj: Record<string, string> | null = null;
+
+    for (const line of lines) {
+      const indentedMatch = line.match(/^ {2}(\S+):\s*(.*)$/);
+      if (indentedMatch && currentObj) {
+        currentObj[indentedMatch[1]] = indentedMatch[2];
+        continue;
+      }
+
+      const kvMatch = line.match(/^(\S+):\s*(.*)$/);
+      if (!kvMatch) continue;
+
+      currentKey = kvMatch[1];
+      const value = kvMatch[2];
+
+      if (!value) {
+        currentObj = {};
+        meta[currentKey] = currentObj;
+      } else {
+        currentObj = null;
+        meta[currentKey] = value;
+      }
+    }
+
+    const id = path.basename(requestFile).replace('.request.yaml', '');
+    const responseFile = requestFile.replace('.request.yaml', '.response.json');
+
+    const result = new RequestResult({
+      id,
+      method: meta.method || 'GET',
+      path: meta.url || '',
+      fullUrl: meta.fullUrl || '',
+      requestHeaders: meta.headers || {},
+      requestBody: frontmatterMatch[2] || undefined,
+      status: Number.parseInt(meta.status) || 0,
+      statusText: meta.statusText || '',
+      responseHeaders: meta.responseHeaders || {},
+      timing: Number.parseInt(meta.timing) || 0,
+      timestamp: new Date(meta.timestamp || Date.now()),
+    });
+
+    result.requestFile = requestFile;
+    result.responseFile = responseFile;
+
+    return result;
+  }
+
+  toSummary(): string {
+    return `${this.method} ${this.path} → ${this.status} (${this.timing}ms)`;
+  }
+
+  extractIdAndTitle(): { id?: string | number; title?: string } {
+    const body = this.responseBody;
+    if (!body || typeof body !== 'object') return {};
+
+    const data = body.data || body;
+    if (Array.isArray(data)) return {};
+
+    const id = data.id ?? data._id ?? data.uuid;
+    const title = data.name ?? data.title ?? data.label;
+
+    const result: { id?: string | number; title?: string } = {};
+    if (id !== undefined) result.id = id;
+    if (title !== undefined) result.title = String(title);
+    return result;
+  }
+
+  toCurlCommand(): string {
+    let cmd = `curl -X ${this.method} '${this.fullUrl}'`;
+    for (const [k, v] of Object.entries(this.requestHeaders)) {
+      cmd += ` -H '${k}: ${v}'`;
+    }
+    if (this.requestBody) {
+      const body = typeof this.requestBody === 'string' ? this.requestBody : JSON.stringify(this.requestBody);
+      cmd += ` -d '${body}'`;
+    }
+    return cmd;
+  }
+}

package/src/api/request-store.ts

@@ -0,0 +1,130 @@
+import { existsSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import { RequestResult } from './request-result.ts';
+
+const AUTH_HEADERS = ['authorization', 'cookie', 'x-api-key', 'x-csrf-token'];
+
+export class RequestStore {
+  private capturedRequests: RequestResult[] = [];
+  private madeRequests: RequestResult[] = [];
+  private outputDir: string;
+
+  constructor(outputDir: string) {
+    this.outputDir = outputDir;
+  }
+
+  addCapturedRequest(result: RequestResult): void {
+    this.capturedRequests.push(result);
+    result.save(this.outputDir);
+  }
+
+  addMadeRequest(result: RequestResult): void {
+    this.madeRequests.push(result);
+    result.save(this.outputDir);
+  }
+
+  addRequest(result: RequestResult): void {
+    this.addMadeRequest(result);
+  }
+
+  getCapturedRequests(): RequestResult[] {
+    return this.capturedRequests;
+  }
+
+  getMadeRequests(): RequestResult[] {
+    return this.madeRequests;
+  }
+
+  getRequests(): RequestResult[] {
+    return this.madeRequests;
+  }
+
+  getLastRequest(): RequestResult | undefined {
+    return this.madeRequests[this.madeRequests.length - 1];
+  }
+
+  getRequestsByEndpoint(pathPrefix: string): RequestResult[] {
+    return this.madeRequests.filter((r) => r.path.startsWith(pathPrefix));
+  }
+
+  getRequestsByMethod(method: string): RequestResult[] {
+    const upper = method.toUpperCase();
+    return this.madeRequests.filter((r) => r.method === upper);
+  }
+
+  getRequestsByStatus(status: number): RequestResult[] {
+    return this.madeRequests.filter((r) => r.status === status);
+  }
+
+  toEndpointList(): string {
+    const seen = new Set<string>();
+    const lines: string[] = [];
+
+    for (const req of this.capturedRequests) {
+      const normalized = normalizePathPattern(req.path);
+      const key = `${req.method} ${normalized}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      lines.push(key);
+    }
+
+    return lines.join('\n');
+  }
+
+  extractAuthHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {};
+
+    for (let i = this.capturedRequests.length - 1; i >= 0; i--) {
+      const req = this.capturedRequests[i];
+      for (const [key, value] of Object.entries(req.requestHeaders)) {
+        if (AUTH_HEADERS.includes(key.toLowerCase()) && !headers[key]) {
+          headers[key] = value;
+        }
+      }
+      if (AUTH_HEADERS.every((h) => Object.keys(headers).some((k) => k.toLowerCase() === h))) break;
+    }
+
+    return headers;
+  }
+
+  findCapturedRequest(method: string, pathPrefix: string): RequestResult | undefined {
+    const upper = method.toUpperCase();
+    return this.capturedRequests.find((r) => r.method === upper && r.path.startsWith(pathPrefix));
+  }
+
+  toLog(): string {
+    return this.madeRequests.map((r, i) => `${i + 1}. ${r.toSummary()} [${r.id}]`).join('\n');
+  }
+
+  loadFromDisk(): void {
+    const requestsDir = path.join(this.outputDir, 'requests');
+    if (!existsSync(requestsDir)) return;
+
+    const existingIds = new Set(this.capturedRequests.map((r) => r.id));
+    const files = readdirSync(requestsDir).filter((f) => f.endsWith('.request.yaml'));
+
+    for (const file of files) {
+      try {
+        const result = RequestResult.load(path.join(requestsDir, file));
+        if (existingIds.has(result.id)) continue;
+        this.capturedRequests.push(result);
+      } catch {
+        // skip invalid files
+      }
+    }
+  }
+
+  getWriteRequestsForScope(scopePath: string): RequestResult[] {
+    const writeMethods = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
+    return this.capturedRequests.filter((r) => writeMethods.has(r.method) && r.path.startsWith(scopePath));
+  }
+
+  clear(): void {
+    this.capturedRequests = [];
+    this.madeRequests = [];
+  }
+}
+
+function normalizePathPattern(urlPath: string): string {
+  return urlPath.replace(/\/[0-9a-f]{24}\b/g, '/{id}').replace(/\/\d+\b/g, '/{id}');
+}

package/src/api/spec-reader.ts

@@ -0,0 +1,174 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { mkdirSync } from 'node:fs';
+import path from 'node:path';
+import { dereference } from '@scalar/openapi-parser';
+import { tag } from '../utils/logger.ts';
+
+export function validateSpecs(specs?: string[]): void {
+  if (!specs?.length) {
+    throw new Error('API spec is required. Set api.spec in your config file.');
+  }
+}
+
+export async function loadSpec(specPaths: string[], outputDir: string): Promise<any> {
+  const contents: string[] = [];
+
+  for (const specPath of specPaths) {
+    if (specPath.startsWith('http://') || specPath.startsWith('https://')) {
+      const response = await fetch(specPath);
+      if (!response.ok) throw new Error(`Failed to fetch spec from ${specPath}: ${response.status}`);
+      const content = await response.text();
+
+      const specDir = path.join(outputDir, 'spec');
+      mkdirSync(specDir, { recursive: true });
+      const filename = specPath.split('/').pop() || 'spec.yaml';
+      writeFileSync(path.join(specDir, filename), content, 'utf8');
+      tag('info').log(`Spec downloaded: ${filename}`);
+
+      contents.push(content);
+    } else {
+      contents.push(readFileSync(specPath, 'utf8'));
+    }
+  }
+
+  const combined = contents[0];
+  const result = dereference(combined);
+
+  if (result.errors?.length) {
+    tag('warning').log(`Spec parse warnings: ${result.errors.map((e) => e.message).join(', ')}`);
+  }
+
+  if (!result.schema) {
+    throw new Error('Failed to parse OpenAPI spec');
+  }
+
+  return result.schema;
+}
+
+export function extractEndpointDefinition(schema: any, endpoint: string, baseEndpoint?: string): string {
+  if (!schema?.paths) {
+    throw new Error('OpenAPI spec has no paths defined');
+  }
+
+  const basePath = toBasePath(baseEndpoint);
+  const matched = collectMatchingPaths(schema, basePath, (normalized) => matchesEndpoint(normalized, endpoint));
+
+  if (!Object.keys(matched).length) {
+    const available = listNormalizedPaths(schema, basePath);
+    throw new Error(`Endpoint "${endpoint}" not found in spec. Available: ${available}`);
+  }
+
+  return safeStringify(matched);
+}
+
+export function searchEndpoints(schema: any, query: string, baseEndpoint?: string): string {
+  if (!schema?.paths) return 'No endpoints available';
+
+  const basePath = toBasePath(baseEndpoint);
+
+  if (!query || query === '*') {
+    const lines = Object.keys(schema.paths).map((p) => {
+      const normalized = stripBasePath(p, basePath);
+      const methods = Object.keys(schema.paths[p])
+        .filter((m) => HTTP_METHODS.includes(m))
+        .map((m) => m.toUpperCase())
+        .join(',');
+      return `${methods} ${normalized}`;
+    });
+    return lines.join('\n');
+  }
+
+  const lowerQuery = query.toLowerCase();
+  const matched = collectMatchingPaths(schema, basePath, (normalized) => normalized.toLowerCase().includes(lowerQuery));
+
+  if (!Object.keys(matched).length) {
+    return `No endpoints matching "${query}". Available: ${listNormalizedPaths(schema, basePath)}`;
+  }
+
+  return safeStringify(matched);
+}
+
+const HTTP_METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'];
+
+export function listAllEndpoints(schema: any, baseEndpoint?: string): string {
+  if (!schema?.paths) return '';
+
+  const basePath = toBasePath(baseEndpoint);
+  const lines: string[] = [];
+
+  for (const specPath of Object.keys(schema.paths)) {
+    const normalized = stripBasePath(specPath, basePath);
+    const pathDef = schema.paths[specPath];
+
+    for (const method of HTTP_METHODS) {
+      if (!pathDef[method]) continue;
+      const summary = pathDef[method].summary || pathDef[method].description || '';
+      const desc = summary ? ` - ${summary}` : '';
+      lines.push(`${method.toUpperCase()} ${normalized}${desc}`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+function toBasePath(baseEndpoint?: string): string {
+  return baseEndpoint ? new URL(baseEndpoint).pathname.replace(/\/$/, '') : '';
+}
+
+function collectMatchingPaths(schema: any, basePath: string, predicate: (normalized: string) => boolean): Record<string, any> {
+  const matched: Record<string, any> = {};
+  for (const specPath of Object.keys(schema.paths)) {
+    const normalized = stripBasePath(specPath, basePath);
+    if (!predicate(normalized)) continue;
+    matched[specPath] = schema.paths[specPath];
+  }
+  return matched;
+}
+
+function listNormalizedPaths(schema: any, basePath: string): string {
+  return Object.keys(schema.paths)
+    .map((p) => stripBasePath(p, basePath))
+    .join(', ');
+}
+
+function safeStringify(obj: any): string {
+  const seen = new WeakSet();
+  return JSON.stringify(
+    obj,
+    (_key, value) => {
+      if (typeof value === 'object' && value !== null) {
+        if (seen.has(value)) return '[Circular]';
+        seen.add(value);
+      }
+      return value;
+    },
+    2
+  );
+}
+
+function stripBasePath(specPath: string, basePath: string): string {
+  if (!basePath) return specPath;
+
+  const specSegments = specPath.split('/').filter(Boolean);
+  const baseSegments = basePath.split('/').filter(Boolean);
+
+  let i = 0;
+  while (i < baseSegments.length && i < specSegments.length) {
+    const spec = specSegments[i];
+    const base = baseSegments[i];
+    if (spec === base || spec.startsWith('{')) {
+      i++;
+      continue;
+    }
+    break;
+  }
+
+  return `/${specSegments.slice(i).join('/')}`;
+}
+
+function matchesEndpoint(specPath: string, endpoint: string): boolean {
+  if (specPath === endpoint) return true;
+  if (specPath.startsWith(`${endpoint}/`)) return true;
+  if (specPath.startsWith(`${endpoint}/{`)) return true;
+  return false;
+}