openclaw-crawleo-skill 0.1.0

package/src/client.js

@@ -0,0 +1,155 @@
+import { createEndpointMethods } from './endpoints.js';
+import { CRAWLEO_BASE_URL, getEndpointByPath } from './contract.js';
+import { CrawleoError, CRAWLEO_ERROR_CODES, errorCodeForStatus, redactSecret } from './errors.js';
+
+export function createCrawleoClient(options = {}) {
+  const apiKey = options.apiKey ?? process.env.CRAWLEO_API_KEY;
+  const fetchImpl = options.fetch ?? globalThis.fetch;
+  const baseUrl = normalizeBaseUrl(options.baseUrl ?? CRAWLEO_BASE_URL);
+
+  const client = {
+    baseUrl,
+    request(endpointPath, params = {}) {
+      return requestCrawleo({ apiKey, fetchImpl, baseUrl, endpointPath, params });
+    }
+  };
+
+  return Object.freeze({
+    ...client,
+    ...createEndpointMethods(client)
+  });
+}
+
+export async function requestCrawleo({ apiKey, fetchImpl, baseUrl = CRAWLEO_BASE_URL, endpointPath, params = {} }) {
+  const endpoint = getEndpointByPath(endpointPath);
+  if (!endpoint) {
+    throw new CrawleoError(`Unknown Crawleo endpoint: ${endpointPath}`, {
+      code: CRAWLEO_ERROR_CODES.VALIDATION,
+      endpoint: endpointPath,
+      field: 'endpointPath'
+    });
+  }
+
+  if (!apiKey) {
+    throw new CrawleoError('CRAWLEO_API_KEY is required for live Crawleo REST calls.', {
+      code: CRAWLEO_ERROR_CODES.MISSING_API_KEY,
+      endpoint: endpointPath
+    });
+  }
+
+  if (typeof fetchImpl !== 'function') {
+    throw new CrawleoError('A fetch implementation is required for Crawleo REST calls.', {
+      code: CRAWLEO_ERROR_CODES.MISSING_FETCH,
+      endpoint: endpointPath,
+      secrets: [apiKey]
+    });
+  }
+
+  const url = buildCrawleoUrl(baseUrl, endpoint.path, params);
+  let response;
+
+  try {
+    response = await fetchImpl(url, {
+      method: endpoint.method,
+      headers: {
+        'x-api-key': apiKey,
+        accept: 'application/json'
+      }
+    });
+  } catch (error) {
+    throw new CrawleoError('Crawleo request failed before receiving a response.', {
+      code: CRAWLEO_ERROR_CODES.TRANSPORT,
+      endpoint: endpoint.path,
+      details: { cause: error instanceof Error ? error.message : String(error) },
+      secrets: [apiKey]
+    });
+  }
+
+  return parseCrawleoResponse(response, { endpointPath: endpoint.path, apiKey });
+}
+
+export function buildCrawleoUrl(baseUrl, endpointPath, params = {}) {
+  const url = new URL(endpointPath, `${normalizeBaseUrl(baseUrl)}/`);
+
+  for (const [key, value] of Object.entries(params)) {
+    if (value === undefined || value === null) continue;
+    const encodedValue = Array.isArray(value) ? value.join(',') : String(value);
+    url.searchParams.set(key, encodedValue);
+  }
+
+  return url;
+}
+
+async function parseCrawleoResponse(response, { endpointPath, apiKey }) {
+  const status = response.status;
+  const ok = response.ok ?? (status >= 200 && status < 300);
+  const rawBody = await readResponseBody(response);
+  const contentType = typeof response.headers?.get === 'function' ? response.headers.get('content-type') || '' : '';
+  const expectsJson = contentType.includes('application/json') || rawBody.trim().startsWith('{') || rawBody.trim().startsWith('[');
+
+  let parsedBody = null;
+  let malformedJson = false;
+
+  if (rawBody && expectsJson) {
+    try {
+      parsedBody = JSON.parse(rawBody);
+    } catch {
+      malformedJson = true;
+    }
+  }
+
+  if (!ok) {
+    throw new CrawleoError(`Crawleo request failed with HTTP ${status}.`, {
+      code: errorCodeForStatus(status),
+      endpoint: endpointPath,
+      status,
+      details: summarizeResponseBody(parsedBody, rawBody, contentType, apiKey),
+      secrets: [apiKey]
+    });
+  }
+
+  if (malformedJson) {
+    throw new CrawleoError('Crawleo returned malformed JSON.', {
+      code: CRAWLEO_ERROR_CODES.RESPONSE_MALFORMED_JSON,
+      endpoint: endpointPath,
+      status,
+      details: {
+        contentType,
+        bodyPreview: redactSecret(rawBody.slice(0, 500), [apiKey])
+      },
+      secrets: [apiKey]
+    });
+  }
+
+  return parsedBody;
+}
+
+async function readResponseBody(response) {
+  if (typeof response.text === 'function') return response.text();
+  if (typeof response.json === 'function') return JSON.stringify(await response.json());
+  return '';
+}
+
+function summarizeResponseBody(body, rawBody, contentType, apiKey) {
+  if (body == null) {
+    if (!rawBody) return null;
+    return {
+      contentType,
+      bodyPreview: redactSecret(rawBody.slice(0, 500), [apiKey])
+    };
+  }
+  if (typeof body !== 'object') return { body };
+
+  const summary = {};
+  for (const key of ['error', 'message', 'code', 'details']) {
+    if (Object.prototype.hasOwnProperty.call(body, key)) {
+      summary[key] = body[key];
+    }
+  }
+
+  return Object.keys(summary).length > 0 ? summary : { body };
+}
+
+function normalizeBaseUrl(baseUrl) {
+  return String(baseUrl).replace(/\/+$/, '');
+}

package/src/contract.js

@@ -0,0 +1,50 @@
+export const CRAWLEO_BASE_URL = 'https://api.crawleo.dev';
+
+export const CRAWLEO_ENDPOINTS = Object.freeze({
+  search: Object.freeze({
+    id: 'search',
+    path: '/search',
+    method: 'GET',
+    requiredQuery: ['query'],
+    enumQuery: Object.freeze({ device: ['desktop', 'mobile', 'tablet'] })
+  }),
+  googleSearch: Object.freeze({
+    id: 'google_search',
+    path: '/google-search',
+    method: 'GET',
+    requiredQuery: ['q'],
+    enumQuery: Object.freeze({
+      tbs: ['qdr:h', 'qdr:d', 'qdr:w', 'qdr:m', 'qdr:y'],
+      type: ['search', 'news', 'images', 'places', 'shopping']
+    })
+  }),
+  googleMaps: Object.freeze({
+    id: 'google_maps',
+    path: '/google-maps',
+    method: 'GET',
+    requiredQuery: ['q'],
+    enumQuery: Object.freeze({})
+  }),
+  crawl: Object.freeze({
+    id: 'crawl',
+    path: '/crawl',
+    method: 'GET',
+    requiredQuery: ['urls'],
+    enumQuery: Object.freeze({})
+  }),
+  headfulBrowser: Object.freeze({
+    id: 'headful_browser',
+    path: '/headful-browser',
+    method: 'GET',
+    requiredQuery: ['urls'],
+    enumQuery: Object.freeze({ output_format: ['markdown', 'enhanced_html', 'raw_html', 'page_text'] })
+  })
+});
+
+export const CRAWLEO_ENDPOINTS_BY_PATH = Object.freeze(
+  Object.fromEntries(Object.values(CRAWLEO_ENDPOINTS).map((endpoint) => [endpoint.path, endpoint]))
+);
+
+export function getEndpointByPath(path) {
+  return CRAWLEO_ENDPOINTS_BY_PATH[path];
+}

package/src/endpoints.js

@@ -0,0 +1,78 @@
+import { CRAWLEO_ENDPOINTS } from './contract.js';
+import { CrawleoError, CRAWLEO_ERROR_CODES } from './errors.js';
+
+export function search(client, params = {}) {
+  return callEndpoint(client, CRAWLEO_ENDPOINTS.search, params);
+}
+
+export function googleSearch(client, params = {}) {
+  return callEndpoint(client, CRAWLEO_ENDPOINTS.googleSearch, params);
+}
+
+export function googleMaps(client, params = {}) {
+  return callEndpoint(client, CRAWLEO_ENDPOINTS.googleMaps, params);
+}
+
+export function crawl(client, params = {}) {
+  return callEndpoint(client, CRAWLEO_ENDPOINTS.crawl, normalizeUrlListParams(params));
+}
+
+export function headfulBrowser(client, params = {}) {
+  return callEndpoint(client, CRAWLEO_ENDPOINTS.headfulBrowser, normalizeUrlListParams(params));
+}
+
+export function validateEndpointParams(endpoint, params = {}) {
+  for (const field of endpoint.requiredQuery) {
+    const value = params[field];
+    if (value === undefined || value === null || value === '' || (Array.isArray(value) && value.length === 0)) {
+      throw new CrawleoError(`Missing required Crawleo parameter: ${field}`, {
+        code: CRAWLEO_ERROR_CODES.VALIDATION,
+        endpoint: endpoint.path,
+        field
+      });
+    }
+  }
+
+  for (const [field, allowedValues] of Object.entries(endpoint.enumQuery)) {
+    const value = params[field];
+    if (value !== undefined && value !== null && !allowedValues.includes(value)) {
+      throw new CrawleoError(`Invalid Crawleo parameter ${field}: expected one of ${allowedValues.join(', ')}`, {
+        code: CRAWLEO_ERROR_CODES.VALIDATION,
+        endpoint: endpoint.path,
+        field,
+        details: { allowedValues, received: value }
+      });
+    }
+  }
+}
+
+export function createEndpointMethods(client) {
+  return Object.freeze({
+    search: (params) => search(client, params),
+    googleSearch: (params) => googleSearch(client, params),
+    googleMaps: (params) => googleMaps(client, params),
+    crawl: (params) => crawl(client, params),
+    headfulBrowser: (params) => headfulBrowser(client, params)
+  });
+}
+
+function callEndpoint(client, endpoint, params) {
+  if (!client || typeof client.request !== 'function') {
+    throw new CrawleoError('A Crawleo client with a request method is required.', {
+      code: CRAWLEO_ERROR_CODES.VALIDATION,
+      endpoint: endpoint.path,
+      field: 'client'
+    });
+  }
+
+  validateEndpointParams(endpoint, params);
+  return client.request(endpoint.path, params);
+}
+
+function normalizeUrlListParams(params) {
+  if (!Object.prototype.hasOwnProperty.call(params, 'urls')) return params;
+  return {
+    ...params,
+    urls: Array.isArray(params.urls) ? params.urls.join(',') : params.urls
+  };
+}

package/src/errors.js

@@ -0,0 +1,89 @@
+export const CRAWLEO_ERROR_CODES = Object.freeze({
+  MISSING_API_KEY: 'CRAWLEO_CONFIG_MISSING_API_KEY',
+  MISSING_FETCH: 'CRAWLEO_CONFIG_MISSING_FETCH',
+  VALIDATION: 'CRAWLEO_VALIDATION_ERROR',
+  HTTP_BAD_REQUEST: 'CRAWLEO_HTTP_BAD_REQUEST',
+  HTTP_AUTH: 'CRAWLEO_HTTP_AUTH',
+  HTTP_PAYMENT_REQUIRED: 'CRAWLEO_HTTP_PAYMENT_REQUIRED',
+  HTTP_FORBIDDEN: 'CRAWLEO_HTTP_FORBIDDEN',
+  HTTP_RATE_LIMIT: 'CRAWLEO_HTTP_RATE_LIMIT',
+  HTTP_UPSTREAM: 'CRAWLEO_HTTP_UPSTREAM',
+  RESPONSE_MALFORMED_JSON: 'CRAWLEO_RESPONSE_MALFORMED_JSON',
+  TRANSPORT: 'CRAWLEO_TRANSPORT_ERROR'
+});
+
+const SECRET_PATTERNS = [
+  /x-api-key\s*[:=]\s*[^\s,}]+/gi,
+  /authorization\s*[:=]\s*bearer\s+[^\s,}]+/gi,
+  /api[_-]?key\s*[:=]\s*[^\s,}]+/gi
+];
+
+export function redactSecret(value, explicitSecrets = []) {
+  if (value == null) return value;
+
+  let redacted = String(value);
+  for (const secret of explicitSecrets) {
+    if (secret) {
+      redacted = redacted.split(String(secret)).join('[REDACTED]');
+    }
+  }
+
+  for (const pattern of SECRET_PATTERNS) {
+    redacted = redacted.replace(pattern, (match) => {
+      const separator = match.includes('=') ? '=' : ':';
+      const [name] = match.split(separator);
+      return `${name}${separator} [REDACTED]`;
+    });
+  }
+
+  return redacted;
+}
+
+export class CrawleoError extends Error {
+  constructor(message, options = {}) {
+    super(redactSecret(message, options.secrets));
+    this.name = 'CrawleoError';
+    this.code = options.code || CRAWLEO_ERROR_CODES.HTTP_UPSTREAM;
+    this.endpoint = options.endpoint;
+    this.status = options.status;
+    this.field = options.field;
+    this.details = redactDetails(options.details, options.secrets);
+
+    if (options.cause) {
+      this.cause = options.cause;
+    }
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+      code: this.code,
+      message: this.message,
+      endpoint: this.endpoint,
+      status: this.status,
+      field: this.field,
+      details: this.details
+    };
+  }
+}
+
+function redactDetails(details, secrets = []) {
+  if (details == null) return details;
+  if (typeof details === 'string') return redactSecret(details, secrets);
+  if (Array.isArray(details)) return details.map((item) => redactDetails(item, secrets));
+  if (typeof details === 'object') {
+    return Object.fromEntries(
+      Object.entries(details).map(([key, value]) => [key, redactDetails(value, secrets)])
+    );
+  }
+  return details;
+}
+
+export function errorCodeForStatus(status) {
+  if (status === 400) return CRAWLEO_ERROR_CODES.HTTP_BAD_REQUEST;
+  if (status === 401) return CRAWLEO_ERROR_CODES.HTTP_AUTH;
+  if (status === 402) return CRAWLEO_ERROR_CODES.HTTP_PAYMENT_REQUIRED;
+  if (status === 403) return CRAWLEO_ERROR_CODES.HTTP_FORBIDDEN;
+  if (status === 429) return CRAWLEO_ERROR_CODES.HTTP_RATE_LIMIT;
+  return CRAWLEO_ERROR_CODES.HTTP_UPSTREAM;
+}

package/src/index.js

@@ -0,0 +1,37 @@
+export { createCrawleoClient, requestCrawleo, buildCrawleoUrl } from './client.js';
+export { search, googleSearch, googleMaps, crawl, headfulBrowser, validateEndpointParams } from './endpoints.js';
+export { CRAWLEO_BASE_URL, CRAWLEO_ENDPOINTS, CRAWLEO_ENDPOINTS_BY_PATH, getEndpointByPath } from './contract.js';
+export { CrawleoError, CRAWLEO_ERROR_CODES, redactSecret } from './errors.js';
+
+export const CRAWLEO_CONTRACT_PATH = 'contracts/crawleo-endpoints.json';
+export const CRAWLEO_SKILL_MANIFEST_PATH = 'skill.json';
+export const CRAWLEO_SKILL_INSTRUCTIONS_PATH = 'SKILL.md';
+
+export const CRAWLEO_REST_ENDPOINTS = Object.freeze([
+  '/search',
+  '/google-search',
+  '/google-maps',
+  '/crawl',
+  '/headful-browser'
+]);
+
+export const CRAWLEO_MCP_TOOLS = Object.freeze([
+  'search_web',
+  'google_search',
+  'google_maps',
+  'crawl_web',
+  'headful_browser'
+]);
+
+export function getScaffoldStatus() {
+  return Object.freeze({
+    packageName: 'openclaw-crawleo-skill',
+    implementationStatus: 'rest-wrappers-implemented',
+    contractPath: CRAWLEO_CONTRACT_PATH,
+    skillManifestPath: CRAWLEO_SKILL_MANIFEST_PATH,
+    skillInstructionsPath: CRAWLEO_SKILL_INSTRUCTIONS_PATH,
+    restEndpoints: CRAWLEO_REST_ENDPOINTS,
+    mcpTools: CRAWLEO_MCP_TOOLS,
+    liveCrawleoCallsEnabledByDefault: false
+  });
+}

package/test/client.test.js

@@ -0,0 +1,104 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { CrawleoError, CRAWLEO_ERROR_CODES, buildCrawleoUrl, createCrawleoClient } from '../src/index.js';
+
+function jsonResponse(body, init = {}) {
+  return {
+    ok: init.ok ?? true,
+    status: init.status ?? 200,
+    headers: new Map([['content-type', 'application/json']]),
+    async text() {
+      return JSON.stringify(body);
+    }
+  };
+}
+
+test('buildCrawleoUrl constructs Crawleo URLs and comma-encodes array values', () => {
+  const url = buildCrawleoUrl('https://api.crawleo.dev/', '/crawl', {
+    urls: ['https://example.com/a', 'https://example.com/b'],
+    markdown: true,
+    empty: null
+  });
+
+  assert.equal(url.toString(), 'https://api.crawleo.dev/crawl?urls=https%3A%2F%2Fexample.com%2Fa%2Chttps%3A%2F%2Fexample.com%2Fb&markdown=true');
+});
+
+test('client request injects x-api-key and returns parsed JSON through an injected fetch', async () => {
+  const calls = [];
+  const client = createCrawleoClient({
+    apiKey: 'secret-test-key',
+    fetch: async (url, init) => {
+      calls.push({ url, init });
+      return jsonResponse({ ok: true, query: 'ai agents' });
+    }
+  });
+
+  const result = await client.request('/search', { query: 'ai agents', max_pages: 1 });
+
+  assert.deepEqual(result, { ok: true, query: 'ai agents' });
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].url.toString(), 'https://api.crawleo.dev/search?query=ai+agents&max_pages=1');
+  assert.equal(calls[0].init.method, 'GET');
+  assert.equal(calls[0].init.headers['x-api-key'], 'secret-test-key');
+  assert.equal(calls[0].init.headers.accept, 'application/json');
+});
+
+test('client request reports missing API key as a structured CrawleoError', async () => {
+  const client = createCrawleoClient({ apiKey: '', fetch: async () => jsonResponse({}) });
+
+  await assert.rejects(
+    () => client.request('/search', { query: 'ai agents' }),
+    (error) => {
+      assert.ok(error instanceof CrawleoError);
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.MISSING_API_KEY);
+      assert.equal(error.endpoint, '/search');
+      assert.deepEqual(error.toJSON(), {
+        name: 'CrawleoError',
+        code: CRAWLEO_ERROR_CODES.MISSING_API_KEY,
+        message: 'CRAWLEO_API_KEY is required for live Crawleo REST calls.',
+        endpoint: '/search',
+        status: undefined,
+        field: undefined,
+        details: undefined
+      });
+      return true;
+    }
+  );
+});
+
+test('transport failures are redacted before serialization', async () => {
+  const apiKey = 'secret-transport-key';
+  const client = createCrawleoClient({
+    apiKey,
+    fetch: async () => {
+      throw new Error(`network failed with apiKey=${apiKey}`);
+    }
+  });
+
+  await assert.rejects(
+    () => client.request('/search', { query: 'ai agents' }),
+    (error) => {
+      const serialized = JSON.stringify(error);
+      assert.ok(error instanceof CrawleoError);
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.TRANSPORT);
+      assert.equal(error.endpoint, '/search');
+      assert.equal(serialized.includes(apiKey), false);
+      assert.match(serialized, /\[REDACTED\]/);
+      return true;
+    }
+  );
+});
+
+test('unknown endpoint paths are validation errors', async () => {
+  const client = createCrawleoClient({ apiKey: 'secret-test-key', fetch: async () => jsonResponse({}) });
+
+  await assert.rejects(
+    () => client.request('/not-crawleo', {}),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.VALIDATION);
+      assert.equal(error.field, 'endpointPath');
+      return true;
+    }
+  );
+});

package/test/endpoints.test.js

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import {
+  CRAWLEO_ERROR_CODES,
+  crawl,
+  createCrawleoClient,
+  googleMaps,
+  googleSearch,
+  headfulBrowser,
+  search
+} from '../src/index.js';
+
+function makeClient() {
+  const calls = [];
+  const client = createCrawleoClient({
+    apiKey: 'secret-wrapper-key',
+    fetch: async (url, init) => {
+      calls.push({ url, init });
+      return {
+        ok: true,
+        status: 200,
+        headers: new Map([['content-type', 'application/json']]),
+        async text() {
+          return JSON.stringify({ ok: true, path: new URL(url).pathname });
+        }
+      };
+    }
+  });
+  return { client, calls };
+}
+
+test('client exposes methods for every documented Crawleo endpoint', async () => {
+  const { client, calls } = makeClient();
+
+  await client.search({ query: 'ai agents', device: 'desktop' });
+  await client.googleSearch({ q: 'best CRM', type: 'news', tbs: 'qdr:d' });
+  await client.googleMaps({ q: 'restaurants in Paris', hl: 'fr' });
+  await client.crawl({ urls: ['https://example.com/a', 'https://example.com/b'], markdown: true });
+  await client.headfulBrowser({ urls: 'https://example.com', output_format: 'markdown' });
+
+  assert.deepEqual(
+    calls.map((call) => new URL(call.url).pathname),
+    ['/search', '/google-search', '/google-maps', '/crawl', '/headful-browser']
+  );
+  assert.equal(new URL(calls[3].url).searchParams.get('urls'), 'https://example.com/a,https://example.com/b');
+});
+
+test('top-level endpoint wrappers accept a client object without accepting secrets', async () => {
+  const { client, calls } = makeClient();
+
+  await search(client, { query: 'ai agents' });
+  await googleSearch(client, { q: 'ai agents' });
+  await googleMaps(client, { q: 'coffee near Paris' });
+  await crawl(client, { urls: 'https://example.com' });
+  await headfulBrowser(client, { urls: ['https://example.com'], output_format: 'page_text' });
+
+  assert.deepEqual(
+    calls.map((call) => new URL(call.url).pathname),
+    ['/search', '/google-search', '/google-maps', '/crawl', '/headful-browser']
+  );
+});
+
+test('wrappers reject missing required parameters with endpoint and field diagnostics', () => {
+  const { client } = makeClient();
+
+  assert.throws(
+    () => client.googleSearch({ type: 'news' }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.VALIDATION);
+      assert.equal(error.endpoint, '/google-search');
+      assert.equal(error.field, 'q');
+      return true;
+    }
+  );
+});
+
+test('wrappers validate only documented enum parameters', () => {
+  const { client } = makeClient();
+
+  assert.throws(
+    () => client.search({ query: 'ai agents', device: 'watch' }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.VALIDATION);
+      assert.equal(error.endpoint, '/search');
+      assert.equal(error.field, 'device');
+      assert.deepEqual(error.details.allowedValues, ['desktop', 'mobile', 'tablet']);
+      return true;
+    }
+  );
+
+  assert.throws(
+    () => client.googleSearch({ q: 'ai agents', type: 'videos' }),
+    (error) => {
+      assert.equal(error.endpoint, '/google-search');
+      assert.equal(error.field, 'type');
+      return true;
+    }
+  );
+
+  assert.throws(
+    () => client.googleSearch({ q: 'ai agents', tbs: 'qdr:decade' }),
+    (error) => {
+      assert.equal(error.endpoint, '/google-search');
+      assert.equal(error.field, 'tbs');
+      return true;
+    }
+  );
+
+  assert.throws(
+    () => client.headfulBrowser({ urls: 'https://example.com', output_format: 'pdf' }),
+    (error) => {
+      assert.equal(error.endpoint, '/headful-browser');
+      assert.equal(error.field, 'output_format');
+      return true;
+    }
+  );
+});
+
+test('wrappers reject an invalid client object before making a request', () => {
+  assert.throws(
+    () => search({}, { query: 'ai agents' }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.VALIDATION);
+      assert.equal(error.endpoint, '/search');
+      assert.equal(error.field, 'client');
+      return true;
+    }
+  );
+});