openclaw-crawleo-skill 0.1.0

package/test/error-fixtures.test.js

@@ -0,0 +1,151 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import {
+  CRAWLEO_ERROR_CODES,
+  CrawleoError,
+  createCrawleoClient,
+  redactSecret,
+  requestCrawleo
+} from '../src/index.js';
+
+const FIXTURE_SECRET = 'fixture-secret-key-never-real';
+
+function response({ status, body, contentType = 'application/json' }) {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    headers: new Map([['content-type', contentType]]),
+    async text() {
+      return typeof body === 'string' ? body : JSON.stringify(body);
+    }
+  };
+}
+
+async function getErrorFromClient(fetchImpl, apiKey = FIXTURE_SECRET) {
+  const client = createCrawleoClient({ apiKey, fetch: fetchImpl });
+
+  try {
+    await client.search({ query: 'fixture query' });
+  } catch (error) {
+    return error;
+  }
+
+  throw new Error('Expected Crawleo client call to fail');
+}
+
+test('HTTP error fixture table maps status codes to stable Crawleo error codes', async () => {
+  const fixtures = [
+    [400, CRAWLEO_ERROR_CODES.HTTP_BAD_REQUEST, 'Bad request fixture'],
+    [401, CRAWLEO_ERROR_CODES.HTTP_AUTH, 'Invalid API key fixture'],
+    [402, CRAWLEO_ERROR_CODES.HTTP_PAYMENT_REQUIRED, 'Insufficient credits fixture'],
+    [403, CRAWLEO_ERROR_CODES.HTTP_FORBIDDEN, 'Inactive subscription fixture'],
+    [429, CRAWLEO_ERROR_CODES.HTTP_RATE_LIMIT, 'Rate limit fixture'],
+    [500, CRAWLEO_ERROR_CODES.HTTP_UPSTREAM, 'Internal server fixture']
+  ];
+
+  for (const [status, expectedCode, message] of fixtures) {
+    const error = await getErrorFromClient(async () => response({
+      status,
+      body: {
+        error: message,
+        code: `fixture_${status}`,
+        details: { endpoint: '/search', status }
+      }
+    }));
+
+    assert.ok(error instanceof CrawleoError);
+    assert.equal(error.code, expectedCode);
+    assert.equal(error.endpoint, '/search');
+    assert.equal(error.status, status);
+    assert.equal(error.details.error, message);
+    assert.equal(error.details.code, `fixture_${status}`);
+  }
+});
+
+test('missing API key and missing fetch produce distinct configuration diagnostics', async () => {
+  await assert.rejects(
+    () => requestCrawleo({ apiKey: '', fetchImpl: async () => response({ status: 200, body: {} }), endpointPath: '/search', params: { query: 'fixture' } }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.MISSING_API_KEY);
+      assert.equal(error.endpoint, '/search');
+      assert.equal(error.status, undefined);
+      return true;
+    }
+  );
+
+  await assert.rejects(
+    () => requestCrawleo({ apiKey: FIXTURE_SECRET, fetchImpl: undefined, endpointPath: '/search', params: { query: 'fixture' } }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.MISSING_FETCH);
+      assert.equal(error.endpoint, '/search');
+      assert.equal(JSON.stringify(error).includes(FIXTURE_SECRET), false);
+      return true;
+    }
+  );
+});
+
+test('malformed successful JSON response carries redacted bounded body preview', async () => {
+  const error = await getErrorFromClient(async () => response({
+    status: 200,
+    body: `{ "message": "token ${FIXTURE_SECRET}"`,
+    contentType: 'application/json'
+  }));
+
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.RESPONSE_MALFORMED_JSON);
+  assert.equal(error.status, 200);
+  assert.equal(error.details.contentType, 'application/json');
+  assert.equal(error.details.bodyPreview.includes(FIXTURE_SECRET), false);
+  assert.match(error.details.bodyPreview, /\[REDACTED\]/);
+});
+
+test('text HTTP error bodies are summarized and redacted without becoming malformed JSON errors', async () => {
+  const error = await getErrorFromClient(async () => response({
+    status: 500,
+    body: `upstream failure x-api-key: ${FIXTURE_SECRET}`,
+    contentType: 'text/plain'
+  }));
+
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.HTTP_UPSTREAM);
+  assert.equal(error.details.contentType, 'text/plain');
+  assert.equal(error.details.bodyPreview.includes(FIXTURE_SECRET), false);
+  assert.match(error.details.bodyPreview, /x-api-key: \[REDACTED\]/);
+});
+
+test('transport error fixture preserves transport classification and redacts thrown messages', async () => {
+  const error = await getErrorFromClient(async () => {
+    throw new Error(`socket closed with Authorization: Bearer ${FIXTURE_SECRET}`);
+  });
+
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.TRANSPORT);
+  assert.equal(error.status, undefined);
+  assert.equal(error.details.cause.includes(FIXTURE_SECRET), false);
+  assert.match(error.details.cause, /Authorization: \[REDACTED\]/i);
+});
+
+test('CrawleoError serialization redacts nested details and explicit secret values', () => {
+  const error = new CrawleoError(`top-level message ${FIXTURE_SECRET}`, {
+    code: CRAWLEO_ERROR_CODES.HTTP_AUTH,
+    endpoint: '/search',
+    status: 401,
+    secrets: [FIXTURE_SECRET],
+    details: {
+      header: `x-api-key=${FIXTURE_SECRET}`,
+      nested: [{ authorization: `Authorization: Bearer ${FIXTURE_SECRET}` }]
+    }
+  });
+
+  const serialized = JSON.stringify(error);
+
+  assert.equal(serialized.includes(FIXTURE_SECRET), false);
+  assert.match(error.message, /\[REDACTED\]/);
+  assert.match(error.details.header, /x-api-key= \[REDACTED\]/);
+  assert.match(error.details.nested[0].authorization, /Authorization: \[REDACTED\]/i);
+});
+
+test('redactSecret handles common API key spellings without explicit secret input', () => {
+  assert.equal(redactSecret('apiKey=abc123'), 'apiKey= [REDACTED]');
+  assert.equal(redactSecret('api-key: abc123'), 'api-key: [REDACTED]');
+  assert.equal(redactSecret('x-api-key: abc123'), 'x-api-key: [REDACTED]');
+  assert.equal(redactSecret('Authorization: Bearer abc123'), 'Authorization: [REDACTED]');
+});

package/test/errors.test.js

@@ -0,0 +1,116 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { CRAWLEO_ERROR_CODES, CrawleoError, createCrawleoClient } from '../src/index.js';
+
+function response({ status, body, contentType = 'application/json' }) {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    headers: new Map([['content-type', contentType]]),
+    async text() {
+      return typeof body === 'string' ? body : JSON.stringify(body);
+    }
+  };
+}
+
+async function captureError(status, body, contentType, apiKey = `secret-key-${status}`) {
+  const client = createCrawleoClient({
+    apiKey,
+    fetch: async () => response({ status, body, contentType })
+  });
+
+  try {
+    await client.search({ query: 'ai agents' });
+  } catch (error) {
+    return { error, apiKey };
+  }
+
+  throw new Error('Expected request to fail');
+}
+
+test('HTTP 400 bad requests map to a stable Crawleo error code', async () => {
+  const { error } = await captureError(400, { error: 'Missing required parameter q' });
+
+  assert.ok(error instanceof CrawleoError);
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.HTTP_BAD_REQUEST);
+  assert.equal(error.endpoint, '/search');
+  assert.equal(error.status, 400);
+  assert.deepEqual(error.details, { error: 'Missing required parameter q' });
+});
+
+test('HTTP 401 auth failures are normalized without leaking the API key', async () => {
+  const apiKey = 'secret-key-401';
+  const { error } = await captureError(401, {
+    error: `Invalid x-api-key: ${apiKey}`,
+    code: 'invalid_key'
+  }, undefined, apiKey);
+
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.HTTP_AUTH);
+  assert.equal(JSON.stringify(error).includes(apiKey), false);
+  assert.match(JSON.stringify(error), /\[REDACTED\]/);
+});
+
+test('HTTP 402 quota failures and HTTP 429 rate limits get distinct codes', async () => {
+  const quota = await captureError(402, { error: 'Insufficient credits' });
+  const rateLimit = await captureError(429, { error: 'Rate limit exceeded' });
+
+  assert.equal(quota.error.code, CRAWLEO_ERROR_CODES.HTTP_PAYMENT_REQUIRED);
+  assert.equal(rateLimit.error.code, CRAWLEO_ERROR_CODES.HTTP_RATE_LIMIT);
+});
+
+test('HTTP 403 forbidden and 5xx upstream failures get stable codes', async () => {
+  const forbidden = await captureError(403, { error: 'Inactive account or expired subscription' });
+  const upstream = await captureError(500, { error: 'Internal server error' });
+
+  assert.equal(forbidden.error.code, CRAWLEO_ERROR_CODES.HTTP_FORBIDDEN);
+  assert.equal(upstream.error.code, CRAWLEO_ERROR_CODES.HTTP_UPSTREAM);
+});
+
+test('non-JSON HTTP errors preserve a bounded redacted body preview', async () => {
+  const apiKey = 'secret-key-500-text';
+  const { error } = await captureError(500, `server failed with api_key=${apiKey}`, 'text/plain', apiKey);
+
+  assert.equal(error.code, CRAWLEO_ERROR_CODES.HTTP_UPSTREAM);
+  assert.equal(error.details.contentType, 'text/plain');
+  assert.equal(error.details.bodyPreview.includes(apiKey), false);
+  assert.match(error.details.bodyPreview, /\[REDACTED\]/);
+});
+
+test('successful malformed JSON responses are response diagnostics, not HTTP errors', async () => {
+  const apiKey = 'secret-malformed-key';
+  const client = createCrawleoClient({
+    apiKey,
+    fetch: async () => response({ status: 200, body: `{ "token": "${apiKey}"`, contentType: 'application/json' })
+  });
+
+  await assert.rejects(
+    () => client.search({ query: 'ai agents' }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.RESPONSE_MALFORMED_JSON);
+      assert.equal(error.status, 200);
+      assert.equal(JSON.stringify(error).includes(apiKey), false);
+      return true;
+    }
+  );
+});
+
+test('transport failures remain distinct from HTTP failures and are redacted', async () => {
+  const apiKey = 'secret-network-key';
+  const client = createCrawleoClient({
+    apiKey,
+    fetch: async () => {
+      throw new Error(`socket closed for ${apiKey}`);
+    }
+  });
+
+  await assert.rejects(
+    () => client.search({ query: 'ai agents' }),
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.TRANSPORT);
+      assert.equal(error.status, undefined);
+      assert.equal(JSON.stringify(error).includes(apiKey), false);
+      return true;
+    }
+  );
+});

package/test/live.test.js

@@ -0,0 +1,28 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { CrawleoError, createCrawleoClient } from '../src/index.js';
+
+const liveEnabled = process.env.CRAWLEO_ENABLE_LIVE_TESTS === '1';
+const apiKey = process.env.CRAWLEO_API_KEY;
+const skipReason = 'Set CRAWLEO_ENABLE_LIVE_TESTS=1 and CRAWLEO_API_KEY to run live Crawleo tests.';
+
+test('live Crawleo /search smoke test is explicitly gated', { skip: liveEnabled && apiKey ? false : skipReason }, async () => {
+  const client = createCrawleoClient({ apiKey });
+
+  try {
+    const result = await client.search({
+      query: 'Crawleo web intelligence',
+      max_pages: 1
+    });
+
+    assert.equal(typeof result, 'object');
+    assert.notEqual(result, null);
+  } catch (error) {
+    if (error instanceof CrawleoError) {
+      throw new Error(`Live Crawleo smoke test failed: ${JSON.stringify(error.toJSON())}`);
+    }
+
+    throw error;
+  }
+});

package/test/scaffold.test.js

@@ -0,0 +1,47 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import {
+  CRAWLEO_CONTRACT_PATH,
+  CRAWLEO_MCP_TOOLS,
+  CRAWLEO_REST_ENDPOINTS,
+  CRAWLEO_SKILL_INSTRUCTIONS_PATH,
+  CRAWLEO_SKILL_MANIFEST_PATH,
+  createCrawleoClient,
+  crawl,
+  getScaffoldStatus,
+  googleMaps,
+  googleSearch,
+  headfulBrowser,
+  search
+} from '../src/index.js';
+
+test('scaffold exports the Crawleo contract location and documented capability names', () => {
+  assert.equal(CRAWLEO_CONTRACT_PATH, 'contracts/crawleo-endpoints.json');
+  assert.equal(CRAWLEO_SKILL_MANIFEST_PATH, 'skill.json');
+  assert.equal(CRAWLEO_SKILL_INSTRUCTIONS_PATH, 'SKILL.md');
+  assert.deepEqual(CRAWLEO_REST_ENDPOINTS, ['/search', '/google-search', '/google-maps', '/crawl', '/headful-browser']);
+  assert.deepEqual(CRAWLEO_MCP_TOOLS, ['search_web', 'google_search', 'google_maps', 'crawl_web', 'headful_browser']);
+});
+
+test('scaffold status is explicit that runtime wrappers are implemented and live calls are opt-in', () => {
+  assert.deepEqual(getScaffoldStatus(), {
+    packageName: 'openclaw-crawleo-skill',
+    implementationStatus: 'rest-wrappers-implemented',
+    contractPath: 'contracts/crawleo-endpoints.json',
+    skillManifestPath: 'skill.json',
+    skillInstructionsPath: 'SKILL.md',
+    restEndpoints: ['/search', '/google-search', '/google-maps', '/crawl', '/headful-browser'],
+    mcpTools: ['search_web', 'google_search', 'google_maps', 'crawl_web', 'headful_browser'],
+    liveCrawleoCallsEnabledByDefault: false
+  });
+});
+
+test('public API exports client factory and all endpoint wrapper functions', () => {
+  assert.equal(typeof createCrawleoClient, 'function');
+  assert.equal(typeof search, 'function');
+  assert.equal(typeof googleSearch, 'function');
+  assert.equal(typeof googleMaps, 'function');
+  assert.equal(typeof crawl, 'function');
+  assert.equal(typeof headfulBrowser, 'function');
+});

package/test/wrapper-fixtures.test.js

@@ -0,0 +1,227 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { CRAWLEO_ERROR_CODES, createCrawleoClient } from '../src/index.js';
+
+function createRecordingClient() {
+  const calls = [];
+  const client = createCrawleoClient({
+    apiKey: 'offline-wrapper-fixture-key',
+    fetch: async (url, init) => {
+      const requestUrl = new URL(url);
+      calls.push({ url: requestUrl, init });
+      return {
+        ok: true,
+        status: 200,
+        headers: new Map([['content-type', 'application/json']]),
+        async text() {
+          return JSON.stringify({
+            ok: true,
+            path: requestUrl.pathname,
+            query: Object.fromEntries(requestUrl.searchParams.entries())
+          });
+        }
+      };
+    }
+  });
+
+  return { client, calls };
+}
+
+async function assertSingleRequest(wrapperCall, { path, query }) {
+  const { client, calls } = createRecordingClient();
+  const result = await wrapperCall(client);
+
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].url.origin, 'https://api.crawleo.dev');
+  assert.equal(calls[0].url.pathname, path);
+  assert.equal(calls[0].init.method, 'GET');
+  assert.equal(calls[0].init.headers['x-api-key'], 'offline-wrapper-fixture-key');
+  assert.equal(calls[0].init.headers.accept, 'application/json');
+
+  for (const [key, value] of Object.entries(query)) {
+    assert.equal(calls[0].url.searchParams.get(key), String(value));
+  }
+
+  assert.equal(result.path, path);
+  return calls[0].url;
+}
+
+function assertValidationError(fn, { endpoint, field }) {
+  assert.throws(
+    fn,
+    (error) => {
+      assert.equal(error.code, CRAWLEO_ERROR_CODES.VALIDATION);
+      assert.equal(error.endpoint, endpoint);
+      assert.equal(error.field, field);
+      return true;
+    }
+  );
+}
+
+test('search wrapper constructs /search URL with documented query params', async () => {
+  await assertSingleRequest(
+    (client) => client.search({
+      query: 'machine learning',
+      max_pages: 2,
+      device: 'mobile',
+      markdown: true,
+      auto_crawling: false
+    }),
+    {
+      path: '/search',
+      query: {
+        query: 'machine learning',
+        max_pages: '2',
+        device: 'mobile',
+        markdown: 'true',
+        auto_crawling: 'false'
+      }
+    }
+  );
+});
+
+test('googleSearch wrapper constructs /google-search URL with documented query params', async () => {
+  await assertSingleRequest(
+    (client) => client.googleSearch({
+      q: 'best CRM software',
+      gl: 'us',
+      hl: 'en',
+      type: 'shopping',
+      tbs: 'qdr:w',
+      num: 10,
+      page: 2
+    }),
+    {
+      path: '/google-search',
+      query: {
+        q: 'best CRM software',
+        gl: 'us',
+        hl: 'en',
+        type: 'shopping',
+        tbs: 'qdr:w',
+        num: '10',
+        page: '2'
+      }
+    }
+  );
+});
+
+test('googleMaps wrapper constructs /google-maps URL with documented query params', async () => {
+  await assertSingleRequest(
+    (client) => client.googleMaps({
+      q: 'restaurants in Paris',
+      hl: 'fr',
+      ll: '@48.8566,2.3522,15z',
+      placeId: 'ChIJLU7jZClu5kcR4PcOOO6p3I0'
+    }),
+    {
+      path: '/google-maps',
+      query: {
+        q: 'restaurants in Paris',
+        hl: 'fr',
+        ll: '@48.8566,2.3522,15z',
+        placeId: 'ChIJLU7jZClu5kcR4PcOOO6p3I0'
+      }
+    }
+  );
+});
+
+test('crawl wrapper serializes string and array urls as documented comma-separated query values', async () => {
+  await assertSingleRequest(
+    (client) => client.crawl({ urls: 'https://example.com/a', markdown: true }),
+    {
+      path: '/crawl',
+      query: {
+        urls: 'https://example.com/a',
+        markdown: 'true'
+      }
+    }
+  );
+
+  await assertSingleRequest(
+    (client) => client.crawl({ urls: ['https://example.com/a', 'https://example.com/b'], render_js: true, screenshot: true }),
+    {
+      path: '/crawl',
+      query: {
+        urls: 'https://example.com/a,https://example.com/b',
+        render_js: 'true',
+        screenshot: 'true'
+      }
+    }
+  );
+});
+
+test('headfulBrowser wrapper serializes urls and output options for /headful-browser', async () => {
+  await assertSingleRequest(
+    (client) => client.headfulBrowser({
+      urls: ['https://example.com/a', 'https://example.com/b'],
+      country: 'gb',
+      output_format: 'page_text',
+      screenshot: false
+    }),
+    {
+      path: '/headful-browser',
+      query: {
+        urls: 'https://example.com/a,https://example.com/b',
+        country: 'gb',
+        output_format: 'page_text',
+        screenshot: 'false'
+      }
+    }
+  );
+});
+
+test('required parameter validation identifies every documented required field', () => {
+  const { client } = createRecordingClient();
+
+  assertValidationError(() => client.search({}), { endpoint: '/search', field: 'query' });
+  assertValidationError(() => client.search({ query: '' }), { endpoint: '/search', field: 'query' });
+  assertValidationError(() => client.googleSearch({}), { endpoint: '/google-search', field: 'q' });
+  assertValidationError(() => client.googleSearch({ q: '' }), { endpoint: '/google-search', field: 'q' });
+  assertValidationError(() => client.googleMaps({}), { endpoint: '/google-maps', field: 'q' });
+  assertValidationError(() => client.googleMaps({ q: '' }), { endpoint: '/google-maps', field: 'q' });
+  assertValidationError(() => client.crawl({}), { endpoint: '/crawl', field: 'urls' });
+  assertValidationError(() => client.crawl({ urls: [] }), { endpoint: '/crawl', field: 'urls' });
+  assertValidationError(() => client.headfulBrowser({}), { endpoint: '/headful-browser', field: 'urls' });
+  assertValidationError(() => client.headfulBrowser({ urls: [] }), { endpoint: '/headful-browser', field: 'urls' });
+});
+
+test('documented enum validators accept all allowed values', async () => {
+  for (const device of ['desktop', 'mobile', 'tablet']) {
+    await assertSingleRequest((client) => client.search({ query: 'ai agents', device }), {
+      path: '/search',
+      query: { query: 'ai agents', device }
+    });
+  }
+
+  for (const type of ['search', 'news', 'images', 'places', 'shopping']) {
+    await assertSingleRequest((client) => client.googleSearch({ q: 'ai agents', type }), {
+      path: '/google-search',
+      query: { q: 'ai agents', type }
+    });
+  }
+
+  for (const tbs of ['qdr:h', 'qdr:d', 'qdr:w', 'qdr:m', 'qdr:y']) {
+    await assertSingleRequest((client) => client.googleSearch({ q: 'ai agents', tbs }), {
+      path: '/google-search',
+      query: { q: 'ai agents', tbs }
+    });
+  }
+
+  for (const output_format of ['markdown', 'enhanced_html', 'raw_html', 'page_text']) {
+    await assertSingleRequest((client) => client.headfulBrowser({ urls: 'https://example.com', output_format }), {
+      path: '/headful-browser',
+      query: { urls: 'https://example.com', output_format }
+    });
+  }
+});
+
+test('documented enum validators reject invalid values with field diagnostics', () => {
+  const { client } = createRecordingClient();
+
+  assertValidationError(() => client.search({ query: 'ai agents', device: 'watch' }), { endpoint: '/search', field: 'device' });
+  assertValidationError(() => client.googleSearch({ q: 'ai agents', type: 'videos' }), { endpoint: '/google-search', field: 'type' });
+  assertValidationError(() => client.googleSearch({ q: 'ai agents', tbs: 'qdr:decade' }), { endpoint: '/google-search', field: 'tbs' });
+  assertValidationError(() => client.headfulBrowser({ urls: 'https://example.com', output_format: 'pdf' }), { endpoint: '/headful-browser', field: 'output_format' });
+});