fetchfox-sdk 1.0.0

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "fetchfox-sdk",
+  "version": "1.0.0",
+  "description": "AI scraper",
+  "main": "src/index.js",
+  "type": "module",
+  "scripts": {
+    "format": "prettier --write .",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "prepare": "husky"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fetchfox/fetchfox.git"
+  },
+  "keywords": [
+    "ai",
+    "scraping"
+  ],
+  "author": "marcell@fetchfoxai.com",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/fetchfox/fetchfox/issues"
+  },
+  "homepage": "https://github.com/fetchfox/fetchfox#readme",
+  "devDependencies": {
+    "@eslint/js": "^9.31.0",
+    "eslint-plugin-promise": "^7.2.1",
+    "husky": "^9.1.7",
+    "jest": "^30.0.4",
+    "prettier": "^3.6.2"
+  }
+}

package/package.json~

@@ -0,0 +1,26 @@
+{
+  "name": "fetchfox",
+  "version": "1.0.0",
+  "description": "AI scraper",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fetchfox/fetchfox.git"
+  },
+  "keywords": [
+    "ai",
+    "scraping"
+  ],
+  "author": "marcell@fetchfoxai.com",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/fetchfox/fetchfox/issues"
+  },
+  "homepage": "https://github.com/fetchfox/fetchfox#readme",
+  "devDependencies": {
+    "jest": "^30.0.4"
+  }
+}

package/src/api.js

@@ -0,0 +1,47 @@
+import { apiKey, host } from './configure.js';
+
+const endpoint = (path) => `${host()}${path}`;
+
+const clean = (dict) => {
+  const result = {};
+
+  for (const key in dict) {
+    if (dict.hasOwnProperty(key)) {
+      const snakeKey = key
+        .replace(/([A-Z])/g, '_$1') // insert _ before uppercase letters
+        .toLowerCase() // convert all to lowercase
+        .replace(/^_/, ''); // remove leading _ if any
+
+      result[snakeKey] = dict[key];
+    }
+  }
+
+  if (result.apiKey) {
+    delete result.apiKey;
+  }
+
+  return result;
+};
+
+export const call = async (method, path, params) => {
+  const key = apiKey(params);
+  params = clean(params);
+
+  const args = {
+    method,
+    headers: {
+      Authorization: `Bearer ${key}`,
+      'Content-Type': 'application/json',
+    },
+  };
+
+  let url = endpoint(path);
+  if (method == 'GET') {
+    url += '?' + new URLSearchParams(params).toString();
+  } else {
+    args.body = JSON.stringify(params);
+  }
+
+  const resp = await fetch(url, args);
+  return resp.json();
+};

package/src/configure.js

@@ -0,0 +1,24 @@
+const config = {
+  host: 'https://api.fetchfox.ai',
+};
+
+const isNode = typeof process !== 'undefined' && 
+               process.versions != null && 
+               process.versions.node != null;
+
+const safeEnv = (key) => isNode ? process.env[key] : null;
+
+export const configure = ({ apiKey, host }) => {
+  if (apiKey) {
+    config.apiKey = apiKey;
+  }
+  if (host) {
+    config.host = host;
+  }
+};
+
+export const apiKey = (options) =>
+  options?.apiKey || config.apiKey || safeEnv('FETCHFOX_API_KEY');
+
+export const host = (options) =>
+  options?.host || config.host || safeEnv('FETCHFOX_HOST');

package/src/configure.js~

@@ -0,0 +1,18 @@
+const config = {
+  host: 'https://api.fetchfox.ai',
+};
+
+export const configure = ({ apiKey, host }) => {
+  if (apiKey) {
+    config.apiKey = apiKey;
+  }
+  if (host) {
+    config.host = host;
+  }
+};
+
+export const apiKey = (options) =>
+  options?.apiKey || config.apiKey || process.env.FETCHFOX_API_KEY;
+
+export const host = (options) =>
+  options?.host || config.host || process.env.FETCHFOX_HOST;

package/src/crawl.js

@@ -0,0 +1,5 @@
+import { call } from './api.js';
+
+export const crawl = async (pattern, options) => {
+  return call('POST', '/api/crawl', { pattern, ...options });
+};

package/src/extract.js

@@ -0,0 +1,5 @@
+import { call } from './api.js';
+
+export const extract = async (urls, template, options) => {
+  return call('POST', '/api/extract', { urls, template, ...options });
+};

package/src/index.js

@@ -0,0 +1,6 @@
+export * from './crawl.js';
+export * from './extract.js';
+export * from './jobs.js';
+export * from './user.js';
+export { call } from './api.js';
+export { configure } from './configure.js';

package/src/jobs.js

@@ -0,0 +1,10 @@
+import { call } from './api.js';
+
+export const jobs = {
+  get: async (id) => {
+    return call('GET', `/api/jobs/${id}`);
+  },
+  list: async (limit = 100, offset = 0) => {
+    return call('GET', '/api/jobs', { limit, offset });
+  },
+};

package/src/jobs.js~

@@ -0,0 +1,7 @@
+import { call } from './api.js';
+
+export const jobs = {
+  list: async (limit = 100, offset = 0) => {
+    return call('GET', '/api/jobs', { limit, offset });
+  },
+};

package/src/user.js

@@ -0,0 +1,5 @@
+import { call } from './api.js';
+
+export const user = {
+  get: async () => call('GET', '/api/user'),
+};

package/tests/crawl.test.js

@@ -0,0 +1,6 @@
+import { crawl } from '../src';
+
+test('crawl a pattern @sanity', async () => {
+  const out = await crawl('https://pokemondb.net/pokedex/*', { maxVisits: 5 });
+  console.log(out);
+}, 30_000);

package/tests/jobs.test.js

@@ -0,0 +1,7 @@
+import { jobs, configure } from '../src';
+
+test('list jobs @sanity', async () => {
+  configure({ host: 'http://localhost:3030' });
+  const out = await jobs.list();
+  console.log(out.results.map((it) => [it.id, it.details.state]));
+});