fetchfox-sdk 1.0.2 → 1.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.2",
+  "version": "1.0.5",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",
@@ -28,6 +28,13 @@
     "eslint-plugin-promise": "^7.2.1",
     "husky": "^9.1.7",
     "jest": "^30.0.4",
+    "lint-staged": "^16.1.2",
     "prettier": "^3.6.2"
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx,css,md}": "prettier --write"
+  },
+  "dependencies": {
+    "socket.io-client": "^4.8.1"
   }
 }

package/src/api.js

@@ -1,31 +1,24 @@
 import { apiKey, host } from './configure.js';
 
-const endpoint = (path) => `${host()}${path}`;
-
-const clean = (dict) => {
-  const result = {};
-
-  for (const key in dict) {
-    if (dict.hasOwnProperty(key)) {
-      const snakeKey = key
-        .replace(/([A-Z])/g, '_$1') // insert _ before uppercase letters
-        .toLowerCase() // convert all to lowercase
-        .replace(/^_/, ''); // remove leading _ if any
+const camelCase = (obj) => {
+  const snakeToCamel = (str) =>
+    str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
+  return Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [snakeToCamel(key), value])
+  );
+};
 
-      result[snakeKey] = dict[key];
-    }
-  }
+const endpoint = (path) => `${host()}${path}`;
 
-  if (result.apiKey) {
-    delete result.apiKey;
+const FetchFoxAPIError = class extends Error {
+  constructor(errors) {
+    super(JSON.stringify(errors));
+    this.errors = errors;
   }
-
-  return result;
 };
 
 export const call = async (method, path, params) => {
   const key = apiKey(params);
-  params = clean(params);
 
   const args = {
     method,
@@ -43,5 +36,24 @@ export const call = async (method, path, params) => {
   }
 
   const resp = await fetch(url, args);
-  return resp.json();
+  const text = await resp.text();
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (e) {
+    throw new Error(`FetchFox returned invalid JSON: ${text}`);
+  }
+
+  if (data.errors) {
+    throw new FetchFoxAPIError(data.errors);
+  }
+
+  if (resp.status >= 400) {
+    throw new FetchFoxAPIError({
+      status: `Received status=${resp.status}`,
+      ...data,
+    });
+  }
+
+  return camelCase(data);
 };

package/src/configure.js

@@ -2,11 +2,12 @@ const config = {
   host: 'https://api.fetchfox.ai',
 };
 
-const isNode = typeof process !== 'undefined' && 
-               process.versions != null && 
-               process.versions.node != null;
+const isNode =
+  typeof process !== 'undefined' &&
+  process.versions != null &&
+  process.versions.node != null;
 
-const safeEnv = (key) => isNode ? process.env[key] : null;
+const safeEnv = (key) => (isNode ? process.env[key] : null);
 
 export const configure = ({ apiKey, host }) => {
   if (apiKey) {
@@ -22,3 +23,9 @@ export const apiKey = (options) =>
 
 export const host = (options) =>
   options?.host || config.host || safeEnv('FETCHFOX_HOST');
+
+export const ws = (options) =>
+  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
+    'http',
+    'ws'
+  );

package/src/crawl.js

@@ -1,5 +1,11 @@
 import { call } from './api.js';
+import { Job } from './detach.js';
 
-export const crawl = async (args) => {
+export const crawl = async function () {
   return call('POST', '/api/crawl', args);
 };
+crawl.detach = async (args) => {
+  const data = await call('POST', '/api/crawl', { ...args, detach: true });
+  console.log('detach data', data);
+  return new Job(data.jobId);
+};

package/src/detach.js

@@ -0,0 +1,139 @@
+import { io } from 'socket.io-client';
+import { ws } from './configure.js';
+import { jobs } from './jobs.js';
+
+class FetchFoxError extends Error {}
+
+const interval = 1_000;
+
+export function getSocket() {}
+
+export const Job = class {
+  #callbacks;
+  #socket;
+
+  constructor(id) {
+    this.id = id;
+    this.#callbacks = {
+      completed: [],
+      error: [],
+      finished: [],
+      progress: [],
+    };
+
+    this.#socket = new io(ws());
+    this.#socket.on('progress', (data) => {
+      this.handleProgress(data);
+      console.log('==> socket got progress', data);
+    });
+    console.log('socket emit sub', this.id);
+    this.#socket.emit('sub', this.id);
+
+    // this.poll();
+  }
+
+  get _finished() {
+    return this._completed || this._error;
+  }
+
+  #select(data) {
+    const s = {};
+    for (const key of [
+      'name',
+      'state',
+      'args',
+      'metrics',
+      'progress',
+      'results',
+      'artifacts',
+      'timer',
+    ]) {
+      s[key] = data[key] || this[key];
+    }
+    return s;
+  }
+
+  async get() {
+    const data = await jobs.get(this.id);
+    const s = this.#select(data);
+    for (const key of Object.keys(s)) {
+      this[key] = s[key];
+    }
+    return this;
+  }
+
+  handleProgress(data) {
+    const last = JSON.stringify(this);
+
+    const s = this.#select(data);
+    for (const key of Object.keys(s)) {
+      this[key] = s[key];
+    }
+
+    const didUpdate = JSON.stringify(this) != last;
+    if (didUpdate) {
+      console.log('=> Job progressed:', this);
+      this.trigger('progress');
+
+      if (this.state == 'completed') {
+        this._completed = true;
+        this.trigger('completed');
+      }
+      if (this.state == 'error') {
+        this._error = true;
+        this.trigger('error');
+      }
+
+      if (['completed', 'error'].includes(this.state)) {
+        this.trigger('finished');
+      }
+    }
+  }
+
+  checkEvent(event) {
+    if (!this.#callbacks[event]) {
+      throw new FetchFoxError(`Invalid event: ${event}`);
+    }
+  }
+
+  trigger(event) {
+    console.log('Trigger:', this.id, event);
+
+    this.checkEvent(event);
+    for (const cb of this.#callbacks[event]) {
+      cb({ ...this });
+    }
+  }
+
+  on(event, cb) {
+    this.checkEvent(event);
+    this.#callbacks[event].push(cb);
+  }
+
+  off(event, cb) {
+    this.checkEvent(event);
+    this.#callbacks[event] = this.#callbacks[event].filter((it) => it != cb);
+  }
+
+  async waitFor(event) {
+    this.checkEvent(event);
+    return new Promise((ok) => {
+      this.on(event, () => {
+        this.off(event, ok);
+        ok({ ...this });
+      });
+    });
+  }
+
+  async completed() {
+    return this.waitFor('completed');
+  }
+
+  async error() {
+    return this.waitFor('error');
+  }
+
+  async finished() {
+    return this.waitFor('finished');
+  }
+};

package/src/extract.js

@@ -1,5 +1,12 @@
 import { call } from './api.js';
+import { Job } from './detach.js';
 
 export const extract = async (args) => {
   return call('POST', '/api/extract', args);
 };
+
+extract.detach = async (args) => {
+  const data = await call('POST', '/api/extract', { ...args, detach: true });
+  console.log('detach data', data);
+  return new Job(data.jobId);
+};

package/src/index.js

@@ -4,5 +4,6 @@ export * from './extract.js';
 export * from './jobs.js';
 export * from './user.js';
 export * from './proxy.js';
+export { Job } from './detach.js';
 export { call } from './api.js';
-export { configure } from './configure.js';
+export { configure, host, apiKey } from './configure.js';