fetchfox-sdk 1.0.3 → 1.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",
@@ -28,6 +28,13 @@
     "eslint-plugin-promise": "^7.2.1",
     "husky": "^9.1.7",
     "jest": "^30.0.4",
+    "lint-staged": "^16.1.2",
     "prettier": "^3.6.2"
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx,css,md}": "prettier --write"
+  },
+  "dependencies": {
+    "socket.io-client": "^4.8.1"
   }
 }

package/src/api.js

@@ -15,7 +15,7 @@ const FetchFoxAPIError = class extends Error {
     super(JSON.stringify(errors));
     this.errors = errors;
   }
-}
+};
 
 export const call = async (method, path, params) => {
   const key = apiKey(params);
@@ -52,7 +52,7 @@ export const call = async (method, path, params) => {
     throw new FetchFoxAPIError({
       status: `Received status=${resp.status}`,
       ...data,
-    })
+    });
   }
 
   return camelCase(data);

package/src/configure.js

@@ -2,11 +2,12 @@ const config = {
   host: 'https://api.fetchfox.ai',
 };
 
-const isNode = typeof process !== 'undefined' && 
-               process.versions != null && 
-               process.versions.node != null;
+const isNode =
+  typeof process !== 'undefined' &&
+  process.versions != null &&
+  process.versions.node != null;
 
-const safeEnv = (key) => isNode ? process.env[key] : null;
+const safeEnv = (key) => (isNode ? process.env[key] : null);
 
 export const configure = ({ apiKey, host }) => {
   if (apiKey) {
@@ -22,3 +23,9 @@ export const apiKey = (options) =>
 
 export const host = (options) =>
   options?.host || config.host || safeEnv('FETCHFOX_HOST');
+
+export const ws = (options) =>
+  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
+    'http',
+    'ws'
+  );

package/src/crawl.js

@@ -3,9 +3,9 @@ import { Job } from './detach.js';
 
 export const crawl = async function () {
   return call('POST', '/api/crawl', args);
-}
+};
 crawl.detach = async (args) => {
   const data = await call('POST', '/api/crawl', { ...args, detach: true });
   console.log('detach data', data);
   return new Job(data.jobId);
-}
+};

package/src/detach.js

@@ -1,11 +1,16 @@
+import { io } from 'socket.io-client';
+import { ws } from './configure.js';
 import { jobs } from './jobs.js';
 
-class FetchFoxError extends Error {};
+class FetchFoxError extends Error {}
 
 const interval = 1_000;
 
+export function getSocket() {}
+
 export const Job = class {
   #callbacks;
+  #socket;
 
   constructor(id) {
     this.id = id;
@@ -16,7 +21,15 @@ export const Job = class {
       progress: [],
     };
 
-    this.poll();
+    this.#socket = new io(ws());
+    this.#socket.on('progress', (data) => {
+      this.handleProgress(data);
+      console.log('==> socket got progress', data);
+    });
+    console.log('socket emit sub', this.id);
+    this.#socket.emit('sub', this.id);
+
+    // this.poll();
   }
 
   get _finished() {
@@ -24,15 +37,20 @@ export const Job = class {
   }
 
   #select(data) {
-    return {
-      name: data.name,
-      state: data.state,
-      args: data.args,
-      metrics: data.metrics,
-      progress: data.progress,
-      results: data.results,
-      artifacts: data.artifacts,
-    };
+    const s = {};
+    for (const key of [
+      'name',
+      'state',
+      'args',
+      'metrics',
+      'progress',
+      'results',
+      'artifacts',
+      'timer',
+    ]) {
+      s[key] = data[key] || this[key];
+    }
+    return s;
   }
 
   async get() {
@@ -44,12 +62,17 @@ export const Job = class {
     return this;
   }
 
-  async poll() {
+  handleProgress(data) {
     const last = JSON.stringify(this);
-    await this.get();
+
+    const s = this.#select(data);
+    for (const key of Object.keys(s)) {
+      this[key] = s[key];
+    }
+
     const didUpdate = JSON.stringify(this) != last;
     if (didUpdate) {
-      console.log('Job progressed:', this);
+      console.log('=> Job progressed:', this);
       this.trigger('progress');
 
       if (this.state == 'completed') {
@@ -65,10 +88,6 @@ export const Job = class {
         this.trigger('finished');
       }
     }
-
-    if (!this._finished) {
-      setTimeout(() => this.poll(), interval);
-    }
   }
 
   checkEvent(event) {
@@ -93,7 +112,7 @@ export const Job = class {
 
   off(event, cb) {
     this.checkEvent(event);
-    this.#callbacks[event] = this.#callbacks[event].filter(it => it != cb);
+    this.#callbacks[event] = this.#callbacks[event].filter((it) => it != cb);
   }
 
   async waitFor(event) {
@@ -117,4 +136,4 @@ export const Job = class {
   async finished() {
     return this.waitFor('finished');
   }
-}
+};

package/src/extract.js

@@ -1,5 +1,12 @@
 import { call } from './api.js';
+import { Job } from './detach.js';
 
 export const extract = async (args) => {
   return call('POST', '/api/extract', args);
 };
+
+extract.detach = async (args) => {
+  const data = await call('POST', '/api/extract', { ...args, detach: true });
+  console.log('detach data', data);
+  return new Job(data.jobId);
+};

package/src/index.js

@@ -6,4 +6,4 @@ export * from './user.js';
 export * from './proxy.js';
 export { Job } from './detach.js';
 export { call } from './api.js';
-export { configure } from './configure.js';
+export { configure, host, apiKey } from './configure.js';