fetchfox-sdk 1.0.24 → 1.0.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.24",
+  "version": "1.0.26",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",

package/src/agent.js

@@ -7,5 +7,5 @@ export async function agent(args) {
 
 agent.detach = async (args) => {
   const data = await call('POST', '/api/agent', { ...args, detach: true });
-  return new Job(data.jobId, args);
+  return new Job(data.jobId, { ...args, name: 'agent' });
 };

package/src/crawl.js

@@ -7,5 +7,5 @@ export async function crawl(args) {
 
 crawl.detach = async (args) => {
   const data = await call('POST', '/api/crawl', { ...args, detach: true });
-  return new Job(data.jobId, args);
+  return new Job(data.jobId, { ...args, name: 'crawl' });
 };

package/src/detach.js

@@ -11,27 +11,55 @@ export function getSocket() {}
 export const Job = class {
   #callbacks;
   #socket;
+  #socket2;
+  #socket3;
   #seen;
   #completed;
   #failed;
 
   constructor(id, options) {
     this.id = id;
+    this.name = options?.name;
     this.#callbacks = {
       item: [],
       completed: [],
       failed: [],
       finished: [],
       progress: [],
+
+      // new
+      update: [],
+      children: [],
     };
 
     this.#seen = {};
-
     this.#socket = new io(ws(options));
     this.#socket.on('progress', (data) => {
       this.handleProgress(data);
     });
     this.#socket.emit('sub', this.id);
+
+    this.#socket2 = new WebSocket(ws(options) + '/ws/jobs/' + id);
+    this.#socket2.onmessage = (data) => {
+      let parsed;
+      try {
+        parsed = JSON.parse(data.data);
+      } catch {
+        return;
+      }
+      this.trigger('update', parsed);
+    };
+
+    this.#socket3 = new WebSocket(ws(options) + '/ws/children/' + id);
+    this.#socket3.onmessage = (data) => {
+      let parsed;
+      try {
+        parsed = JSON.parse(data.data);
+      } catch {
+        return;
+      }
+      this.trigger('children', parsed);
+    };
   }
 
   get _finished() {
@@ -39,7 +67,9 @@ export const Job = class {
   }
 
   get appUrl() {
-    return appHost() + '/jobs/' + this.id;
+    return [appHost(), this.name == 'agent' ? 'agents' : 'jobs', this.id].join(
+      '/'
+    );
   }
 
   #select(data) {
@@ -108,6 +138,8 @@ export const Job = class {
     }
 
     if (['completed', 'failed'].includes(this.state)) {
+      console.log('Got state:', this.state);
+
       if (this.progress?.children?.jobs) {
         this.progress.children.jobs = this.progress.children.jobs.filter(
           (it) => it.state != 'active'
@@ -116,13 +148,16 @@ export const Job = class {
 
       this.#socket.disconnect();
 
-      this.get()
-        .then(() => {
-          this.trigger('progress', this);
-          this.trigger(this.state, this);
-          this.trigger('finished', this);
-        })
-        .catch((e) => console.error(e));
+      console.log('Set timeout before triggering completion');
+      setTimeout(() => {
+        this.get()
+          .then(() => {
+            this.trigger('progress', this);
+            this.trigger(this.state, this);
+            this.trigger('finished', this);
+          })
+          .catch((e) => console.error(e));
+      }, 500);
     }
   }
 

package/src/extract.js

@@ -7,5 +7,5 @@ export async function extract(args) {
 
 extract.detach = async (args) => {
   const data = await call('POST', '/api/extract', { ...args, detach: true });
-  return new Job(data.jobId, args);
+  return new Job(data.jobId, { ...args, name: 'extract' });
 };

package/src/scrape.js

@@ -7,5 +7,5 @@ export async function scrape(args) {
 
 scrape.detach = async (args) => {
   const data = await call('POST', '/api/scrape', { ...args, detach: true });
-  return new Job(data.jobId, args);
+  return new Job(data.jobId, { ...args, name: 'scrape' });
 };

package/src/visit.js

@@ -7,5 +7,5 @@ export async function visit(args) {
 
 visit.detach = async (args) => {
   const data = await call('POST', '/api/visit', { ...args, detach: true });
-  return new Job(data.jobId, args);
+  return new Job(data.jobId, { ...args, name: 'visit' });
 };