fetchfox-sdk 1.0.19 → 1.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",

package/src/detach.js

@@ -12,13 +12,15 @@ export const Job = class {
   #callbacks;
   #socket;
   #seen;
+  #completed;
+  #failed;
 
   constructor(id, options) {
     this.id = id;
     this.#callbacks = {
       item: [],
       completed: [],
-      error: [],
+      failed: [],
       finished: [],
       progress: [],
     };
@@ -33,7 +35,7 @@ export const Job = class {
   }
 
   get _finished() {
-    return this._completed || this._error;
+    return this.#completed || this.#failed;
   }
 
   get appUrl() {
@@ -42,22 +44,15 @@ export const Job = class {
 
   #select(data) {
     const s = {};
-    for (const key of [
-      'name',
-      'state',
-      'args',
-      'metrics',
-      'progress',
-      'results',
-      'artifacts',
-      'timer',
-    ]) {
-      s[key] = data[key] || this[key];
+    for (const key of Object.keys(data)) {
+      const val = data[key] ?? this[key];
+      if (val === undefined) {
+        continue;
+      }
+      s[key] = val;
     }
 
     if (s.progress?.children?.jobs) {
-      // const late = this.progress.children.jobs.filter(it => it.late);
-      // console.log('late jobs:', late);
       s.progress.children.jobs = s.progress.children.jobs.filter(
         (it) => !it.late
       );
@@ -66,24 +61,30 @@ export const Job = class {
     return s;
   }
 
-  async get() {
-    const data = await jobs.get(this.id);
+  updateFromData(data) {
     const s = this.#select(data);
     for (const key of Object.keys(s)) {
       this[key] = s[key];
     }
+    if (['completed', 'failed'].includes(this.state)) {
+      if (this.progress?.children?.jobs) {
+        this.progress.children.jobs = this.progress.children.jobs.filter(
+          (it) => it.state != 'active'
+        );
+      }
+    }
+  }
+
+  async get() {
+    const data = await jobs.get(this.id);
+    this.updateFromData(data);
     return this;
   }
 
   handleProgress(data) {
-    console.log('handleProgress', data);
-
     const last = JSON.stringify(this);
 
-    const s = this.#select(data);
-    for (const key of Object.keys(s)) {
-      this[key] = s[key];
-    }
+    this.updateFromData(data);
 
     const didUpdate = JSON.stringify(this) != last;
     if (didUpdate) {
@@ -99,23 +100,29 @@ export const Job = class {
       }
 
       if (this.state == 'completed') {
-        this._completed = true;
-        this.trigger('completed', this);
+        this.#completed = true;
       }
-      if (this.state == 'error') {
-        this._error = true;
-        this.trigger('error', this);
+      if (this.state == 'failed') {
+        this.#failed = true;
       }
+    }
 
-      if (['completed', 'error'].includes(this.state)) {
-        if (this.progress?.children?.jobs) {
-          this.progress.children.jobs = this.progress.children.jobs.filter(
-            (it) => it.state != 'active'
-          );
-        }
-        this.trigger('finished', this);
-        this.#socket.disconnect();
+    if (['completed', 'failed'].includes(this.state)) {
+      if (this.progress?.children?.jobs) {
+        this.progress.children.jobs = this.progress.children.jobs.filter(
+          (it) => it.state != 'active'
+        );
       }
+
+      this.#socket.disconnect();
+
+      this.get()
+        .then(() => {
+          this.trigger('progress', this);
+          this.trigger(this.state, this);
+          this.trigger('finished', this);
+        })
+        .catch((e) => console.error(e));
     }
   }
 
@@ -156,8 +163,8 @@ export const Job = class {
     return this.waitFor('completed');
   }
 
-  async error() {
-    return this.waitFor('error');
+  async failed() {
+    return this.waitFor('failed');
   }
 
   async finished() {

package/src/fetchfox.js

@@ -7,7 +7,7 @@ export const FetchFox = class {
     this.apiKey = apiKey;
     this.host = host;
 
-    const fns = [crawl, extract, scrape];
+    const fns = [visit, crawl, extract, scrape];
 
     for (const fn of fns) {
       this[fn.name] = function (args) {

package/src/index.js

@@ -1,11 +1,14 @@
-export * from './scrape.js';
+export * from './visit.js';
 export * from './crawl.js';
 export * from './extract.js';
+export * from './scrape.js';
+
 export * from './jobs.js';
 export * from './user.js';
 export * from './credits.js';
 export * from './proxy.js';
 export * from './urls.js';
+
 export { Job } from './detach.js';
 export { call } from './api.js';
 export { FetchFox } from './fetchfox.js';

package/src/visit.js

@@ -0,0 +1,11 @@
+import { call } from './api.js';
+import { Job } from './detach.js';
+
+export async function visit(args) {
+  return call('POST', '/api/visit', args);
+}
+
+visit.detach = async (args) => {
+  const data = await call('POST', '/api/visit', { ...args, detach: true });
+  return new Job(data.jobId, args);
+};

package/src/#index.js#

@@ -1,13 +0,0 @@
-export * from './scrape.js';
-export * from './crawl.js';
-export * from './extract.js';
-export * from './jobs.js';
-export * from './user.js';
-export * from './credits.js';
-export * from './proxy.js';
-export * from './urls.js';
-
-export { Job } from './detach.js';
-export { call } from './api.js';
-export { FetchFox } from './fetchfox.js';
-export { configure, host, ws, apiKey } from './configure.js';