fetchfox-sdk 1.0.18 → 1.0.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",

package/src/api.js

@@ -29,6 +29,7 @@ export const call = async (method, path, params) => {
   };
 
   let url = endpoint(path, params);
+  console.log('Url', url);
   if (method == 'GET') {
     url += '?' + new URLSearchParams(params).toString();
   } else {

package/src/configure.js

@@ -1,6 +1,4 @@
-const config = {
-  host: 'https://api.fetchfox.ai',
-};
+const config = {};
 
 const isNode =
   typeof process !== 'undefined' &&
@@ -22,16 +20,12 @@ export const apiKey = (options) =>
   options?.apiKey || config.apiKey || safeEnv('FETCHFOX_API_KEY');
 
 export const host = (options) =>
-  options?.host || config.host || safeEnv('FETCHFOX_HOST');
+  options?.host ||
+  config.host ||
+  safeEnv('FETCHFOX_HOST') ||
+  'https://api.fetchfox.ai';
 
 export const appHost = (options) =>
-  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
-    'api.fetchfox.ai',
-    'app.fetchfox.ai'
-  );
+  host(options).replace('api.fetchfox.ai', 'app.fetchfox.ai');
 
-export const ws = (options) =>
-  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
-    'http',
-    'ws'
-  );
+export const ws = (options) => host(options).replace('http', 'ws');

package/src/detach.js

@@ -11,16 +11,20 @@ export function getSocket() {}
 export const Job = class {
   #callbacks;
   #socket;
+  #seen;
 
   constructor(id, options) {
     this.id = id;
     this.#callbacks = {
+      item: [],
       completed: [],
       error: [],
       finished: [],
       progress: [],
     };
 
+    this.#seen = {};
+
     this.#socket = new io(ws(options));
     this.#socket.on('progress', (data) => {
       this.handleProgress(data);
@@ -50,6 +54,15 @@ export const Job = class {
     ]) {
       s[key] = data[key] || this[key];
     }
+
+    if (s.progress?.children?.jobs) {
+      // const late = this.progress.children.jobs.filter(it => it.late);
+      // console.log('late jobs:', late);
+      s.progress.children.jobs = s.progress.children.jobs.filter(
+        (it) => !it.late
+      );
+    }
+
     return s;
   }
 
@@ -63,6 +76,8 @@ export const Job = class {
   }
 
   handleProgress(data) {
+    console.log('handleProgress', data);
+
     const last = JSON.stringify(this);
 
     const s = this.#select(data);
@@ -72,21 +87,34 @@ export const Job = class {
 
     const didUpdate = JSON.stringify(this) != last;
     if (didUpdate) {
-      this.trigger('progress');
+      this.trigger('progress', this);
+
+      for (const item of this.results?.items || []) {
+        const ser = JSON.stringify(item);
+        if (this.#seen[ser]) {
+          continue;
+        }
+        this.#seen[ser] = true;
+        this.trigger('item', item);
+      }
 
       if (this.state == 'completed') {
         this._completed = true;
-        this.trigger('completed');
+        this.trigger('completed', this);
       }
       if (this.state == 'error') {
         this._error = true;
-        this.trigger('error');
+        this.trigger('error', this);
       }
 
       if (['completed', 'error'].includes(this.state)) {
-        this.trigger('finished');
-        // Just in case there are some straggler events, wait a few seconds
-        setTimeout(() => this.#socket.disconnect(), 5000);
+        if (this.progress?.children?.jobs) {
+          this.progress.children.jobs = this.progress.children.jobs.filter(
+            (it) => it.state != 'active'
+          );
+        }
+        this.trigger('finished', this);
+        this.#socket.disconnect();
       }
     }
   }
@@ -97,10 +125,10 @@ export const Job = class {
     }
   }
 
-  trigger(event) {
+  trigger(event, data) {
     this.checkEvent(event);
     for (const cb of this.#callbacks[event]) {
-      cb({ ...this });
+      cb(data);
     }
   }
 

package/src/fetchfox.js

@@ -7,7 +7,7 @@ export const FetchFox = class {
     this.apiKey = apiKey;
     this.host = host;
 
-    const fns = [crawl, extract, scrape];
+    const fns = [visit, crawl, extract, scrape];
 
     for (const fn of fns) {
       this[fn.name] = function (args) {

package/src/index.js

@@ -1,11 +1,14 @@
-export * from './scrape.js';
+export * from './visit.js';
 export * from './crawl.js';
 export * from './extract.js';
+export * from './scrape.js';
+
 export * from './jobs.js';
 export * from './user.js';
 export * from './credits.js';
 export * from './proxy.js';
 export * from './urls.js';
+
 export { Job } from './detach.js';
 export { call } from './api.js';
 export { FetchFox } from './fetchfox.js';

package/src/visit.js

@@ -0,0 +1,11 @@
+import { call } from './api.js';
+import { Job } from './detach.js';
+
+export async function visit(args) {
+  return call('POST', '/api/visit', args);
+}
+
+visit.detach = async (args) => {
+  const data = await call('POST', '/api/visit', { ...args, detach: true });
+  return new Job(data.jobId, args);
+};

package/tests/fetchfox.test.js

@@ -37,18 +37,29 @@ test('use fetchfox object for extract detach @fetchfox @sanity', async () => {
     apiKey: process.env.FETCHFOX_API_KEY,
   });
   const job = await fox.extract.detach({
-    urls: ['https://pokemondb.net/pokedex/bulbasaur'],
+    urls: [
+      'https://pokemondb.net/pokedex/bulbasaur',
+      'https://pokemondb.net/pokedex/charmander',
+    ],
     template: 'name and number',
   });
 
   let count = 0;
+
   job.on('progress', (data) => {
     count++;
   });
 
+  let itemCount = 0;
+  job.on('item', (item) => {
+    itemCount++;
+    console.log('got item:', item);
+  });
+
   await job.finished();
 
   expect(count).toBeGreaterThan(0);
+  expect(itemCount).toBe(2);
 }, 30_000);
 
 test('invalid key fails @fetchfox @sanity', async () => {

package/src/#index.js#

@@ -1,13 +0,0 @@
-export * from './scrape.js';
-export * from './crawl.js';
-export * from './extract.js';
-export * from './jobs.js';
-export * from './user.js';
-export * from './credits.js';
-export * from './proxy.js';
-export * from './urls.js';
-
-export { Job } from './detach.js';
-export { call } from './api.js';
-export { FetchFox } from './fetchfox.js';
-export { configure, host, ws, apiKey } from './configure.js';