fetchfox-sdk 1.0.17 → 1.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",
@@ -11,7 +11,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/fetchfox/fetchfox.git"
+    "url": "git+https://github.com/fetchfox/fetchfox-sdk.git"
   },
   "keywords": [
     "ai",
@@ -20,9 +20,9 @@
   "author": "marcell@fetchfoxai.com",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/fetchfox/fetchfox/issues"
+    "url": "https://github.com/fetchfox/fetchfox-sdk/issues"
   },
-  "homepage": "https://github.com/fetchfox/fetchfox#readme",
+  "homepage": "https://fetchfox.ai",
   "devDependencies": {
     "@eslint/js": "^9.31.0",
     "eslint-plugin-promise": "^7.2.1",

package/src/api.js

@@ -29,6 +29,7 @@ export const call = async (method, path, params) => {
   };
 
   let url = endpoint(path, params);
+  console.log('Url', url);
   if (method == 'GET') {
     url += '?' + new URLSearchParams(params).toString();
   } else {

package/src/configure.js

@@ -1,6 +1,4 @@
-const config = {
-  host: 'https://api.fetchfox.ai',
-};
+const config = {};
 
 const isNode =
   typeof process !== 'undefined' &&
@@ -22,16 +20,12 @@ export const apiKey = (options) =>
   options?.apiKey || config.apiKey || safeEnv('FETCHFOX_API_KEY');
 
 export const host = (options) =>
-  options?.host || config.host || safeEnv('FETCHFOX_HOST');
+  options?.host ||
+  config.host ||
+  safeEnv('FETCHFOX_HOST') ||
+  'https://api.fetchfox.ai';
 
 export const appHost = (options) =>
-  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
-    'api.fetchfox.ai',
-    'app.fetchfox.ai'
-  );
+  host(options).replace('api.fetchfox.ai', 'app.fetchfox.ai');
 
-export const ws = (options) =>
-  (options?.host || config.host || safeEnv('FETCHFOX_HOST')).replace(
-    'http',
-    'ws'
-  );
+export const ws = (options) => host(options).replace('http', 'ws');

package/src/crawl.js

@@ -7,5 +7,5 @@ export async function crawl(args) {
 
 crawl.detach = async (args) => {
   const data = await call('POST', '/api/crawl', { ...args, detach: true });
-  return new Job(data.jobId);
+  return new Job(data.jobId, args);
 };

package/src/detach.js

@@ -11,17 +11,21 @@ export function getSocket() {}
 export const Job = class {
   #callbacks;
   #socket;
+  #seen;
 
-  constructor(id) {
+  constructor(id, options) {
     this.id = id;
     this.#callbacks = {
+      item: [],
       completed: [],
       error: [],
       finished: [],
       progress: [],
     };
 
-    this.#socket = new io(ws());
+    this.#seen = {};
+
+    this.#socket = new io(ws(options));
     this.#socket.on('progress', (data) => {
       this.handleProgress(data);
     });
@@ -50,6 +54,15 @@ export const Job = class {
     ]) {
       s[key] = data[key] || this[key];
     }
+
+    if (s.progress?.children?.jobs) {
+      // const late = this.progress.children.jobs.filter(it => it.late);
+      // console.log('late jobs:', late);
+      s.progress.children.jobs = s.progress.children.jobs.filter(
+        (it) => !it.late
+      );
+    }
+
     return s;
   }
 
@@ -63,6 +76,8 @@ export const Job = class {
   }
 
   handleProgress(data) {
+    console.log('handleProgress', data);
+
     const last = JSON.stringify(this);
 
     const s = this.#select(data);
@@ -72,21 +87,34 @@ export const Job = class {
 
     const didUpdate = JSON.stringify(this) != last;
     if (didUpdate) {
-      this.trigger('progress');
+      this.trigger('progress', this);
+
+      for (const item of this.results?.items || []) {
+        const ser = JSON.stringify(item);
+        if (this.#seen[ser]) {
+          continue;
+        }
+        this.#seen[ser] = true;
+        this.trigger('item', item);
+      }
 
       if (this.state == 'completed') {
         this._completed = true;
-        this.trigger('completed');
+        this.trigger('completed', this);
       }
       if (this.state == 'error') {
         this._error = true;
-        this.trigger('error');
+        this.trigger('error', this);
       }
 
       if (['completed', 'error'].includes(this.state)) {
-        this.trigger('finished');
-        // Just in case there are some straggler events, wait a few seconds
-        setTimeout(() => this.#socket.disconnect(), 5000);
+        if (this.progress?.children?.jobs) {
+          this.progress.children.jobs = this.progress.children.jobs.filter(
+            (it) => it.state != 'active'
+          );
+        }
+        this.trigger('finished', this);
+        this.#socket.disconnect();
       }
     }
   }
@@ -97,10 +125,10 @@ export const Job = class {
     }
   }
 
-  trigger(event) {
+  trigger(event, data) {
     this.checkEvent(event);
     for (const cb of this.#callbacks[event]) {
-      cb({ ...this });
+      cb(data);
     }
   }
 

package/src/extract.js

@@ -7,5 +7,5 @@ export async function extract(args) {
 
 extract.detach = async (args) => {
   const data = await call('POST', '/api/extract', { ...args, detach: true });
-  return new Job(data.jobId);
+  return new Job(data.jobId, args);
 };

package/src/scrape.js

@@ -7,5 +7,5 @@ export async function scrape(args) {
 
 scrape.detach = async (args) => {
   const data = await call('POST', '/api/scrape', { ...args, detach: true });
-  return new Job(data.jobId);
+  return new Job(data.jobId, args);
 };

package/tests/fetchfox.test.js

@@ -32,6 +32,36 @@ test('use fetchfox object for detach @fetchfox @sanity', async () => {
   expect(count).toBeGreaterThan(0);
 }, 30_000);
 
+test('use fetchfox object for extract detach @fetchfox @sanity', async () => {
+  const fox = new FetchFox({
+    apiKey: process.env.FETCHFOX_API_KEY,
+  });
+  const job = await fox.extract.detach({
+    urls: [
+      'https://pokemondb.net/pokedex/bulbasaur',
+      'https://pokemondb.net/pokedex/charmander',
+    ],
+    template: 'name and number',
+  });
+
+  let count = 0;
+
+  job.on('progress', (data) => {
+    count++;
+  });
+
+  let itemCount = 0;
+  job.on('item', (item) => {
+    itemCount++;
+    console.log('got item:', item);
+  });
+
+  await job.finished();
+
+  expect(count).toBeGreaterThan(0);
+  expect(itemCount).toBe(2);
+}, 30_000);
+
 test('invalid key fails @fetchfox @sanity', async () => {
   const fox = new FetchFox({
     apiKey: 'invalid',