fetchfox-sdk 1.0.2 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetchfox-sdk",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "AI scraper",
   "main": "src/index.js",
   "type": "module",

package/src/api.js

@@ -1,31 +1,24 @@
 import { apiKey, host } from './configure.js';
 
-const endpoint = (path) => `${host()}${path}`;
-
-const clean = (dict) => {
-  const result = {};
-
-  for (const key in dict) {
-    if (dict.hasOwnProperty(key)) {
-      const snakeKey = key
-        .replace(/([A-Z])/g, '_$1') // insert _ before uppercase letters
-        .toLowerCase() // convert all to lowercase
-        .replace(/^_/, ''); // remove leading _ if any
+const camelCase = (obj) => {
+  const snakeToCamel = (str) =>
+    str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
+  return Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [snakeToCamel(key), value])
+  );
+};
 
-      result[snakeKey] = dict[key];
-    }
-  }
+const endpoint = (path) => `${host()}${path}`;
 
-  if (result.apiKey) {
-    delete result.apiKey;
+const FetchFoxAPIError = class extends Error {
+  constructor(errors) {
+    super(JSON.stringify(errors));
+    this.errors = errors;
   }
-
-  return result;
-};
+}
 
 export const call = async (method, path, params) => {
   const key = apiKey(params);
-  params = clean(params);
 
   const args = {
     method,
@@ -43,5 +36,24 @@ export const call = async (method, path, params) => {
   }
 
   const resp = await fetch(url, args);
-  return resp.json();
+  const text = await resp.text();
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (e) {
+    throw new Error(`FetchFox returned invalid JSON: ${text}`);
+  }
+
+  if (data.errors) {
+    throw new FetchFoxAPIError(data.errors);
+  }
+
+  if (resp.status >= 400) {
+    throw new FetchFoxAPIError({
+      status: `Received status=${resp.status}`,
+      ...data,
+    })
+  }
+
+  return camelCase(data);
 };

package/src/crawl.js

@@ -1,5 +1,11 @@
 import { call } from './api.js';
+import { Job } from './detach.js';
 
-export const crawl = async (args) => {
+export const crawl = async function () {
   return call('POST', '/api/crawl', args);
-};
+}
+crawl.detach = async (args) => {
+  const data = await call('POST', '/api/crawl', { ...args, detach: true });
+  console.log('detach data', data);
+  return new Job(data.jobId);
+}

package/src/detach.js

@@ -0,0 +1,120 @@
+import { jobs } from './jobs.js';
+
+class FetchFoxError extends Error {};
+
+const interval = 1_000;
+
+export const Job = class {
+  #callbacks;
+
+  constructor(id) {
+    this.id = id;
+    this.#callbacks = {
+      completed: [],
+      error: [],
+      finished: [],
+      progress: [],
+    };
+
+    this.poll();
+  }
+
+  get _finished() {
+    return this._completed || this._error;
+  }
+
+  #select(data) {
+    return {
+      name: data.name,
+      state: data.state,
+      args: data.args,
+      metrics: data.metrics,
+      progress: data.progress,
+      results: data.results,
+      artifacts: data.artifacts,
+    };
+  }
+
+  async get() {
+    const data = await jobs.get(this.id);
+    const s = this.#select(data);
+    for (const key of Object.keys(s)) {
+      this[key] = s[key];
+    }
+    return this;
+  }
+
+  async poll() {
+    const last = JSON.stringify(this);
+    await this.get();
+    const didUpdate = JSON.stringify(this) != last;
+    if (didUpdate) {
+      console.log('Job progressed:', this);
+      this.trigger('progress');
+
+      if (this.state == 'completed') {
+        this._completed = true;
+        this.trigger('completed');
+      }
+      if (this.state == 'error') {
+        this._error = true;
+        this.trigger('error');
+      }
+
+      if (['completed', 'error'].includes(this.state)) {
+        this.trigger('finished');
+      }
+    }
+
+    if (!this._finished) {
+      setTimeout(() => this.poll(), interval);
+    }
+  }
+
+  checkEvent(event) {
+    if (!this.#callbacks[event]) {
+      throw new FetchFoxError(`Invalid event: ${event}`);
+    }
+  }
+
+  trigger(event) {
+    console.log('Trigger:', this.id, event);
+
+    this.checkEvent(event);
+    for (const cb of this.#callbacks[event]) {
+      cb({ ...this });
+    }
+  }
+
+  on(event, cb) {
+    this.checkEvent(event);
+    this.#callbacks[event].push(cb);
+  }
+
+  off(event, cb) {
+    this.checkEvent(event);
+    this.#callbacks[event] = this.#callbacks[event].filter(it => it != cb);
+  }
+
+  async waitFor(event) {
+    this.checkEvent(event);
+    return new Promise((ok) => {
+      this.on(event, () => {
+        this.off(event, ok);
+        ok({ ...this });
+      });
+    });
+  }
+
+  async completed() {
+    return this.waitFor('completed');
+  }
+
+  async error() {
+    return this.waitFor('error');
+  }
+
+  async finished() {
+    return this.waitFor('finished');
+  }
+}

package/src/index.js

@@ -4,5 +4,6 @@ export * from './extract.js';
 export * from './jobs.js';
 export * from './user.js';
 export * from './proxy.js';
+export { Job } from './detach.js';
 export { call } from './api.js';
 export { configure } from './configure.js';