@suprsend/node-sdk 0.1.1 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suprsend/node-sdk",
-  "version": "0.1.1",
+  "version": "1.1.0",
   "description": "Suprsend Node SDK to trigger workflow from backend",
   "main": "dist/index.js",
   "scripts": {
@@ -26,6 +26,7 @@
     "@babel/runtime": "^7.16.3",
     "axios": "^0.27.2",
     "jsonschema": "^1.4.0",
+    "lodash": "^4.17.21",
     "mime-types": "^2.1.34",
     "uuid": "^8.3.2"
   },

package/src/attachment.js

@@ -0,0 +1,44 @@
+import path from "path";
+import mime from "mime-types";
+import { base64Encode, resolveTilde } from "./utils";
+
+function check_is_web_url(file_path = "") {
+  return file_path.startsWith("http://") || file_path.startsWith("https://");
+}
+
+function get_attachment_json_for_file(file_path, file_name, ignore_if_error) {
+  const abs_path = path.resolve(resolveTilde(file_path));
+  let final_file_name = path.basename(abs_path);
+  if (file_name && file_name.trim()) {
+    final_file_name = file_name.trim();
+  }
+  return {
+    filename: final_file_name,
+    contentType: mime.lookup(abs_path),
+    data: base64Encode(abs_path),
+    url: null,
+    ignore_if_error: ignore_if_error,
+  };
+}
+
+function get_attachment_json_for_url(file_url, file_name, ignore_if_error) {
+  return {
+    filename: file_name,
+    contentType: null,
+    data: null,
+    url: file_url,
+    ignore_if_error: ignore_if_error,
+  };
+}
+
+export default function get_attachment_json(
+  file_path,
+  file_name,
+  ignore_if_error = false
+) {
+  if (check_is_web_url(file_path)) {
+    return get_attachment_json_for_url(file_path, file_name, ignore_if_error);
+  } else {
+    return get_attachment_json_for_file(file_path, file_name, ignore_if_error);
+  }
+}

package/src/bulk_response.js

@@ -0,0 +1,35 @@
+export default class BulkResponse {
+  constructor() {
+    this.status;
+    this.failed_records = [];
+    this.total = 0;
+    this.success = 0;
+    this.failure = 0;
+    this.warnings = [];
+  }
+
+  merge_chunk_response(ch_resp) {
+    if (!ch_resp) {
+      return;
+    }
+    // possible status: success/partial/fail
+    if (!this.status) {
+      this.status = ch_resp["status"];
+    } else {
+      if (this.status === "success") {
+        if (ch_resp.status === "fail") {
+          this.status = "partial";
+        }
+      } else if (this.status === "fail") {
+        if (ch_resp.status === "success") {
+          this.status = "partial";
+        }
+      }
+    }
+    this.total += ch_resp.total || 0;
+    this.success += ch_resp.success || 0;
+    this.failure += ch_resp.failure || 0;
+    const failed_recs = ch_resp.failed_records || [];
+    this.failed_records = [...this.failed_records, ...failed_recs];
+  }
+}

package/src/constants.js

@@ -0,0 +1,32 @@
+//  Default urls
+export const DEFAULT_URL = "https://hub.suprsend.com/";
+export const DEFAULT_UAT_URL =
+  "https://collector-staging.suprsend.workers.dev/";
+
+//  an Event should not have apparent body size of more than 100KB
+export const SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES = 100 * 1024; // 100 * 1024
+export const SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE = "100KB";
+
+// a API call should not have apparent body size of more than 800KB
+export const BODY_MAX_APPARENT_SIZE_IN_BYTES = 800 * 1024; // 800 * 1024
+export const BODY_MAX_APPARENT_SIZE_IN_BYTES_READABLE = "800KB";
+
+// in general url-size wont exceed 2048 chars or 2048 utf-8 bytes
+export const ATTACHMENT_URL_POTENTIAL_SIZE_IN_BYTES = 2100;
+
+// few keys added in-flight, amounting to almost 200 bytes increase per workflow-body
+export const WORKFLOW_RUNTIME_KEYS_POTENTIAL_SIZE_IN_BYTES = 200;
+
+// max workflow-records in one bulk api call.
+export const MAX_WORKFLOWS_IN_BULK_API = 100;
+// max event-records in one bulk api call
+export const MAX_EVENTS_IN_BULK_API = 100;
+
+export const ALLOW_ATTACHMENTS_IN_BULK_API = true;
+export const ATTACHMENT_UPLOAD_ENABLED = false;
+
+// -- single Identity event limit
+export const IDENTITY_SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES = 2 * 1024;
+export const IDENTITY_SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE = "2KB";
+
+export const MAX_IDENTITY_EVENTS_IN_BULK_API = 400;

package/src/event.js

@@ -5,12 +5,16 @@ import {
   has_special_char,
   uuid,
   epoch_milliseconds,
+  validate_track_event_schema,
+  get_apparent_event_size,
 } from "./utils";
 import get_request_signature from "./signature";
-import { Validator } from "jsonschema";
 import axios from "axios";
-
-const event_schema = require("./request_json/event.json");
+import get_attachment_json from "./attachment";
+import {
+  SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES,
+  SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE,
+} from "./constants";
 
 const RESERVED_EVENT_NAMES = [
   "$identify",
@@ -22,12 +26,103 @@ const RESERVED_EVENT_NAMES = [
   "$user_logout",
 ];
 
-class EventCollector {
+export default class Event {
+  constructor(distinct_id, event_name, properties, kwargs = {}) {
+    this.distinct_id = distinct_id;
+    this.event_name = event_name;
+    this.properties = properties;
+    this.idempotency_key = kwargs?.idempotency_key;
+    // --- validate
+    this.__validate_distinct_id();
+    this.__validate_event_name();
+    this.__validate_properties();
+  }
+
+  __validate_distinct_id() {
+    if (this.distinct_id instanceof String) {
+      throw new SuprsendError(
+        "distinct_id must be a string. an Id which uniquely identify a user in your app"
+      );
+    }
+    const distinct_id = this.distinct_id.trim();
+    if (!distinct_id) {
+      throw new SuprsendError("distinct_id missing");
+    }
+    this.distinct_id = distinct_id;
+  }
+
+  __validate_properties() {
+    if (!this.properties) {
+      this.properties = {};
+    }
+    if (!(this.properties instanceof Object)) {
+      throw new SuprsendError("properties must be a dictionary");
+    }
+  }
+
+  __check_event_prefix(event_name) {
+    if (!RESERVED_EVENT_NAMES.includes(event_name)) {
+      if (has_special_char(event_name)) {
+        throw new SuprsendError(
+          "event_names starting with [$,ss_] are reserved by SuprSend"
+        );
+      }
+    }
+  }
+
+  __validate_event_name() {
+    if (!is_string(this.event_name)) {
+      throw new SuprsendError("event_name must be a string");
+    }
+    const event_name = this.event_name.trim();
+    this.__check_event_prefix(event_name);
+    this.event_name = event_name;
+  }
+
+  add_attachment(file_path, kwargs = {}) {
+    const file_name = kwargs?.file_name;
+    const ignore_if_error = kwargs?.ignore_if_error ?? false;
+    const attachment = get_attachment_json(
+      file_path,
+      file_name,
+      ignore_if_error
+    );
+    // --- add the attachment to properties->$attachments
+    if (!this.properties["$attachments"]) {
+      this.properties["$attachments"] = [];
+    }
+    this.properties["$attachments"].push(attachment);
+  }
+
+  get_final_json(config, is_part_of_bulk = false) {
+    const super_props = { $ss_sdk_version: config.user_agent };
+    let event_dict = {
+      $insert_id: uuid(),
+      $time: epoch_milliseconds(),
+      event: this.event_name,
+      env: config.workspace_key,
+      distinct_id: this.distinct_id,
+      properties: { ...this.properties, ...super_props },
+    };
+    if (this.idempotency_key) {
+      event_dict["$idempotency_key"] = this.idempotency_key;
+    }
+    event_dict = validate_track_event_schema(event_dict);
+    const apparent_size = get_apparent_event_size(event_dict, is_part_of_bulk);
+    if (apparent_size > SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES) {
+      throw new SuprsendError(
+        `Event size too big - ${apparent_size} Bytes,must not cross ${SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE}`
+      );
+    }
+    return [event_dict, apparent_size];
+  }
+}
+
+export class EventCollector {
   constructor(config) {
     this.config = config;
     this.__url = this.__get_url();
     this.__headers = this.__common_headers();
-    this.__supr_props = this.__super_properties();
   }
 
   __get_url() {
@@ -54,63 +149,9 @@ class EventCollector {
     };
   }
 
-  __super_properties() {
-    return {
-      $ss_sdk_version: this.config.user_agent,
-    };
-  }
-
-  __check_event_prefix(event_name) {
-    if (!RESERVED_EVENT_NAMES.includes(event_name)) {
-      if (has_special_char(event_name)) {
-        throw new SuprsendError(
-          "event_names starting with [$,ss_] are reserved"
-        );
-      }
-    }
-  }
-
-  __validate_event_name(event_name) {
-    if (!is_string(event_name)) {
-      throw new SuprsendError("event_name must be a string");
-    }
-    event_name = event_name.trim();
-    this.__check_event_prefix(event_name);
-    return event_name;
-  }
-
-  validate_track_event_schema(data) {
-    if (!data["properties"]) {
-      data["properties"] = {};
-    }
-    const schema = event_schema;
-    var v = new Validator();
-    const validated_data = v.validate(data, schema);
-    if (validated_data.valid) {
-      return data;
-    } else {
-      const error_obj = validated_data.errors[0];
-      const error_msg = `${error_obj.property} ${error_obj.message}`;
-      throw new SuprsendError(error_msg);
-    }
-  }
-
-  collect(distinct_id, event_name, properties = {}) {
-    event_name = this.__validate_event_name(event_name);
-    if (!is_object(properties)) {
-      throw new SuprsendError("properties must be a dictionary");
-    }
-    properties = { ...properties, ...this.__supr_props };
-    let event = {
-      $insert_id: uuid(),
-      $time: epoch_milliseconds(),
-      event: event_name,
-      env: this.config.env_key,
-      distinct_id: distinct_id,
-      properties: properties,
-    };
-    event = this.validate_track_event_schema(event);
-    return this.send(event);
+  collect(event) {
+    const [event_dict, event_size] = event.get_final_json(this.config, false);
+    return this.send(event_dict);
   }
 
   async send(event) {
@@ -122,27 +163,35 @@ class EventCollector {
         "POST",
         content_text,
         headers,
-        this.config.env_secret
+        this.config.workspace_secret
       );
-      headers["Authorization"] = `${this.config.env_key}:${signature}`;
+      headers["Authorization"] = `${this.config.workspace_key}:${signature}`;
     }
     try {
-      const response = await axios.post(this.__url, content_text, {
-        headers,
-      });
-      return {
-        status_code: response.status,
-        success: true,
-        message: response.statusText,
-      };
+      const response = await axios.post(this.__url, content_text, { headers });
+      const ok_response = Math.floor(response.status / 100) == 2;
+      if (ok_response) {
+        return {
+          success: true,
+          status: "success",
+          status_code: response.status,
+          message: response.statusText,
+        };
+      } else {
+        return {
+          success: false,
+          status: "fail",
+          status_code: response.status,
+          message: response.statusText,
+        };
+      }
     } catch (err) {
       return {
-        status_code: 400,
         success: false,
+        status: "fail",
+        status_code: err.status || 500,
         message: err.message,
       };
     }
   }
 }
-
-export default EventCollector;

package/src/events_bulk.js

@@ -0,0 +1,236 @@
+import {
+  SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES,
+  SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE,
+  ALLOW_ATTACHMENTS_IN_BULK_API,
+  BODY_MAX_APPARENT_SIZE_IN_BYTES,
+  BODY_MAX_APPARENT_SIZE_IN_BYTES_READABLE,
+  MAX_EVENTS_IN_BULK_API,
+} from "./constants";
+import get_request_signature from "./signature";
+import BulkResponse from "./bulk_response";
+import Event from "./event";
+import { SuprsendError } from "./utils";
+import { cloneDeep } from "lodash";
+import axios from "axios";
+
+export class BulkEventsFactory {
+  constructor(config) {
+    this.config = config;
+  }
+
+  new_instance() {
+    return new BulkEvents(this.config);
+  }
+}
+
+class _BulkEventsChunk {
+  constructor(config) {
+    this.config = config;
+    this.__chunk = [];
+    this.__url = this.__get_url();
+    this.__headers = this.__common_headers();
+
+    this.__running_size = 0;
+    this.__running_length = 0;
+    this.response;
+  }
+
+  __get_url() {
+    let url_template = "event/";
+    if (this.config.include_signature_param) {
+      if (this.config.auth_enabled) {
+        url_template = url_template + "?verify=true";
+      } else {
+        url_template = url_template + "?verify=false";
+      }
+    }
+    const url_formatted = `${this.config.base_url}${url_template}`;
+    return url_formatted;
+  }
+
+  __common_headers() {
+    return {
+      "Content-Type": "application/json; charset=utf-8",
+      "User-Agent": this.config.user_agent,
+    };
+  }
+
+  __dynamic_headers() {
+    return {
+      Date: new Date().toUTCString(),
+    };
+  }
+
+  __add_event_to_chunk(event, event_size) {
+    //  First add size, then body to reduce effects of race condition
+    this.__running_size += event_size;
+    this.__chunk.push(event);
+    this.__running_length += 1;
+  }
+
+  __check_limit_reached() {
+    if (
+      this.__running_length >= MAX_EVENTS_IN_BULK_API ||
+      this.__running_size >= BODY_MAX_APPARENT_SIZE_IN_BYTES
+    ) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  try_to_add_into_chunk(event, event_size) {
+    if (!event) {
+      return true;
+    }
+    if (this.__check_limit_reached()) {
+      return false;
+    }
+    if (event_size > SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES) {
+      throw new SuprsendError(
+        `Event properties too big - ${event_size} Bytes, must not cross ${SINGLE_EVENT_MAX_APPARENT_SIZE_IN_BYTES_READABLE}`
+      );
+    }
+    if (this.__running_size + event_size > BODY_MAX_APPARENT_SIZE_IN_BYTES) {
+      return false;
+    }
+    if (!ALLOW_ATTACHMENTS_IN_BULK_API) {
+      delete event.properties["$attachments"];
+    }
+
+    this.__add_event_to_chunk(event, event_size);
+    return true;
+  }
+
+  async trigger() {
+    const headers = { ...this.__headers, ...this.__dynamic_headers() };
+    const content_text = JSON.stringify(this.__chunk);
+    // Based on whether signature is required or not, add Authorization header
+    if (this.config.auth_enabled) {
+      const signature = get_request_signature(
+        this.__url,
+        "POST",
+        content_text,
+        headers,
+        this.config.workspace_secret
+      );
+      headers["Authorization"] = `${this.config.workspace_key}:${signature}`;
+    }
+    try {
+      const response = await axios.post(this.__url, content_text, { headers });
+      const ok_response = Math.floor(response.status / 100) == 2;
+      if (ok_response) {
+        this.response = {
+          status: "success",
+          status_code: response.status,
+          total: this.__chunk.length,
+          success: this.__chunk.length,
+          failure: 0,
+          failed_records: [],
+        };
+      } else {
+        this.response = {
+          status: "fail",
+          status_code: response.status,
+          total: this.__chunk.length,
+          success: 0,
+          failure: this.__chunk.length,
+          failed_records: this.__chunk.map((item) => ({
+            record: item,
+            error: response.statusText,
+            code: response.status,
+          })),
+        };
+      }
+    } catch (err) {
+      const error_status = err.status || 500;
+      return {
+        status: "fail",
+        status_code: error_status,
+        message: err.message,
+        total: this.__chunk.length,
+        success: 0,
+        failure: this.__chunk.length,
+        failed_records: this.__chunk.map((item) => ({
+          record: item,
+          error: err.message,
+          code: error_status,
+        })),
+      };
+    }
+  }
+}
+
+class BulkEvents {
+  constructor(config) {
+    this.config = config;
+    this.__events = [];
+    this.__pending_records = [];
+    this.chunks = [];
+    this.response = new BulkResponse();
+  }
+
+  __validate_events() {
+    if (!this.__events) {
+      throw new SuprsendError("events list is empty in bulk request");
+    }
+    for (let ev of this.__events) {
+      const is_part_of_bulk = true;
+      const [ev_json, body_size] = ev.get_final_json(
+        this.config,
+        is_part_of_bulk
+      );
+      this.__pending_records.push([ev_json, body_size]);
+    }
+  }
+
+  __chunkify(start_idx = 0) {
+    const curr_chunk = new _BulkEventsChunk(this.config);
+    this.chunks.push(curr_chunk);
+    const entries = this.__pending_records.slice(start_idx).entries();
+    for (const [rel_idx, rec] of entries) {
+      const is_added = curr_chunk.try_to_add_into_chunk(rec[0], rec[1]);
+      if (!is_added) {
+        // create chunks from remaining records
+        this.__chunkify(start_idx + rel_idx);
+        // Don't forget to break. As current loop must not continue further
+        break;
+      }
+    }
+  }
+
+  append(...events) {
+    if (!events) {
+      throw new SuprsendError(
+        "events list empty. must pass one or more events"
+      );
+    }
+    for (let ev of events) {
+      if (!ev) {
+        throw new SuprsendError("null/empty element found in bulk instance");
+      }
+      if (!(ev instanceof Event)) {
+        throw new SuprsendError(
+          "element must be an instance of suprsend.Event"
+        );
+      }
+      const ev_copy = cloneDeep(ev);
+      this.__events.push(ev_copy);
+    }
+  }
+
+  async trigger() {
+    this.__validate_events();
+    this.__chunkify();
+    for (const [c_idx, ch] of this.chunks.entries()) {
+      if (this.config.req_log_level > 0) {
+        console.log(`DEBUG: triggering api call for chunk: ${c_idx}`);
+      }
+      // do api call
+      await ch.trigger();
+      // merge response
+      this.response.merge_chunk_response(ch.response);
+    }
+    return this.response;
+  }
+}