@miso.ai/server-sdk 0.6.5-beta.6 → 0.6.5-beta.7

package/package.json

@@ -16,12 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.5-beta.6",
+    "@miso.ai/server-commons": "0.6.5-beta.7",
     "axios": "^1.6.2",
     "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.6"
+  "version": "0.6.5-beta.7"
 }

package/src/api/helpers.js

@@ -2,9 +2,44 @@ import { asArray, trimObj, computeIfAbsent } from '@miso.ai/server-commons';
 import { Buffer } from 'buffer';
 
 export async function upload(client, type, records, options = {}) {
-  const url = buildUrl(client, type, { ...options, async: true });
+  const url = buildUrl(client, type, options);
   const payload = buildUploadPayload(records);
-  return client._axios.post(url, payload);
+  try {
+    // await to catch errors
+    return await client._axios.post(url, payload);
+  } catch (error) {
+    await recoverValidRecords(client, type, records, options, error.response);
+    throw error;
+  }
+}
+
+async function recoverValidRecords(client, type, records, options, response) {
+  if (!response || response.status !== 422 || !options.recoverValidRecordsOn422) {
+    return;
+  }
+  records = extractRecordsFromUploadPayload(records);
+  // try to collect valid records and resend them, which should pass the validation
+  const { groups = [], unrecognized = [] } = process422ResponseBody(records, response.data); // it takes records too
+  if (groups.length === 0 || groups.length === records.length || unrecognized.length > 0) {
+    // if there are unrecognized messages, it's hard to tell which records are valid
+    return;
+  }
+  const invalidIndices = new Set(groups.map(group => group.index));
+  const validRecords = records.filter((_, index) => !invalidIndices.has(index));
+  if (validRecords.length === 0) {
+    return; // should not be, just in case
+  }
+  const url = buildUrl(client, type, options);
+  const validPayload = buildUploadPayload(validRecords);
+  try {
+    await client._axios.post(url, validPayload);
+  } catch (_) {
+    return; // still fail, never mind...
+  }
+  response.recovered = {
+    records: validRecords.length,
+    bytes: validPayload.length * 2,
+  };
 }
 
 export async function merge(client, type, record, { mergeFn = defaultMerge } = {}) {
@@ -61,10 +96,10 @@ export function shimRecordForMerging(record) {
   return record;
 }
 
-const RE_422_MSG_LINE = /^\s*data\.(\d+)(?:\.(\S+))?\s+(?:\(([^)]*)\))?\s+is\s+invalid\.\s+(.*)$/;
+const RE_422_MSG_LINE = /^\s*data\.(\d+)(?:\.(\S+))?\s+(?:\(([^)]*)\)\s+)?is\s+invalid\.\s+(.*)$/;
 
 export function process422ResponseBody(payload, { data } = {}) {
-  const records = extractUploadPayload(payload);
+  const records = extractRecordsFromUploadPayload(payload);
   const unrecognized = [];
   const groupsMap = new Map();
   for (const line of data) {
@@ -122,10 +157,10 @@ export function buildUrl(client, path, { async, dryRun, params: extraParams } =
 export function buildUploadPayload(records) {
   return typeof records === 'string' ? records :
     Buffer.isBuffer(records) ? records.toString() :
-    { data: Array.isArray(records) ? records : [records] };
+    JSON.stringify({ data: Array.isArray(records) ? records : [records] });
 }
 
-export function extractUploadPayload(records) {
+export function extractRecordsFromUploadPayload(records) {
   if (Buffer.isBuffer(records)) {
     records = records.toString();
   }

package/src/stream/api-sink.js

@@ -52,9 +52,13 @@ export default class ApiSink extends sink.BpsSink {
       } else if (status) {
         data = { status, ...data };
       }
+      if (error.response.recovered) {
+        data.recovered = error.response.recovered;
+      }
     }
 
     // keep track of service stats on successful calls
+    // TODO: handle recovered records?
     if (!data.errors) {
       this._serviceStats.track({ records, bytes, took: data.took });
     }

package/src/stream/upload-sink.js

@@ -22,7 +22,7 @@ class UploadSink extends ApiSink {
 
   async _execute(payload) {
     const { type, dryRun, params } = this._options;
-    const { data } = await upload(this._client, type, payload, { dryRun, params });
+    const { data } = await upload(this._client, type, payload, { recoverValidRecordsOn422: true, dryRun, params });
     return data;
   }
 

package/src/stream/upload.js

@@ -65,12 +65,13 @@ export default class UploadStream extends stream.BufferedWriteStream {
 
     // if upload fails, emit extracted payload at response event
     if (message.event === 'response') {
+      // TODO: we can do these near recoverValidRecords()
       const { response, payload } = args;
       if (payload) {
         output.payload = JSON.parse(payload);
-      }
-      if (response && response.status === 422) {
-        output.issues = process422ResponseBody(payload, response);
+        if (response && response.status === 422) {
+          output.issues = process422ResponseBody(payload, response);
+        }
       }
     }
 

package/src/version.js

@@ -1 +1 @@
-export default '0.6.5-beta.6';
+export default '0.6.5-beta.7';