@foxtware/mineral 0.1.9 → 0.1.11

package/api/peoplevox/peoplevox.constants.js

@@ -1,3 +1,3 @@
 module.exports = {
   MAX_REQUEST_ITEMS: 100,
-};
+};

package/api/peoplevox/peoplevox.sessions.js

@@ -48,40 +48,7 @@ const setSessionId = async (credsPayload, sessionId) => {
   SESSION_IDS.set(CLIENT_ID, sessionId);
 };
 
-const withPeoplevoxAuth = async (fetchPayload, next) => {
-  const { context = {} } = fetchPayload;
-  const { credsPayload } = context;
-
-  const sessionIdResponse = await getSessionId(credsPayload);
-
-  const { 
-    ok: sessionIdOk, 
-    data: sessionId,
-  } = sessionIdResponse;
-
-  if (!sessionIdOk) {
-    return sessionIdResponse;
-  }
-
-  const response = await next({
-    ...fetchPayload,
-    context: {
-      ...context,
-      sessionId,
-    },
-  });
-
-  if (response.ok) {
-    await setSessionId(credsPayload, sessionId);
-    return response;
-  }
-  
-  // TODO: Check if it was an auth error, and try another auth method if so
-  return response;
-};
-
 module.exports = {
   getSessionId,
   setSessionId,
-  withPeoplevoxAuth,
 };

package/api/peoplevox/peoplevox.utils.js

@@ -1,7 +1,12 @@
 const csvtojson = require('csvtojson');
 const xml2js = require('xml2js');
-const { FetchClient, credsFromPayload, appendUrlToBase, logDeep, Chain } = require('../utils');
-const { withPeoplevoxAuth } = require('../peoplevox/peoplevox.sessions');
+const { resolveCreds } = require('../pipelineSteps');
+const {
+  FetchClient,
+  appendUrlToBase,
+  logDeep,
+} = require('../utils');
+const { getSessionId, setSessionId } = require('../peoplevox/peoplevox.sessions');
 
 const xml2jsBuilder = new xml2js.Builder({
   headless: true,
@@ -45,17 +50,17 @@ const buildSoapEnvelope = ({
   return xml2jsBuilder.buildObject(envelopeObject);
 };
 
-// TODO: Split into multiple steps, allow mutating context
-const peoplevoxRequestPreparer = async (requestPayload, context) => {
-  const { headers, body } = requestPayload;
-  const { credsPayload, action, sessionId } = context;
-  const { CLIENT_ID } = await credsFromPayload(credsPayload);
+const useSoapEnvelope = async (state) => {
+  const { requestPayload, context } = state;
+  const { creds, action, sessionId } = context;
+  const { CLIENT_ID } = creds;
 
   if (!sessionId) {
     throw new Error('PeopleVox sessionId is required');
   }
 
   const baseUrl = `https://ap.peoplevox.net/${ CLIENT_ID }/Resources/IntegrationServicev4.asmx`;
+  const { headers, body } = requestPayload;
 
   const envelopeXml = buildSoapEnvelope({
     action,
@@ -67,14 +72,16 @@ const peoplevoxRequestPreparer = async (requestPayload, context) => {
   logDeep({ envelopeXml });
 
   return {
-    ...requestPayload,
-    url: appendUrlToBase(baseUrl, requestPayload.url),
-    headers: {
-      ...headers,
-      'Content-Type': 'text/xml; charset=utf-8',
-      'SOAPAction': `http://www.peoplevox.net/${ action }`,
+    requestPayload: {
+      ...requestPayload,
+      url: appendUrlToBase(baseUrl, requestPayload.url),
+      headers: {
+        ...headers,
+        'Content-Type': 'text/xml; charset=utf-8',
+        'SOAPAction': `http://www.peoplevox.net/${ action }`,
+      },
+      body: envelopeXml,
     },
-    body: envelopeXml,
   };
 };
 
@@ -167,17 +174,54 @@ const hoistDetail = (state) => {
 };
 
 const peoplevoxClient = new FetchClient({
-  requestPreparer: peoplevoxRequestPreparer,
-  responseInterpreter: new Chain([
+  pipeline: [
+    resolveCreds,
+    useSoapEnvelope,
+    'fetch',
     stripEnvelope,
     tryToParseDetailAsCsv,
     unwrapSingleDetail,
     hoistDetail,
-  ]),
+  ],
 });
 
-peoplevoxClient.use(withPeoplevoxAuth);
+const fetchWithoutAuth = peoplevoxClient.fetch.bind(peoplevoxClient);
+
+peoplevoxClient.fetch = async ({
+  requestPayload,
+  context = {},
+  inspect = false,
+}) => {
+  const { credsPayload } = context;
+
+  const sessionIdResponse = await getSessionId(credsPayload);
+
+  const {
+    ok: sessionIdOk,
+    data: sessionId,
+  } = sessionIdResponse;
+
+  if (!sessionIdOk) {
+    return sessionIdResponse;
+  }
+
+  const response = await fetchWithoutAuth({
+    requestPayload,
+    context: {
+      ...context,
+      sessionId,
+    },
+    inspect,
+  });
+
+  if (response.ok) {
+    await setSessionId(credsPayload, sessionId);
+  }
+
+  // TODO: Check if it was an auth error, and try another auth method if so
+  return response;
+};
 
 module.exports = {
   peoplevoxClient,
-};
+};

package/api/peoplevox/peoplevoxGetSingle.js

@@ -32,11 +32,13 @@ const peoplevoxGetSingle = async (
   const resolvedSearchClause = searchClause || `${ idName }.Equals("${ id }")`;
 
   const response = await peoplevoxClient.fetch({
-    method: 'post',
-    body: {
-      getRequest: {
-        TemplateName: templateName,
-        SearchClause: resolvedSearchClause,
+    requestPayload: {
+      method: 'post',
+      body: {
+        getRequest: {
+          TemplateName: templateName,
+          SearchClause: resolvedSearchClause,
+        },
       },
     },
     context: {

package/api/peoplevox/peoplevoxOrderEdit.js

@@ -37,11 +37,13 @@ const peoplevoxOrderEdit = async (
   }
 
   return peoplevoxClient.fetch({
-    method: 'post',
-    body: {
-      saveRequest: {
-        TemplateName: 'Sales orders',
-        CsvData: csvData,
+    requestPayload: {
+      method: 'post',
+      body: {
+        saveRequest: {
+          TemplateName: 'Sales orders',
+          CsvData: csvData,
+        },
       },
     },
     context: {

package/api/peoplevox/peoplevoxReportGet.js

@@ -25,15 +25,17 @@ const peoplevoxReportGet = async (
   }
 
   const reportGetResponse = await peoplevoxClient.fetch({
-    method: 'post',
-    body: {
-      getReportRequest: {
-        TemplateName: reportName,
-        ...(searchClause ? { SearchClause: searchClause } : {}),
-        ...(perPage ? { ItemsPerPage: perPage } : {}),
-        ...(filter ? { FilterClause: filter } : {}),
-        ...(orderBy ? { OrderBy: orderBy } : {}),
-        ...(columns ? { Columns: columns.join(',') } : {}),
+    requestPayload: {
+      method: 'post',
+      body: {
+        getReportRequest: {
+          TemplateName: reportName,
+          ...(searchClause ? { SearchClause: searchClause } : {}),
+          ...(perPage ? { ItemsPerPage: perPage } : {}),
+          ...(filter ? { FilterClause: filter } : {}),
+          ...(orderBy ? { OrderBy: orderBy } : {}),
+          ...(columns ? { Columns: columns.join(',') } : {}),
+        },
       },
     },
     context: {

package/api/pipelineSteps.js

@@ -0,0 +1,49 @@
+const { appendUrlToBase, credsFromPayload } = require('./utils');
+
+const resolveCreds = async (state) => {
+  const { context } = state;
+  const { creds, credsPayload } = context;
+
+  if (creds || !credsPayload) {
+    return {};
+  }
+
+  const resolvedCreds = await credsFromPayload(credsPayload);
+
+  return {
+    context: {
+      ...context,
+      creds: resolvedCreds,
+    },
+  };
+};
+
+const useBaseUrl = (baseUrl) => async (state) => {
+  const { requestPayload, context } = state;
+  const resolvedBaseUrl = baseUrl ?? context.baseUrl;
+
+  if (!resolvedBaseUrl) {
+    return {
+      breakChain: true,
+      response: {
+        ok: false,
+        error: {
+          code: 'NO_BASE_URL',
+          message: 'No base URL provided',
+        },
+      },
+    };
+  }
+
+  return {
+    requestPayload: {
+      ...requestPayload,
+      url: appendUrlToBase(resolvedBaseUrl, requestPayload.url),
+    },
+  };
+};
+
+module.exports = {
+  resolveCreds,
+  useBaseUrl,
+};

package/api/shopify/shopify.utils.js

@@ -1,11 +1,13 @@
 const { DEFAULT_API_VERSION } = require('../shopify/shopify.constants');
+const { resolveCreds } = require('../pipelineSteps');
 const {
   FetchClient,
-  Chain,
   appendUrlToBase,
   fetchClientCommonSteps,
+  jsonlToObjectArray,
   pathAsArray,
   objectDigNodeAtPath,
+  sentenceCaseString,
 } = require('../utils');
 
 const handleMutationUserErrors = async (state) => {
@@ -49,11 +51,11 @@ const handleMutationUserErrors = async (state) => {
   };
 };
 
-const addUrlAndAuthHeaders = async (state) => {
+const useUrlAndAuthHeaders = async (state) => {
   const { requestPayload, context } = state;
   const {
-    creds,
     apiVersion = DEFAULT_API_VERSION,
+    creds,
   } = context;
   const {
     STORE_HANDLE,
@@ -61,15 +63,13 @@ const addUrlAndAuthHeaders = async (state) => {
   } = creds;
 
   const baseUrl = `https://${ STORE_HANDLE }.myshopify.com/admin/api/${ apiVersion }/graphql.json`;
-  const baseHeaders = {
-    'X-Shopify-Access-Token': API_KEY,
-  };
 
   return {
     requestPayload: {
+      ...requestPayload,
       url: appendUrlToBase(baseUrl, requestPayload.url),
       headers: {
-        ...baseHeaders,
+        'X-Shopify-Access-Token': API_KEY,
         ...requestPayload.headers,
       },
     },
@@ -99,25 +99,96 @@ const movePageInfoToMeta = async (state) => {
   };
 };
 
-const shopifyClientRequestPreparer = new Chain([
-  addUrlAndAuthHeaders,
-]);
+// https://shopify.dev/docs/api/usage/bulk-operations/queries
+const parseShopifyJsonl = (jsonl) => {
+  const objects = jsonlToObjectArray(jsonl);
+  const objectsMap = new Map();
+  const objectsWithoutStitching = [];
+
+  for (const object of objects) {
+    const {
+      id: gid,
+      __parentId: parentGid,
+    } = object;
+
+    if (!gid) {
+      objectsWithoutStitching.push(object);
+      continue;
+    }
+
+    const [objectType] = gid.split('gid://shopify/')[1].split(/[^a-zA-Z0-9]+/);
+
+    objectsMap.set(gid, {
+      ...object,
+      selfType: objectType,
+      ...parentGid && { parentGid },
+    });
+  }
+
+  const objectTypeToProperty = (objectType) => `${ sentenceCaseString(objectType) }s`;
+
+  for (const [gid, object] of objectsMap) {
+    const {
+      selfType,
+      parentGid,
+    } = object;
+
+    if (!parentGid) {
+      continue;
+    }
+
+    const objectProperty = objectTypeToProperty(selfType);
+
+    let parentObject = objectsMap.get(parentGid);
+    if (!parentObject) {
+      continue;
+    }
 
-const shopifyClientResponseInterpreter = new Chain([
-  movePageInfoToMeta,
-  fetchClientCommonSteps.stripEdgesAndNodes,
-  fetchClientCommonSteps.collapseDataWithOneValue,
-  fetchClientCommonSteps.exitEarlyOnNotOk,
-  fetchClientCommonSteps.exitEarlyOnGraphqlErrors,
-  fetchClientCommonSteps.digToPath,
-  handleMutationUserErrors,
-]);
+    const nestedParent = gid.split('?')?.[1];
+
+    if (nestedParent) {
+      let [nestedParentType] = nestedParent.split('=');
+      nestedParentType = nestedParentType
+        .replaceAll('_id', '')
+        .split('_')
+        .map((word) => word[0].toUpperCase() + word.slice(1))
+        .join('');
+      nestedParentType = sentenceCaseString(nestedParentType);
+      parentObject = parentObject[nestedParentType];
+    }
+
+    if (!parentObject) {
+      continue;
+    }
+
+    parentObject[objectProperty] = parentObject[objectProperty] || [];
+    parentObject[objectProperty].push(objectsMap.get(gid));
+  }
+
+  const topLevelObjects = Array.from(objectsMap.values()).filter((object) => !object?.parentGid);
+
+  return [
+    ...topLevelObjects,
+    ...objectsWithoutStitching,
+  ];
+};
 
 const shopifyClient = new FetchClient({
-  requestPreparer: shopifyClientRequestPreparer,
-  responseInterpreter: shopifyClientResponseInterpreter,
+  pipeline: [
+    resolveCreds,
+    useUrlAndAuthHeaders,
+    'fetch',
+    movePageInfoToMeta,
+    fetchClientCommonSteps.stripEdgesAndNodes,
+    fetchClientCommonSteps.collapseDataWithOneValue,
+    fetchClientCommonSteps.exitEarlyOnNotOk,
+    fetchClientCommonSteps.exitEarlyOnGraphqlErrors,
+    fetchClientCommonSteps.digToPath,
+    handleMutationUserErrors,
+  ],
 });
 
 module.exports = {
+  parseShopifyJsonl,
   shopifyClient,
-};
+};

package/api/shopify/shopifyBulkMutationDo.js

@@ -0,0 +1,211 @@
+const { randomUUID } = require('crypto');
+const { createReadStream } = require('fs');
+const fs = require('fs').promises;
+
+const { HOSTED, TEMP_DIR } = require('../constants');
+const { credsValidator } = require('../validators');
+const {
+  ArgsWarden,
+  customFetch,
+  gidToId,
+  objectToFormData,
+  objHasAny,
+  objectArrayToJsonl,
+  jsonlToObjectArray,
+  wait,
+  valueProvided,
+} = require('../utils');
+const { shopifyStagedUploadCreate } = require('./shopifyStagedUploadCreate');
+const { shopifyBulkOperationRunMutation } = require('./shopifyBulkOperationRunMutation');
+const { shopifyBulkOperationGet } = require('./shopifyBulkOperationGet');
+
+const bulkOpAttrs = `
+  id
+  status
+  type
+  objectCount
+  url
+  errorCode
+`;
+
+const createOrResumeBulkOpPayloadValidator = (payload) => {
+  const { mutation, input, bulkOperationId } = payload;
+
+  if (valueProvided(bulkOperationId)) {
+    return true;
+  }
+
+  return valueProvided(mutation) && objHasAny(input, ['data', 'filepath']);
+};
+
+const argsWarden = new ArgsWarden([
+  ['credsPayload', credsValidator],
+  ['createOrResumeBulkOpPayload', createOrResumeBulkOpPayloadValidator],
+]);
+
+const shopifyBulkMutationDo = async (
+  credsPayload,
+  createOrResumeBulkOpPayload,
+  {
+    apiVersion,
+    clientIdentifier,
+    waitForResult = true,
+  } = {},
+) => {
+
+  const rejectResponse = await argsWarden.responseIfRejectingArgs({ 
+    credsPayload, 
+    createOrResumeBulkOpPayload,
+  });
+  if (rejectResponse) {
+    return rejectResponse;
+  }
+
+  const {
+    mutation,
+    input,
+    bulkOperationId,
+  } = createOrResumeBulkOpPayload;
+  
+  let bulkOperation;
+
+  if (!bulkOperationId) {
+
+    if (HOSTED) {
+      return {
+        ok: false,
+        error: 'This function is only available locally',
+      };
+    }
+
+    let {
+      data,
+      filepath,
+    } = input;
+
+    if (!filepath) {
+      await fs.mkdir(TEMP_DIR, { recursive: true });
+      filepath = `${ TEMP_DIR }/shopifyBulkMutationDo_${ randomUUID() }.jsonl`;
+      await fs.writeFile(filepath, objectArrayToJsonl(data));
+    }
+
+    const stagedUploadResponse = await shopifyStagedUploadCreate(
+      credsPayload,
+      {
+        resource: 'BULK_MUTATION_VARIABLES',
+        filename: filepath.split('/').pop(),
+        mimeType: 'text/jsonl',
+        httpMethod: 'POST',
+      },
+      { apiVersion },
+    );
+    if (!stagedUploadResponse.ok) {
+      return stagedUploadResponse;
+    }
+
+    const { url, parameters } = stagedUploadResponse.data.stagedTargets[0];
+
+    const formData = objectToFormData(
+      Object.fromEntries(parameters.map(({ name, value }) => [name, value])),
+    );
+    formData.append('file', createReadStream(filepath));
+
+    const uploadResponse = await customFetch(url, {
+      method: 'post',
+      body: formData,
+    });
+    if (!uploadResponse.ok) {
+      return uploadResponse;
+    }
+
+    const stagedUploadPath = uploadResponse.data?.PostResponse?.Key;
+    
+    const mutationRunResponse = await shopifyBulkOperationRunMutation(
+      credsPayload,
+      mutation,
+      stagedUploadPath,
+      {
+        apiVersion,
+        clientIdentifier,
+        returnAttrs: bulkOpAttrs,
+      },
+    );
+
+    if (!waitForResult) {
+      return mutationRunResponse;
+    }
+
+    const { ok: mutationRunOk, data: mutationRunData } = mutationRunResponse;
+    if (!mutationRunOk) {
+      return mutationRunResponse;
+    }
+    
+    bulkOperation = mutationRunData.bulkOperation;
+    bulkOperationId = gidToId(bulkOperation.id);
+  }
+  
+  while (['CREATED', 'RUNNING'].includes(bulkOperation?.status)) {
+
+    if (bulkOperation) {
+      await wait(5000);
+    }
+
+    const operationResponse = await shopifyBulkOperationGet(
+      credsPayload,
+      bulkOperationId,
+      {
+        apiVersion,
+        attrs: bulkOpAttrs,
+      },
+    );
+    if (!operationResponse.ok) {
+      return operationResponse;
+    }
+
+    bulkOperation = operationResponse.data;
+  }
+
+  if (bulkOperation.status !== 'COMPLETED') {
+    return {
+      ok: false,
+      error: {
+        code: 'BULK_OPERATION_FAILED',
+        message: `Bulk operation failed with status ${ bulkOperation.status }`,
+        details: bulkOperation,
+      },
+    };
+  }
+
+  const resultsResponse = await customFetch(bulkOperation.url);
+  if (!resultsResponse.ok) {
+    return resultsResponse;
+  }
+
+  const results = jsonlToObjectArray(resultsResponse.data);
+
+  return {
+    ok: true,
+    data: results,
+    meta: {
+      bulkOperation,
+    },
+  };
+};
+
+const funcApiConfig = {
+  argsWarden,
+};
+
+module.exports = {
+  shopifyBulkMutationDo,
+  funcApiConfig,
+};
+
+/*
+curl -X POST "http://localhost:8000/shopifyBulkMutationDo" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "credsPayload": { "credsPath": "shopify.au" },
+    "arg": "1234"
+  }'
+*/