@stackql/provider-utils 0.4.1 → 0.4.3

package/README.md

@@ -1,6 +1,7 @@
 # StackQL Provider Utils
 
-![NPM Version](https://img.shields.io/npm/v/%40stackql%2Fprovider-utils) | ![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/stackql/stackql/total?style=plastic&label=stackql%20downloads)
+![NPM Version](https://img.shields.io/npm/v/%40stackql%2Fprovider-utils)
+![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/stackql/stackql/total?style=plastic&label=stackql%20downloads)
 
 A comprehensive toolkit for transforming OpenAPI specifications into StackQL providers. This library streamlines the process of parsing, mapping, validating, testing, and generating documentation for StackQL providers.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackql/provider-utils",
-  "version": "0.4.1",
+  "version": "0.4.3",
   "description": "Utilities for building StackQL providers from OpenAPI specifications.",
   "type": "module",
   "main": "./src/index.js",

package/src/docgen/resource/methods.js

@@ -4,6 +4,24 @@ import {
     sanitizeHtml
 } from '../helpers.js';
 
+const getRequiredBodyParams = (methodDetails, accessType) => {
+    // Only process request body for insert, update, replace, and exec
+    if (!['insert', 'update', 'replace', 'exec'].includes(accessType)) {
+        return [];
+    }
+    
+    // Get required body params if they exist
+    const requiredBodyProps = methodDetails.requestBody?.required ? methodDetails.requestBody.required : [];
+    
+    // For insert, update, and replace, prefix with data__
+    if (['insert', 'update', 'replace'].includes(accessType)) {
+        return requiredBodyProps.map(prop => `data__${prop}`);
+    } else {
+        // For exec, don't prefix
+        return requiredBodyProps;
+    }
+};
+
 export function createMethodsSection(resourceData, dereferencedAPI) {
     
     let content = `\n## Methods\n\n`;
@@ -37,10 +55,16 @@ export function createMethodsSection(resourceData, dereferencedAPI) {
         for (const [methodName, methodDetails] of Object.entries(methods)) {
             console.info(`Adding ${accessType} method to table: ${methodName}`);
             
+            // Get required params from both the standard params and the request body
+            const reqParamsArr = Object.keys(methodDetails.requiredParams || {});
+            const reqBodyParamsArr = getRequiredBodyParams(methodDetails, accessType);
+            
+            // Combine both types of required parameters
+            const allReqParamsArr = [...reqParamsArr, ...reqBodyParamsArr];
+            
             // Format required params as comma-delimited list with hyperlinks
-            const requiredParamsArr = Object.keys(methodDetails.requiredParams || {});
-            const requiredParamsStr = requiredParamsArr.length > 0 
-                ? requiredParamsArr.map(param => `<a href="#parameter-${param}"><code>${param}</code></a>`).join(', ') 
+            const requiredParamsStr = allReqParamsArr.length > 0 
+                ? allReqParamsArr.map(param => `<a href="#parameter-${param}"><code>${param}</code></a>`).join(', ') 
                 : '';
             
             // Format optional params as comma-delimited list with hyperlinks
@@ -48,7 +72,7 @@ export function createMethodsSection(resourceData, dereferencedAPI) {
             const optionalParamsStr = optionalParamsArr.length > 0 
                 ? optionalParamsArr.map(param => `<a href="#parameter-${param}"><code>${param}</code></a>`).join(', ') 
                 : '';
-            
+
             // Add the method row to the table
             content += `
 <tr>

package/src/providerdev/analyze.js

@@ -51,6 +51,108 @@ function extractMain2xxResponse(responseObj) {
   return '';
 }
 
+/**
+ * Detect if an object should have an objectKey and determine what it should be
+ * @param {Object} spec - Full OpenAPI spec
+ * @param {Object} operation - Operation object
+ * @returns {string} - Suggested objectKey or empty string if none
+ */
+function detectObjectKey(spec, operation) {
+  // Only applicable for GET operations
+  if (!operation || !operation.responses) {
+    return '';
+  }
+  
+  let responseObject = null;
+  let responseCode = null;
+  
+  // Find the first 2xx response
+  for (const [code, response] of Object.entries(operation.responses)) {
+    if (code.startsWith('2')) {
+      responseCode = code;
+      // Handle direct response or reference
+      if (response.$ref) {
+        // Resolve reference
+        const refParts = response.$ref.split('/');
+        const componentType = refParts[2];
+        const responseName = refParts[3];
+        responseObject = spec.components?.[componentType]?.[responseName];
+      } else {
+        responseObject = response;
+      }
+      break;
+    }
+  }
+  
+  if (!responseObject) {
+    return '';
+  }
+  
+  // Get the schema from the response
+  let schema = null;
+  
+  // If it's a direct response object
+  if (responseObject.content?.['application/json']?.schema) {
+    schema = responseObject.content['application/json'].schema;
+  } 
+  
+  // If the schema is a reference, resolve it
+  if (schema && schema.$ref) {
+    const refParts = schema.$ref.split('/');
+    const componentType = refParts[2];
+    const schemaName = refParts[3];
+    schema = spec.components?.[componentType]?.[schemaName];
+  }
+  
+  // Handle allOf case (like in the droplets_list example)
+  if (schema && schema.allOf) {
+    // Look at the first object in allOf that has properties
+    for (const subSchema of schema.allOf) {
+      if (subSchema.properties && Object.keys(subSchema.properties).length === 1) {
+        const key = Object.keys(subSchema.properties)[0];
+        return `$.${key}`;
+      }
+    }
+  }
+  
+  // Handle direct properties case (like in the droplets_get example)
+  if (schema && schema.properties) {
+    // If there's only one property at the top level, and it's not a primitive
+    const propKeys = Object.keys(schema.properties);
+    if (propKeys.length === 1) {
+      const key = propKeys[0];
+      const prop = schema.properties[key];
+      
+      // Check if the property is an object or array, not a primitive
+      if (prop.$ref || 
+          prop.type === 'object' || 
+          prop.type === 'array' ||
+          (prop.properties && Object.keys(prop.properties).length > 0)) {
+        return `$.${key}`;
+      }
+    }
+  }
+  
+  return '';
+}
+
+/**
+ * Map HTTP verb to SQL verb
+ * @param {string} httpVerb - HTTP verb (get, post, put, etc)
+ * @returns {string} - Corresponding SQL verb
+ */
+function mapToSqlVerb(httpVerb) {
+  const verbMap = {
+    'get': 'select',
+    'post': 'insert',
+    'delete': 'delete',
+    'put': 'replace',
+    'patch': 'update'
+  };
+  
+  return verbMap[httpVerb] || 'exec';
+}
+
 /**
  * Find existing mapping in x-stackQL-resources
  * @param {Object} spec - OpenAPI spec
@@ -77,10 +179,14 @@ function findExistingMapping(spec, pathRef) {
           }
         }
         
+        // Get objectKey if present
+        const objectKey = method.response?.objectKey || '';
+        
         return {
           resourceName,
           methodName,
-          sqlVerb
+          sqlVerb,
+          objectKey
         };
       }
     }
@@ -91,10 +197,28 @@ function findExistingMapping(spec, pathRef) {
   return {
     resourceName: '',
     methodName: '',
-    sqlVerb: ''
+    sqlVerb: '',
+    objectKey: ''
   };
 }
 
+/**
+ * Escape and sanitize a CSV field value
+ * @param {string} value - Field value to escape
+ * @returns {string} - Escaped value
+ */
+function escapeCsvField(value) {
+  if (!value) return '';
+  
+  // If the value contains commas, double quotes, or newlines, wrap it in quotes
+  // and escape any existing double quotes by doubling them
+  if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+  
+  return value;
+}
+
 /**
  * Analyze OpenAPI specs and generate mapping CSV
  * @param {Object} options - Options for analysis
@@ -133,7 +257,8 @@ export async function analyze(options) {
                 existingMappings[key] = {
                   resourceName: row.stackql_resource_name || '',
                   methodName: row.stackql_method_name || '',
-                  sqlVerb: row.stackql_verb || ''
+                  sqlVerb: row.stackql_verb || '',
+                  objectKey: row.stackql_object_key || ''
                 };
               }
             })
@@ -159,7 +284,7 @@ export async function analyze(options) {
 
     // Only write header if creating a new file
     if (!fileExists) {
-      writer.write('filename,path,operationId,formatted_op_id,verb,response_object,tags,formatted_tags,stackql_resource_name,stackql_method_name,stackql_verb\n');
+      writer.write('filename,path,operationId,formatted_op_id,verb,response_object,tags,formatted_tags,stackql_resource_name,stackql_method_name,stackql_verb,stackql_object_key,op_description\n');
     }
     
     const files = fs.readdirSync(inputDir);
@@ -214,17 +339,45 @@ export async function analyze(options) {
           const pathRef = `#/paths/${encodedPath}/${verb}`;
           
           // Find existing mapping if available
-          const { resourceName, methodName, sqlVerb } = findExistingMapping(spec, pathRef);
+          let { resourceName, methodName, sqlVerb, objectKey } = findExistingMapping(spec, pathRef);
+          
+          // CHANGE 1: Default methodName to formattedOpId if not found
+          if (!methodName) {
+            methodName = formattedOpId;
+          }
+          
+          // CHANGE 2: Default sqlVerb based on HTTP verb if not found
+          if (!sqlVerb) {
+            sqlVerb = mapToSqlVerb(verb);
+          }
+
+          // CHANGE 3: Detect and set objectKey for GET operations if not already set
+          if (!objectKey && verb === 'get') {
+            objectKey = detectObjectKey(spec, operation);
+          }
+          
+          // Get operation description
+          const opDescription = operation.summary || operation.description || '';
           
-          // Escape commas in fields
-          const escapedPath = pathKey.includes(',') ? `"${pathKey}"` : pathKey;
-          const escapedOperationId = operationId.includes(',') ? `"${operationId}"` : operationId;
-          const escapedFormattedOpId = formattedOpId.includes(',') ? `"${formattedOpId}"` : formattedOpId;
-          const escapedTags = tagsStr.includes(',') ? `"${tagsStr}"` : tagsStr;
-          const escapedFormattedTags = formattedTags.includes(',') ? `"${formattedTags}"` : formattedTags;
+          // Escape fields that might contain commas, quotes, or other special characters
+          const escapedFields = {
+            filename: escapeCsvField(filename),
+            path: escapeCsvField(pathKey),
+            operationId: escapeCsvField(operationId),
+            formattedOpId: escapeCsvField(formattedOpId),
+            verb: escapeCsvField(verb),
+            responseRef: escapeCsvField(responseRef),
+            tagsStr: escapeCsvField(tagsStr),
+            formattedTags: escapeCsvField(formattedTags),
+            resourceName: escapeCsvField(resourceName),
+            methodName: escapeCsvField(methodName),
+            sqlVerb: escapeCsvField(sqlVerb),
+            objectKey: escapeCsvField(objectKey),
+            opDescription: escapeCsvField(opDescription)
+          };
           
           // Write row
-          writer.write(`${filename},${escapedPath},${escapedOperationId},${escapedFormattedOpId},${verb},${responseRef},${escapedTags},${escapedFormattedTags},${resourceName},${methodName},${sqlVerb}\n`);
+          writer.write(`${escapedFields.filename},${escapedFields.path},${escapedFields.operationId},${escapedFields.formattedOpId},${escapedFields.verb},${escapedFields.responseRef},${escapedFields.tagsStr},${escapedFields.formattedTags},${escapedFields.resourceName},${escapedFields.methodName},${escapedFields.sqlVerb},${escapedFields.objectKey},${escapedFields.opDescription}\n`);
         }
       }
     }

package/src/providerdev/generate.js

@@ -1,4 +1,3 @@
-// src/providerdev/generate.js
 import fs from 'fs';
 import path from 'path';
 import yaml from 'js-yaml';
@@ -75,14 +74,15 @@ function getSuccessResponseInfo(operation) {
     .sort();
 
   if (twoXxCodes.length === 0) {
-    return { mediaType: '', openAPIDocKey: '' };
+    throw new Error('No 2xx response found, openAPIDocKey is required');
   }
 
   const lowest2xx = twoXxCodes[0];
   const content = responses[lowest2xx]?.content || {};
   const mediaTypes = Object.keys(content);
 
-  const mediaType = mediaTypes.length > 0 ? mediaTypes[0] : '';
+  // Default to 'application/json' if mediaType is not found
+  const mediaType = mediaTypes.length > 0 ? mediaTypes[0] : 'application/json';
 
   return {
     mediaType,
@@ -99,6 +99,71 @@ function snakeCase(name) {
   return name.replace(/-/g, '_');
 }
 
+/**
+ * Count the number of path parameters in a path
+ * @param {string} path - HTTP path
+ * @returns {number} - Number of path parameters
+ */
+function countPathParams(path) {
+  // Match all path parameters like {param_name}
+  const matches = path.match(/\{[^}]+\}/g);
+  return matches ? matches.length : 0;
+}
+
+/**
+ * Sort operations from most specific to least specific based on path parameters
+ * @param {Object} resources - Resources object containing methods and sqlVerbs
+ * @param {Object} spec - Full OpenAPI specification
+ * @returns {Object} - Resources with sorted sqlVerbs
+ */
+function sortOperationsBySpecificity(resources, spec) {
+  // For each resource
+  for (const resourceName in resources) {
+    const resource = resources[resourceName];
+    const methods = resource.methods;
+    
+    // Create a map of method references to their specificity (path param count)
+    const methodSpecificityMap = {};
+    
+    // For each method, find its operation ref and count path params
+    for (const methodName in methods) {
+      const method = methods[methodName];
+      const operationRef = method.operation.$ref;
+      
+      // Extract path and verb from the reference
+      // Reference format: '#/paths/{encodedPath}/{verb}'
+      const refParts = operationRef.split('/');
+      const verb = refParts.pop();
+      // Remove '#/paths/' and the verb, then decode the path
+      const encodedPath = refParts.slice(2).join('/');
+      const path = encodedPath.replace(/~1/g, '/');
+      
+      // Count path parameters
+      const paramCount = countPathParams(path);
+      
+      // Store the method reference and its path parameter count
+      const methodRef = `#/components/x-stackQL-resources/${resourceName}/methods/${methodName}`;
+      methodSpecificityMap[methodRef] = paramCount;
+    }
+    
+    // For each SQL verb, sort the operations by specificity
+    for (const verbName in resource.sqlVerbs) {
+      const operations = resource.sqlVerbs[verbName];
+      
+      if (operations && operations.length > 0) {
+        // Sort operations from most specific (more path params) to least specific
+        operations.sort((a, b) => {
+          const aRef = a.$ref;
+          const bRef = b.$ref;
+          return methodSpecificityMap[bRef] - methodSpecificityMap[aRef];
+        });
+      }
+    }
+  }
+  
+  return resources;
+}
+
 /**
  * Generate StackQL provider extensions
  * @param {Object} options - Options for generation
@@ -234,6 +299,11 @@ export async function generate(options) {
             response: responseInfo
           };
 
+          // Add objectKey to the response info if it exists in the manifest and is for a GET operation
+          if (entry.stackql_object_key && verb === 'get') {
+            methodEntry.response.objectKey = entry.stackql_object_key;
+          }
+
           resources[resource].methods[method] = methodEntry;
           if (sqlverb && sqlverb === 'exec') {
             logger.info(`exec method skipped:  ${resource}.${method}`);
@@ -247,11 +317,14 @@ export async function generate(options) {
         }
       }
       
+      // Sort operations by specificity before injecting into spec
+      const sortedResources = sortOperationsBySpecificity(resources, spec);
+      
       // Inject into spec
       if (!spec.components) {
         spec.components = {};
       }
-      spec.components['x-stackQL-resources'] = resources;
+      spec.components['x-stackQL-resources'] = sortedResources;
       
       // Inject servers if provided
       if (servers) {

package/src/providerdev/split.js

@@ -11,7 +11,6 @@ import {
 
 // Constants
 const OPERATIONS = ["get", "post", "put", "delete", "patch", "options", "head", "trace"];
-const NON_OPERATIONS = ["parameters", "servers", "summary", "description"];
 const COMPONENTS_CHILDREN = ["schemas", "responses", "parameters", "examples", "requestBodies", "headers", "securitySchemes", "links", "callbacks"];
 
 /**
@@ -42,9 +41,10 @@ function isOperationExcluded(exclude, opItem) {
  * @param {string} svcDiscriminator - Service discriminator
  * @param {Object[]} allTags - All tags from API doc
  * @param {boolean} debug - Debug flag
+ * @param {Object} svcNameOverrides - Service name overrides
  * @returns {[string, string]} - [service name, service description]
  */
-function retServiceNameAndDesc(providerName, opItem, pathKey, svcDiscriminator, allTags, debug) {
+function retServiceNameAndDesc(providerName, opItem, pathKey, svcDiscriminator, allTags, debug, svcNameOverrides) {
   let service = "default";
   let serviceDesc = `${providerName} API`;
   
@@ -84,9 +84,25 @@ function retServiceNameAndDesc(providerName, opItem, pathKey, svcDiscriminator,
     return ["skip", ""];
   }
   
+  // Apply service name overrides if present
+  if (svcNameOverrides && svcNameOverrides[service]) {
+    const newName = svcNameOverrides[service];
+    if (debug) {
+      logger.debug(`Overriding service name: ${service} -> ${newName}`);
+    }
+    
+    // Update service description for path-based services
+    if (svcDiscriminator === "path") {
+      serviceDesc = `${providerName} ${newName} API`;
+    }
+    
+    service = newName;
+  }
+  
   return [service, serviceDesc];
 }
 
+
 /**
  * Initialize service map
  * @param {Object} services - Services map
@@ -192,43 +208,6 @@ function getPathLevelRefs(pathItem) {
   return refs;
 }
 
-/**
- * Add referenced components to service
- * @param {Set<string>} refs - Set of refs
- * @param {Object} service - Service object
- * @param {Object} components - Components from API doc
- * @param {boolean} debug - Debug flag
- */
-function addRefsToComponents(refs, service, components, debug) {
-  for (const ref of refs) {
-    const parts = ref.split('/');
-    
-    // Only process refs that point to components
-    if (parts.length >= 4 && parts[1] === "components") {
-      const componentType = parts[2];
-      const componentName = parts[3];
-      
-      // Check if component type exists in service
-      if (!service.components[componentType]) {
-        service.components[componentType] = {};
-      }
-      
-      // Skip if component already added
-      if (service.components[componentType][componentName]) {
-        continue;
-      }
-      
-      // Add component if it exists in source document
-      if (components[componentType] && components[componentType][componentName]) {
-        service.components[componentType][componentName] = components[componentType][componentName];
-        if (debug) {
-          logger.debug(`Added component ${componentType}/${componentName}`);
-        }
-      }
-    }
-  }
-}
-
 /**
  * Add missing type: object to schema objects
  * @param {any} obj - Object to process
@@ -338,7 +317,8 @@ export async function split(options) {
     svcDiscriminator = "tag",
     exclude = null,
     overwrite = true,
-    verbose = false
+    verbose = false,
+    svcNameOverrides = {}  // Add this new parameter with default empty object
   } = options;
   
   // Setup logging based on verbosity
@@ -351,6 +331,13 @@ export async function split(options) {
   logger.info(`Output: ${outputDir}`);
   logger.info(`Service Discriminator: ${svcDiscriminator}`);
   
+  if (Object.keys(svcNameOverrides).length > 0) {
+    logger.info(`Using ${Object.keys(svcNameOverrides).length} service name overrides`);
+    if (verbose) {
+      logger.debug(`Service name overrides: ${JSON.stringify(svcNameOverrides, null, 2)}`);
+    }
+  }
+  
   // Process exclude list
   const excludeList = exclude ? exclude.split(",") : [];
   
@@ -409,12 +396,11 @@ export async function split(options) {
         continue;
       }
       
-      // Determine service name
       const [service, serviceDesc] = retServiceNameAndDesc(
         providerName, opItem, pathKey, svcDiscriminator, 
-        apiDocObj.tags || [], verbose
-      );
-      
+        apiDocObj.tags || [], verbose, svcNameOverrides
+      );      
+
       // Skip if service is marked to skip
       if (service === 'skip') {
         logger.warn(`⭐️ Skipping service: ${service}`);