@stackql/provider-utils 0.2.4 → 0.3.1

package/README.md

@@ -159,6 +159,11 @@ node tests/docgen/test-docgen.js google
 node tests/docgen/test-docgen.js homebrew
 ```
 
+
+node tests/providerdev/test-split.js okta tests/providerdev/split-source/okta/management-minimal.yaml path
+node tests/providerdev/test-analyze.js okta
+node tests/providerdev/test-generate.js okta
+
 ## Using the Documentation Generator
 
 ### Basic Example

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackql/provider-utils",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "description": "Utilities for building StackQL providers from OpenAPI specifications.",
   "type": "module",
   "main": "./src/index.js",
@@ -36,6 +36,7 @@
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.1",
     "@stackql/deno-openapi-dereferencer": "npm:@jsr/stackql__deno-openapi-dereferencer@^0.3.1",
+    "csv-parser": "^3.2.0",
     "js-yaml": "^4.1.0",
     "pluralize": "^8.0.0"
   },

package/src/index.js

@@ -1,4 +1,5 @@
-// src/index.js
+// @stackql/provider-utils/src/index.js
 
 // Main entry point for Node.js
-export * as docgen from './docgen/index.js';
+export * as docgen from './docgen/index.js';
+export * as providerdev from './providerdev/index.js';

package/src/logger.js

@@ -0,0 +1,52 @@
+// @stackql/provider-utils/src/logger.js
+
+// Simple logger implementation
+const logLevels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  debug: 3
+};
+
+let currentLevel = 'info';
+
+const logger = {
+  get level() {
+    return currentLevel;
+  },
+  
+  set level(newLevel) {
+    if (logLevels[newLevel] !== undefined) {
+      currentLevel = newLevel;
+    } else {
+      console.warn(`Invalid log level: ${newLevel}. Using 'info' instead.`);
+      currentLevel = 'info';
+    }
+  },
+  
+  error: (message) => {
+    if (logLevels[currentLevel] >= logLevels.error) {
+      console.error(`ERROR: ${message}`);
+    }
+  },
+  
+  warn: (message) => {
+    if (logLevels[currentLevel] >= logLevels.warn) {
+      console.warn(`WARNING: ${message}`);
+    }
+  },
+  
+  info: (message) => {
+    if (logLevels[currentLevel] >= logLevels.info) {
+      console.info(`INFO: ${message}`);
+    }
+  },
+  
+  debug: (message) => {
+    if (logLevels[currentLevel] >= logLevels.debug) {
+      console.debug(`DEBUG: ${message}`);
+    }
+  }
+};
+
+export default logger;

package/src/providerdev/analyze.js

@@ -0,0 +1,172 @@
+// src/providerdev/analyze.js
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import logger from '../logger.js';
+import { camelToSnake } from '../utils.js';
+
+/**
+ * Load specification from YAML or JSON file
+ * @param {string} filepath - Path to specification file
+ * @returns {Object} - Loaded specification
+ */
+function loadSpec(filepath) {
+  const content = fs.readFileSync(filepath, 'utf-8');
+  if (filepath.endsWith('.json')) {
+    return JSON.parse(content);
+  }
+  return yaml.load(content);
+}
+
+/**
+ * Extract main 2xx response schema reference
+ * @param {Object} responseObj - Response object
+ * @returns {string} - Schema reference name
+ */
+function extractMain2xxResponse(responseObj) {
+  for (const [code, response] of Object.entries(responseObj)) {
+    if (code.startsWith('2')) {
+      const content = response.content || {};
+      const appJson = content['application/json'] || {};
+      const schema = appJson.schema || {};
+
+      // Case 1: Direct $ref
+      if (schema.$ref) {
+        return schema.$ref.split('/').pop();
+      }
+
+      // Case 2: Array of items
+      if (schema.type === 'array') {
+        const items = schema.items || {};
+        if (items.$ref) {
+          return items.$ref.split('/').pop();
+        }
+      }
+      
+      return '';
+    }
+  }
+  return '';
+}
+
+/**
+ * Find existing mapping in x-stackQL-resources
+ * @param {Object} spec - OpenAPI spec
+ * @param {string} pathRef - Reference to path item
+ * @returns {Object} - Mapping info (resource, method, verb)
+ */
+function findExistingMapping(spec, pathRef) {
+  const stackQLResources = spec.components?.['x-stackQL-resources'] || {};
+  
+  for (const [resourceName, resource] of Object.entries(stackQLResources)) {
+    // Check methods
+    for (const [methodName, method] of Object.entries(resource.methods || {})) {
+      if (method.operation?.$ref === pathRef) {
+        logger.info(`Found mapping for ${pathRef}: ${resourceName}.${methodName}`);
+        // Find SQL verb for this method
+        let sqlVerb = 'exec'; // Default if no explicit mapping
+        
+        for (const [verb, methods] of Object.entries(resource.sqlVerbs || {})) {
+          for (const methodRef of methods) {
+            if (methodRef.$ref === `#/components/x-stackQL-resources/${resourceName}/methods/${methodName}`) {
+              sqlVerb = verb;
+              break;
+            }
+          }
+        }
+        
+        return {
+          resourceName,
+          methodName,
+          sqlVerb
+        };
+      }
+    }
+  }
+
+  logger.info(`No mapping for ${pathRef}`);
+  
+  return {
+    resourceName: '',
+    methodName: '',
+    sqlVerb: ''
+  };
+}
+
+/**
+ * Analyze OpenAPI specs and generate mapping CSV
+ * @param {Object} options - Options for analysis
+ * @returns {Promise<boolean>} - Success status
+ */
+export async function analyze(options) {
+  const {
+    inputDir,
+    outputDir
+  } = options;
+
+  try {
+    fs.mkdirSync(outputDir, { recursive: true });
+    const outputPath = path.join(outputDir, 'all_services.csv');
+    
+    const writer = fs.createWriteStream(outputPath, { encoding: 'utf8' });
+    
+    // Write header
+    writer.write('filename,path,operationId,formatted_op_id,verb,response_object,tags,formatted_tags,stackql_resource_name,stackql_method_name,stackql_verb\n');
+    
+    const files = fs.readdirSync(inputDir);
+    
+    for (const filename of files) {
+      if (!filename.endsWith('.yaml') && !filename.endsWith('.yml') && !filename.endsWith('.json')) {
+        continue;
+      }
+      
+      const filepath = path.join(inputDir, filename);
+      const spec = loadSpec(filepath);
+      
+      for (const [pathKey, pathItem] of Object.entries(spec.paths || {})) {
+        for (const [verb, operation] of Object.entries(pathItem)) {
+          if (typeof operation !== 'object' || operation === null) {
+            continue;
+          }
+          
+          const operationId = operation.operationId || '';
+          // Format operationId as snake_case
+          const formattedOpId = operationId ? camelToSnake(operationId) : '';
+          
+          const responseObj = operation.responses || {};
+          const responseRef = extractMain2xxResponse(responseObj);
+          const tagsList = operation.tags || [];
+          const tagsStr = tagsList.join('|');
+          
+          // Format tags as snake_case
+          const formattedTags = tagsList.map(tag => camelToSnake(tag)).join('|');
+          
+          // Construct the path reference as it would appear in x-stackQL-resources
+          const encodedPath = pathKey.replace(/\//g, '~1');
+          const pathRef = `#/paths/${encodedPath}/${verb}`;
+          
+          // Find existing mapping if available
+          const { resourceName, methodName, sqlVerb } = findExistingMapping(spec, pathRef);
+          
+          // Escape commas in fields
+          const escapedPath = pathKey.includes(',') ? `"${pathKey}"` : pathKey;
+          const escapedOperationId = operationId.includes(',') ? `"${operationId}"` : operationId;
+          const escapedFormattedOpId = formattedOpId.includes(',') ? `"${formattedOpId}"` : formattedOpId;
+          const escapedTags = tagsStr.includes(',') ? `"${tagsStr}"` : tagsStr;
+          const escapedFormattedTags = formattedTags.includes(',') ? `"${formattedTags}"` : formattedTags;
+          
+          // Write row
+          writer.write(`${filename},${escapedPath},${escapedOperationId},${escapedFormattedOpId},${verb},${responseRef},${escapedTags},${escapedFormattedTags},${resourceName},${methodName},${sqlVerb}\n`);
+        }
+      }
+    }
+    
+    writer.end();
+    
+    logger.info(`✅ Analysis complete. Output written to: ${outputPath}`);
+    return true;
+  } catch (error) {
+    logger.error(`Failed to analyze OpenAPI specs: ${error.message}`);
+    return false;
+  }
+}

package/src/providerdev/generate.js

@@ -0,0 +1,307 @@
+// src/providerdev/generate.js
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import csv from 'csv-parser';
+import logger from '../logger.js';
+import { createReadStream } from 'fs';
+
+/**
+ * Load manifest from CSV file
+ * @param {string} configPath - Path to CSV config file
+ * @returns {Promise<Object>} - Manifest object
+ */
+async function loadManifest(configPath) {
+  const manifest = {};
+  
+  return new Promise((resolve, reject) => {
+    createReadStream(configPath)
+      .pipe(csv())
+      .on('data', (row) => {
+        const key = `${row.filename}::${row.operationId}`;
+        manifest[key] = row;
+      })
+      .on('end', () => {
+        resolve(manifest);
+      })
+      .on('error', (error) => {
+        reject(error);
+      });
+  });
+}
+
+/**
+ * Load specification from YAML or JSON file
+ * @param {string} filepath - Path to specification file
+ * @returns {Object} - Loaded specification
+ */
+function loadSpec(filepath) {
+  const content = fs.readFileSync(filepath, 'utf-8');
+  if (filepath.endsWith('.json')) {
+    return JSON.parse(content);
+  }
+  return yaml.load(content);
+}
+
+/**
+ * Write specification to file
+ * @param {string} filepath - Output file path
+ * @param {Object} data - Data to write
+ */
+function writeSpec(filepath, data) {
+  fs.writeFileSync(filepath, yaml.dump(data, { sortKeys: false }));
+}
+
+/**
+ * Encode reference path
+ * @param {string} path - HTTP path
+ * @param {string} verb - HTTP verb
+ * @returns {string} - Encoded reference path
+ */
+function encodeRefPath(path, verb) {
+  const encodedPath = path.replace(/\//g, '~1');
+  return `#/paths/${encodedPath}/${verb}`;
+}
+
+/**
+ * Get success response information
+ * @param {Object} operation - Operation object
+ * @returns {Object} - Response information
+ */
+function getSuccessResponseInfo(operation) {
+  const responses = operation.responses || {};
+  const twoXxCodes = Object.keys(responses)
+    .filter(code => code.startsWith('2'))
+    .sort();
+
+  if (twoXxCodes.length === 0) {
+    return { mediaType: '', openAPIDocKey: '' };
+  }
+
+  const lowest2xx = twoXxCodes[0];
+  const content = responses[lowest2xx]?.content || {};
+  const mediaTypes = Object.keys(content);
+
+  const mediaType = mediaTypes.length > 0 ? mediaTypes[0] : '';
+
+  return {
+    mediaType,
+    openAPIDocKey: lowest2xx
+  };
+}
+
+/**
+ * Convert string to snake_case
+ * @param {string} name - String to convert
+ * @returns {string} - Converted string
+ */
+function snakeCase(name) {
+  return name.replace(/-/g, '_');
+}
+
+/**
+ * Generate StackQL provider extensions
+ * @param {Object} options - Options for generation
+ * @returns {Promise<boolean>} - Success status
+ */
+export async function generate(options) {
+  const {
+    inputDir,
+    outputDir,
+    configPath,
+    providerId,
+    servers = null,
+    providerConfig = null,
+    skipFiles = []
+  } = options;
+
+  const version = 'v00.00.00000';
+  const servicesPath = path.join(outputDir, version, 'services');
+  
+  // Create directories
+  fs.mkdirSync(servicesPath, { recursive: true });
+  
+  // Clean all files in services output dir
+  try {
+    const files = fs.readdirSync(servicesPath);
+    for (const file of files) {
+      const filePath = path.join(servicesPath, file);
+      if (fs.statSync(filePath).isFile()) {
+        fs.unlinkSync(filePath);
+      }
+    }
+    logger.info(`🧹 Cleared all files in ${servicesPath}`);
+  } catch (error) {
+    logger.error(`Failed to clear files in ${servicesPath}: ${error.message}`);
+    return false;
+  }
+  
+  // Delete provider.yaml file
+  const providerManifestFile = path.join(outputDir, version, 'provider.yaml');
+  if (fs.existsSync(providerManifestFile)) {
+    fs.unlinkSync(providerManifestFile);
+    logger.info(`🧹 Deleted ${providerManifestFile}`);
+  }
+  
+  // Load manifest
+  let manifest;
+  try {
+    manifest = await loadManifest(configPath);
+  } catch (error) {
+    logger.error(`Failed to load manifest: ${error.message}`);
+    return false;
+  }
+  
+  const providerServices = {};
+  
+  try {
+    const files = fs.readdirSync(inputDir);
+    
+    for (const filename of files) {
+      if (skipFiles.includes(filename)) {
+        logger.info(`⭐️ Skipping ${filename} (matched --skip)`);
+        continue;
+      }
+      
+      if (!filename.endsWith('.yaml') && !filename.endsWith('.yml') && !filename.endsWith('.json')) {
+        continue;
+      }
+      
+      const baseName = path.basename(filename, path.extname(filename));
+      const serviceName = snakeCase(baseName);
+
+      console.log(`processing service: ${serviceName}`);
+
+      const specPath = path.join(inputDir, filename);
+      const spec = loadSpec(specPath);
+      
+      // Initialize resources object with defaultdict-like behavior
+      const resources = {};
+      
+      for (const [pathKey, pathItem] of Object.entries(spec.paths || {})) {
+        for (const [verb, operation] of Object.entries(pathItem)) {
+          if (typeof operation !== 'object' || operation === null) {
+            continue;
+          }
+          
+          const operationId = operation.operationId;
+          if (!operationId) {
+            continue;
+          }
+          
+          const manifestKey = `${filename}::${operationId}`;
+          const entry = manifest[manifestKey];
+          if (!entry) {
+            logger.error(`❌ ERROR: ${filename} → ${operationId} not found in manifest`);
+            return false;
+          }
+          
+          const resource = entry.stackql_resource_name;
+          const method = entry.stackql_method_name;
+          const sqlverb = entry.stackql_verb;
+          
+          // Initialize resource if it doesn't exist
+          if (!resources[resource]) {
+            resources[resource] = {
+              id: `${providerId}.${serviceName}.${resource}`,
+              name: resource,
+              title: resource.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase()),
+              methods: {},
+              sqlVerbs: { 
+                select: [], 
+                insert: [], 
+                update: [], 
+                delete: [], 
+                replace: [] 
+              }
+            };
+          }
+          
+          const pathRef = encodeRefPath(pathKey, verb);
+          const responseInfo = getSuccessResponseInfo(operation);
+          
+          const methodEntry = {
+            operation: { $ref: pathRef },
+            response: responseInfo
+          };
+
+          resources[resource].methods[method] = methodEntry;
+          if (sqlverb && sqlverb === 'exec') {
+            logger.info(`exec method skipped:  ${resource}.${method}`);
+          } else if (sqlverb && resources[resource].sqlVerbs[sqlverb]) {
+            resources[resource].sqlVerbs[sqlverb].push({
+              $ref: `#/components/x-stackQL-resources/${resource}/methods/${method}`
+            });
+          } else if (sqlverb) {
+            logger.warn(`⚠️ Unknown SQL verb '${sqlverb}' for ${resource}.${method}, skipping`);
+          }
+        }
+      }
+      
+      // Inject into spec
+      if (!spec.components) {
+        spec.components = {};
+      }
+      spec.components['x-stackQL-resources'] = resources;
+      
+      // Inject servers if provided
+      if (servers) {
+        try {
+          const serversJson = JSON.parse(servers);
+          spec.servers = serversJson;
+        } catch (error) {
+          logger.error(`❌ Failed to parse servers JSON: ${error.message}`);
+          return false;
+        }
+      }
+      
+      // Write enriched spec
+      const outputPath = path.join(servicesPath, filename);
+      writeSpec(outputPath, spec);
+      logger.info(`✅ Wrote enriched spec: ${outputPath}`);
+      
+      // Add providerService entry
+      const info = spec.info || {};
+      const specTitle = info.title || `${serviceName.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase())} API`;
+      const specDescription = info.description || `TODO: add description for ${serviceName}`;
+      
+      providerServices[serviceName] = {
+        id: `${serviceName}:${version}`,
+        name: serviceName,
+        preferred: true,
+        service: {
+          $ref: `${providerId}/${version}/services/${filename}`
+        },
+        title: specTitle,
+        version: version,
+        description: specDescription
+      };
+    }
+    
+    // Write provider.yaml
+    const providerYaml = {
+      id: providerId,
+      name: providerId,
+      version: version,
+      providerServices: providerServices,
+    };
+    
+    if (providerConfig) {
+      try {
+        const providerConfigJson = JSON.parse(providerConfig);
+        providerYaml.config = providerConfigJson;
+      } catch (error) {
+        logger.error(`❌ Failed to parse provider config JSON: ${error.message}`);
+        return false;
+      }
+    }
+    
+    writeSpec(path.join(outputDir, version, 'provider.yaml'), providerYaml);
+    logger.info(`📦 Wrote provider.yaml to ${outputDir}/${version}/provider.yaml`);
+    
+    return true;
+  } catch (error) {
+    logger.error(`Failed to generate provider: ${error.message}`);
+    return false;
+  }
+}

package/src/providerdev/index.js

@@ -0,0 +1,10 @@
+// src/providerdev/index.js
+import { split } from './split.js';
+import { generate } from './generate.js';
+import { analyze } from './analyze.js';
+
+export {
+  split,
+  generate,
+  analyze
+};

package/src/providerdev/split.js

@@ -0,0 +1,492 @@
+// @stackql/provider-utils/src/providerdev/split.js
+
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import logger from '../logger.js';
+import {
+  camelToSnake,
+  createDestDir
+} from '../utils.js';
+
+// Constants
+const OPERATIONS = ["get", "post", "put", "delete", "patch", "options", "head", "trace"];
+const NON_OPERATIONS = ["parameters", "servers", "summary", "description"];
+const COMPONENTS_CHILDREN = ["schemas", "responses", "parameters", "examples", "requestBodies", "headers", "securitySchemes", "links", "callbacks"];
+
+/**
+ * Check if operation should be excluded
+ * @param {string[]} exclude - List of exclusion criteria
+ * @param {Object} opItem - Operation item from OpenAPI doc
+ * @param {string} svcDiscriminator - Service discriminator 
+ * @returns {boolean} - Whether operation should be excluded
+ */
+function isOperationExcluded(exclude, opItem) {
+  if (!exclude || exclude.length === 0) {
+    return false;
+  }
+  
+  // Example: exclude based on tags or other criteria
+  if (opItem.tags && opItem.tags.some(tag => exclude.includes(tag))) {
+    return true;
+  }
+  
+  return false;
+}
+
+/**
+ * Determine service name and description using discriminator
+ * @param {string} providerName - Provider name
+ * @param {Object} opItem - Operation item
+ * @param {string} pathKey - Path key
+ * @param {string} svcDiscriminator - Service discriminator
+ * @param {Object[]} allTags - All tags from API doc
+ * @param {boolean} debug - Debug flag
+ * @returns {[string, string]} - [service name, service description]
+ */
+function retServiceNameAndDesc(providerName, opItem, pathKey, svcDiscriminator, allTags, debug) {
+  let service = "default";
+  let serviceDesc = `${providerName} API`;
+  
+  // Use tags if discriminator is "tag"
+  if (svcDiscriminator === "tag" && opItem.tags && opItem.tags.length > 0) {
+    service = opItem.tags[0].toLowerCase().replace(/-/g, '_').replace(/ /g, '_');
+    
+    // Find description in all_tags
+    for (const tag of allTags) {
+      if (tag.name === service) {
+        serviceDesc = tag.description || serviceDesc;
+        break;
+      }
+    }
+  }
+  
+  // Use first significant path segment if discriminator is "path"
+  else if (svcDiscriminator === "path") {
+    const pathParts = pathKey.replace(/^\//, '').split('/');
+    if (pathParts.length > 0) {
+      // Find the first path segment that is not 'api' or 'v{number}'
+      for (const part of pathParts) {
+        const lowerPart = part.toLowerCase();
+        // Skip if it's 'api' or matches version pattern 'v1', 'v2', etc.
+        if (lowerPart === 'api' || /^v\d+$/.test(lowerPart)) {
+          continue;
+        }
+        service = lowerPart.replace(/-/g, '_').replace(/ /g, '_').replace(/\./g, '_');
+        break;
+      }
+      serviceDesc = `${providerName} ${service} API`;
+    }
+  }
+  
+  // Check if service should be skipped
+  if (service === "skip") {
+    return ["skip", ""];
+  }
+  
+  return [service, serviceDesc];
+}
+
+/**
+ * Initialize service map
+ * @param {Object} services - Services map
+ * @param {string[]} componentsChildren - Components children
+ * @param {string} service - Service name
+ * @param {string} serviceDesc - Service description
+ * @param {Object} apiDoc - API doc
+ * @returns {Object} - Updated services map
+ */
+function initService(services, componentsChildren, service, serviceDesc, apiDoc) {
+  services[service] = {
+    openapi: apiDoc.openapi || "3.0.0",
+    info: {
+      title: `${service} API`,
+      description: serviceDesc,
+      version: apiDoc.info?.version || "1.0.0"
+    },
+    paths: {},
+    components: {}
+  };
+  
+  // Initialize components sections
+  services[service].components = {};
+  for (const child of componentsChildren) {
+    services[service].components[child] = {};
+  }
+  
+  // Copy servers if present
+  if (apiDoc.servers) {
+    services[service].servers = apiDoc.servers;
+  }
+  
+  return services;
+}
+
+/**
+ * Extract all $ref values from an object recursively
+ * @param {any} obj - Object to extract refs from
+ * @returns {Set<string>} - Set of refs
+ */
+function getAllRefs(obj) {
+  const refs = new Set();
+  
+  if (typeof obj === 'object' && obj !== null) {
+    if (Array.isArray(obj)) {
+      for (const item of obj) {
+        for (const ref of getAllRefs(item)) {
+          refs.add(ref);
+        }
+      }
+    } else {
+      for (const [key, value] of Object.entries(obj)) {
+        if (key === "$ref" && typeof value === 'string') {
+          refs.add(value);
+        } else if (typeof value === 'object' && value !== null) {
+          for (const ref of getAllRefs(value)) {
+            refs.add(ref);
+          }
+        }
+      }
+    }
+  }
+  
+  return refs;
+}
+
+/**
+ * Add referenced components to service
+ * @param {Set<string>} refs - Set of refs
+ * @param {Object} service - Service object
+ * @param {Object} components - Components from API doc
+ * @param {boolean} debug - Debug flag
+ */
+function addRefsToComponents(refs, service, components, debug) {
+  for (const ref of refs) {
+    const parts = ref.split('/');
+    
+    // Only process refs that point to components
+    if (parts.length >= 4 && parts[1] === "components") {
+      const componentType = parts[2];
+      const componentName = parts[3];
+      
+      // Check if component type exists in service
+      if (!service.components[componentType]) {
+        service.components[componentType] = {};
+      }
+      
+      // Skip if component already added
+      if (service.components[componentType][componentName]) {
+        continue;
+      }
+      
+      // Add component if it exists in source document
+      if (components[componentType] && components[componentType][componentName]) {
+        service.components[componentType][componentName] = components[componentType][componentName];
+        if (debug) {
+          logger.debug(`Added component ${componentType}/${componentName}`);
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Add missing type: object to schema objects
+ * @param {any} obj - Object to process
+ * @returns {any} - Processed object
+ */
+function addMissingObjectTypes(obj) {
+  if (typeof obj !== 'object' || obj === null) {
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    return obj.map(item => addMissingObjectTypes(item));
+  }
+  
+  // If it has properties but no type, add type: object
+  if (obj.properties && !obj.type) {
+    obj.type = "object";
+  }
+  
+  // Process nested objects
+  for (const [key, value] of Object.entries(obj)) {
+    if (typeof value === 'object' && value !== null) {
+      obj[key] = addMissingObjectTypes(value);
+    }
+  }
+  
+  return obj;
+}
+
+/**
+ * Split OpenAPI document into service-specific files
+ * @param {Object} options - Options for splitting
+ * @returns {Promise<boolean>} - Success status
+ */
+export async function split(options) {
+  const {
+    apiDoc,
+    providerName,
+    outputDir,
+    svcDiscriminator = "tag",
+    exclude = null,
+    overwrite = true,
+    verbose = false
+  } = options;
+  
+  // Setup logging based on verbosity
+  if (verbose) {
+    logger.level = 'debug';
+  }
+  
+  logger.info(`📄 Splitting OpenAPI doc for ${providerName}`);
+  logger.info(`API Doc: ${apiDoc}`);
+  logger.info(`Output: ${outputDir}`);
+  logger.info(`Service Discriminator: ${svcDiscriminator}`);
+  
+  // Process exclude list
+  const excludeList = exclude ? exclude.split(",") : [];
+  
+  // Read the OpenAPI document
+  let apiDocObj;
+  try {
+    const apiDocContent = fs.readFileSync(apiDoc, 'utf8');
+    apiDocObj = yaml.load(apiDocContent);
+  } catch (e) {
+    logger.error(`❌ Failed to parse ${apiDoc}: ${e.message}`);
+    return false;
+  }
+  
+  // Create destination directory
+  if (!createDestDir(outputDir, overwrite)) {
+    return false;
+  }
+  
+  // Get API paths
+  const apiPaths = apiDocObj.paths || {};
+  logger.info(`🔑 Iterating over ${Object.keys(apiPaths).length} paths`);
+  
+  const services = {};
+  let opCounter = 0;
+  
+  // Process each path
+  for (const [pathKey, pathItem] of Object.entries(apiPaths)) {
+    if (verbose) {
+      logger.debug(`Processing path ${pathKey}`);
+    }
+    
+    if (!pathItem) {
+      continue;
+    }
+    
+    // Process each operation (HTTP verb)
+    for (const [verbKey, opItem] of Object.entries(pathItem)) {
+      if (!OPERATIONS.includes(verbKey) || !opItem) {
+        continue;
+      }
+      
+      opCounter += 1;
+      if (opCounter % 100 === 0) {
+        logger.info(`⚙️ Operations processed: ${opCounter}`);
+      }
+      
+      if (verbose) {
+        logger.debug(`Processing operation ${pathKey}:${verbKey}`);
+      }
+      
+      // Skip excluded operations
+      if (isOperationExcluded(excludeList, opItem, svcDiscriminator)) {
+        continue;
+      }
+      
+      // Determine service name
+      const [service, serviceDesc] = retServiceNameAndDesc(
+        providerName, opItem, pathKey, svcDiscriminator, 
+        apiDocObj.tags || [], verbose
+      );
+      
+      // Skip if service is marked to skip
+      if (service === 'skip') {
+        logger.warn(`⭐️ Skipping service: ${service}`);
+        continue;
+      }
+      
+      if (verbose) {
+        logger.debug(`Service name: ${service}`);
+        logger.debug(`Service desc: ${serviceDesc}`);
+      }
+      
+      // Initialize service if first occurrence
+      if (!services[service]) {
+        if (verbose) {
+          logger.debug(`First occurrence of ${service}`);
+        }
+        initService(services, COMPONENTS_CHILDREN, service, serviceDesc, apiDocObj);
+      }
+      
+      // Add operation to service
+      if (!services[service].paths[pathKey]) {
+        if (verbose) {
+          logger.debug(`First occurrence of ${pathKey}`);
+        }
+        services[service].paths[pathKey] = {};
+      }
+      
+      services[service].paths[pathKey][verbKey] = opItem;
+      
+      // Special case for GitHub
+      if (providerName === 'github' && 
+          opItem['x-github'] && 
+          opItem['x-github'].subcategory) {
+        services[service].paths[pathKey][verbKey]['x-stackQL-resource'] = camelToSnake(
+          opItem['x-github'].subcategory
+        );
+      }
+      
+      // Get all refs for operation
+      const opRefs = getAllRefs(opItem);
+      
+      if (verbose) {
+        logger.debug(`Found ${opRefs.size} refs for ${service}`);
+      }
+      
+      // Add refs to components
+      addRefsToComponents(opRefs, services[service], apiDocObj.components || {}, verbose);
+      
+      // Get internal refs
+      for (let i = 0; i < 3; i++) {  // Internal ref depth
+        const intRefs = getAllRefs(services[service].components);
+        if (verbose) {
+          logger.debug(`Found ${intRefs.size} INTERNAL refs for service ${service}`);
+        }
+        addRefsToComponents(intRefs, services[service], apiDocObj.components || {}, verbose);
+      }
+      
+      // Get deeply nested schema refs
+      for (let i = 0; i < 10; i++) {  // Schema max ref depth
+        const intRefs = getAllRefs(services[service].components);
+        // Filter refs that are already in service components
+        const filteredRefs = new Set();
+        for (const ref of intRefs) {
+          const parts = ref.split('/');
+          if (parts.length >= 4 && parts[1] === "components" && parts[2] === "schemas" &&
+              !services[service].components.schemas[parts[3]]) {
+            filteredRefs.add(ref);
+          }
+        }
+        
+        if (verbose) {
+          logger.debug(`Found ${filteredRefs.size} INTERNAL schema refs for service ${service}`);
+        }
+        
+        if (filteredRefs.size > 0) {
+          if (verbose) {
+            logger.debug(`Adding ${filteredRefs.size} INTERNAL schema refs for service ${service}`);
+          }
+          addRefsToComponents(filteredRefs, services[service], apiDocObj.components || {}, verbose);
+        } else {
+          if (verbose) {
+            logger.debug(`Exiting INTERNAL schema refs for ${service}`);
+          }
+          break;
+        }
+      }
+    }
+  }
+  
+  // Add non-operations to each service
+  for (const service in services) {
+    for (const pathKey of Object.keys(services[service].paths)) {
+      if (verbose) {
+        logger.debug(`Adding non operations to ${service} for path ${pathKey}`);
+      }
+      
+      for (const nonOp of NON_OPERATIONS) {
+        if (verbose) {
+          logger.debug(`Looking for non operation ${nonOp} in ${service} under path ${pathKey}`);
+        }
+        
+        if (apiPaths[pathKey] && apiPaths[pathKey][nonOp]) {
+          if (verbose) {
+            logger.debug(`Adding ${nonOp} to ${service} for path ${pathKey}`);
+          }
+          
+          // Special case for parameters
+          if (nonOp === 'parameters') {
+            for (const verbKey in services[service].paths[pathKey]) {
+              services[service].paths[pathKey][verbKey].parameters = apiPaths[pathKey].parameters;
+            }
+          }
+        }
+      }
+    }
+  }
+  
+  // Update path param names (replace hyphens with underscores)
+  for (const service in services) {
+    if (services[service].paths) {
+      const pathKeys = Object.keys(services[service].paths);
+      for (const pathKey of pathKeys) {
+        if (verbose) {
+          logger.debug(`Renaming path params in ${service} for path ${pathKey}`);
+        }
+        
+        // Replace hyphens with underscores in path parameters
+        const updatedPathKey = pathKey.replace(/(?<=\{)([^}]+?)-([^}]+?)(?=\})/g, '$1_$2');
+        
+        if (updatedPathKey !== pathKey) {
+          if (verbose) {
+            logger.debug(`Updated path key from ${pathKey} to ${updatedPathKey}`);
+          }
+          
+          services[service].paths[updatedPathKey] = services[service].paths[pathKey];
+          delete services[service].paths[pathKey];
+          
+          // Also update parameter names in operations
+          for (const verbKey in services[service].paths[updatedPathKey]) {
+            const operation = services[service].paths[updatedPathKey][verbKey];
+            if (operation.parameters) {
+              for (const param of operation.parameters) {
+                if (param.in === 'path' && param.name.includes('-')) {
+                  const originalName = param.name;
+                  param.name = param.name.replace(/-/g, '_');
+                  if (verbose) {
+                    logger.debug(`Updated parameter name from ${originalName} to ${param.name} in path ${updatedPathKey}`);
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  
+  // Fix missing type: object
+  for (const service in services) {
+    if (verbose) {
+      logger.debug(`Updating paths for ${service}`);
+    }
+    services[service].paths = addMissingObjectTypes(services[service].paths);
+    services[service].components = addMissingObjectTypes(services[service].components);
+  }
+  
+  // Write out service docs
+  for (const service in services) {
+    logger.info(`✅ Writing out OpenAPI doc for [${service}]`);
+
+    // const svcDir = path.join(outputDir, service);
+    // const outputFile = path.join(svcDir, `${service}.yaml`);
+    const outputFile = path.join(outputDir, `${service}.yaml`);    
+    // fs.mkdirSync(svcDir, { recursive: true });
+    
+    fs.writeFileSync(outputFile, yaml.dump(services[service], { 
+      noRefs: true,
+      sortKeys: false
+    }));
+  }
+  
+  logger.info(`🎉 Successfully split OpenAPI doc into ${Object.keys(services).length} services`);
+  return true;
+}

package/src/utils.js

@@ -0,0 +1,42 @@
+// Add to utils.js at the top:
+import fs from 'fs';
+import logger from './logger.js';
+
+/**
+ * Convert camelCase to snake_case
+ * @param {string} name - The string to convert
+ * @returns {string} - Converted string in snake_case
+ */
+export function camelToSnake(name) {
+  const s1 = name.replace(/([a-z0-9])([A-Z][a-z]+)/g, '$1_$2');
+  return s1.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase();
+}
+
+/**
+ * Create destination directory
+ * @param {string} destDir - Destination directory path
+ * @param {boolean} overwrite - Whether to overwrite existing directory
+ * @returns {boolean} - Success status
+ */
+export function createDestDir(destDir, overwrite) {
+  if (fs.existsSync(destDir)) {
+    if (!overwrite) {
+      logger.error(`Destination directory ${destDir} already exists. Use --overwrite to force.`);
+      return false;
+    }
+    
+    // Clean the directory if overwrite is true
+    try {
+      // Remove all files and subdirectories
+      fs.rmSync(destDir, { recursive: true, force: true });
+      logger.info(`Cleaned destination directory ${destDir}`);
+    } catch (error) {
+      logger.error(`Failed to clean destination directory ${destDir}: ${error.message}`);
+      return false;
+    }
+  }
+  
+  // Create the directory (or recreate it if it was deleted)
+  fs.mkdirSync(destDir, { recursive: true });
+  return true;
+}