@stackql/provider-utils 0.2.3 → 0.3.0

package/src/providerdev/analyze.js

@@ -0,0 +1,169 @@
+// src/providerdev/analyze.js
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import logger from '../logger.js';
+import { camelToSnake } from '../utils.js';
+
+/**
+ * Load specification from YAML or JSON file
+ * @param {string} filepath - Path to specification file
+ * @returns {Object} - Loaded specification
+ */
+function loadSpec(filepath) {
+  const content = fs.readFileSync(filepath, 'utf-8');
+  if (filepath.endsWith('.json')) {
+    return JSON.parse(content);
+  }
+  return yaml.load(content);
+}
+
+/**
+ * Extract main 2xx response schema reference
+ * @param {Object} responseObj - Response object
+ * @returns {string} - Schema reference name
+ */
+function extractMain2xxResponse(responseObj) {
+  for (const [code, response] of Object.entries(responseObj)) {
+    if (code.startsWith('2')) {
+      const content = response.content || {};
+      const appJson = content['application/json'] || {};
+      const schema = appJson.schema || {};
+
+      // Case 1: Direct $ref
+      if (schema.$ref) {
+        return schema.$ref.split('/').pop();
+      }
+
+      // Case 2: Array of items
+      if (schema.type === 'array') {
+        const items = schema.items || {};
+        if (items.$ref) {
+          return items.$ref.split('/').pop();
+        }
+      }
+      
+      return '';
+    }
+  }
+  return '';
+}
+
+/**
+ * Find existing mapping in x-stackQL-resources
+ * @param {Object} spec - OpenAPI spec
+ * @param {string} pathRef - Reference to path item
+ * @returns {Object} - Mapping info (resource, method, verb)
+ */
+function findExistingMapping(spec, pathRef) {
+  const stackQLResources = spec.components?.['x-stackQL-resources'] || {};
+  
+  for (const [resourceName, resource] of Object.entries(stackQLResources)) {
+    // Check methods
+    for (const [methodName, method] of Object.entries(resource.methods || {})) {
+      if (method.operation?.$ref === pathRef) {
+        // Find SQL verb for this method
+        let sqlVerb = 'exec'; // Default if no explicit mapping
+        
+        for (const [verb, methods] of Object.entries(resource.sqlVerbs || {})) {
+          for (const methodRef of methods) {
+            if (methodRef.$ref === `#/components/x-stackQL-resources/${resourceName}/methods/${methodName}`) {
+              sqlVerb = verb;
+              break;
+            }
+          }
+        }
+        
+        return {
+          resourceName,
+          methodName,
+          sqlVerb
+        };
+      }
+    }
+  }
+  
+  return {
+    resourceName: '',
+    methodName: '',
+    sqlVerb: ''
+  };
+}
+
+/**
+ * Analyze OpenAPI specs and generate mapping CSV
+ * @param {Object} options - Options for analysis
+ * @returns {Promise<boolean>} - Success status
+ */
+export async function analyze(options) {
+  const {
+    inputDir,
+    outputDir
+  } = options;
+
+  try {
+    fs.mkdirSync(outputDir, { recursive: true });
+    const outputPath = path.join(outputDir, 'all_services.csv');
+    
+    const writer = fs.createWriteStream(outputPath, { encoding: 'utf8' });
+    
+    // Write header
+    writer.write('filename,path,operationId,formatted_op_id,verb,response_object,tags,formatted_tags,stackql_resource_name,stackql_method_name,stackql_verb\n');
+    
+    const files = fs.readdirSync(inputDir);
+    
+    for (const filename of files) {
+      if (!filename.endsWith('.yaml') && !filename.endsWith('.yml') && !filename.endsWith('.json')) {
+        continue;
+      }
+      
+      const filepath = path.join(inputDir, filename);
+      const spec = loadSpec(filepath);
+      
+      for (const [pathKey, pathItem] of Object.entries(spec.paths || {})) {
+        for (const [verb, operation] of Object.entries(pathItem)) {
+          if (typeof operation !== 'object' || operation === null) {
+            continue;
+          }
+          
+          const operationId = operation.operationId || '';
+          // Format operationId as snake_case
+          const formattedOpId = operationId ? camelToSnake(operationId) : '';
+          
+          const responseObj = operation.responses || {};
+          const responseRef = extractMain2xxResponse(responseObj);
+          const tagsList = operation.tags || [];
+          const tagsStr = tagsList.join('|');
+          
+          // Format tags as snake_case
+          const formattedTags = tagsList.map(tag => camelToSnake(tag)).join('|');
+          
+          // Construct the path reference as it would appear in x-stackQL-resources
+          const encodedPath = pathKey.replace(/\//g, '~1');
+          const pathRef = `#/paths/${encodedPath}/${verb}`;
+          
+          // Find existing mapping if available
+          const { resourceName, methodName, sqlVerb } = findExistingMapping(spec, pathRef);
+          
+          // Escape commas in fields
+          const escapedPath = pathKey.includes(',') ? `"${pathKey}"` : pathKey;
+          const escapedOperationId = operationId.includes(',') ? `"${operationId}"` : operationId;
+          const escapedFormattedOpId = formattedOpId.includes(',') ? `"${formattedOpId}"` : formattedOpId;
+          const escapedTags = tagsStr.includes(',') ? `"${tagsStr}"` : tagsStr;
+          const escapedFormattedTags = formattedTags.includes(',') ? `"${formattedTags}"` : formattedTags;
+          
+          // Write row
+          writer.write(`${filename},${escapedPath},${escapedOperationId},${escapedFormattedOpId},${verb},${responseRef},${escapedTags},${escapedFormattedTags},${resourceName},${methodName},${sqlVerb}\n`);
+        }
+      }
+    }
+    
+    writer.end();
+    
+    logger.info(`✅ Analysis complete. Output written to: ${outputPath}`);
+    return true;
+  } catch (error) {
+    logger.error(`Failed to analyze OpenAPI specs: ${error.message}`);
+    return false;
+  }
+}

package/src/providerdev/generate.js

@@ -0,0 +1,307 @@
+// src/providerdev/generate.js
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import csv from 'csv-parser';
+import logger from '../logger.js';
+import { createReadStream } from 'fs';
+
+/**
+ * Load manifest from CSV file
+ * @param {string} configPath - Path to CSV config file
+ * @returns {Promise<Object>} - Manifest object
+ */
+async function loadManifest(configPath) {
+  const manifest = {};
+  
+  return new Promise((resolve, reject) => {
+    createReadStream(configPath)
+      .pipe(csv())
+      .on('data', (row) => {
+        const key = `${row.filename}::${row.operationId}`;
+        manifest[key] = row;
+      })
+      .on('end', () => {
+        resolve(manifest);
+      })
+      .on('error', (error) => {
+        reject(error);
+      });
+  });
+}
+
+/**
+ * Load specification from YAML or JSON file
+ * @param {string} filepath - Path to specification file
+ * @returns {Object} - Loaded specification
+ */
+function loadSpec(filepath) {
+  const content = fs.readFileSync(filepath, 'utf-8');
+  if (filepath.endsWith('.json')) {
+    return JSON.parse(content);
+  }
+  return yaml.load(content);
+}
+
+/**
+ * Write specification to file
+ * @param {string} filepath - Output file path
+ * @param {Object} data - Data to write
+ */
+function writeSpec(filepath, data) {
+  fs.writeFileSync(filepath, yaml.dump(data, { sortKeys: false }));
+}
+
+/**
+ * Encode reference path
+ * @param {string} path - HTTP path
+ * @param {string} verb - HTTP verb
+ * @returns {string} - Encoded reference path
+ */
+function encodeRefPath(path, verb) {
+  const encodedPath = path.replace(/\//g, '~1');
+  return `#/paths/${encodedPath}/${verb}`;
+}
+
+/**
+ * Get success response information
+ * @param {Object} operation - Operation object
+ * @returns {Object} - Response information
+ */
+function getSuccessResponseInfo(operation) {
+  const responses = operation.responses || {};
+  const twoXxCodes = Object.keys(responses)
+    .filter(code => code.startsWith('2'))
+    .sort();
+
+  if (twoXxCodes.length === 0) {
+    return { mediaType: '', openAPIDocKey: '' };
+  }
+
+  const lowest2xx = twoXxCodes[0];
+  const content = responses[lowest2xx]?.content || {};
+  const mediaTypes = Object.keys(content);
+
+  const mediaType = mediaTypes.length > 0 ? mediaTypes[0] : '';
+
+  return {
+    mediaType,
+    openAPIDocKey: lowest2xx
+  };
+}
+
+/**
+ * Convert string to snake_case
+ * @param {string} name - String to convert
+ * @returns {string} - Converted string
+ */
+function snakeCase(name) {
+  return name.replace(/-/g, '_');
+}
+
+/**
+ * Generate StackQL provider extensions
+ * @param {Object} options - Options for generation
+ * @returns {Promise<boolean>} - Success status
+ */
+export async function generate(options) {
+  const {
+    inputDir,
+    outputDir,
+    configPath,
+    providerId,
+    servers = null,
+    providerConfig = null,
+    skipFiles = []
+  } = options;
+
+  const version = 'v00.00.00000';
+  const servicesPath = path.join(outputDir, version, 'services');
+  
+  // Create directories
+  fs.mkdirSync(servicesPath, { recursive: true });
+  
+  // Clean all files in services output dir
+  try {
+    const files = fs.readdirSync(servicesPath);
+    for (const file of files) {
+      const filePath = path.join(servicesPath, file);
+      if (fs.statSync(filePath).isFile()) {
+        fs.unlinkSync(filePath);
+      }
+    }
+    logger.info(`🧹 Cleared all files in ${servicesPath}`);
+  } catch (error) {
+    logger.error(`Failed to clear files in ${servicesPath}: ${error.message}`);
+    return false;
+  }
+  
+  // Delete provider.yaml file
+  const providerManifestFile = path.join(outputDir, version, 'provider.yaml');
+  if (fs.existsSync(providerManifestFile)) {
+    fs.unlinkSync(providerManifestFile);
+    logger.info(`🧹 Deleted ${providerManifestFile}`);
+  }
+  
+  // Load manifest
+  let manifest;
+  try {
+    manifest = await loadManifest(configPath);
+  } catch (error) {
+    logger.error(`Failed to load manifest: ${error.message}`);
+    return false;
+  }
+  
+  const providerServices = {};
+  
+  try {
+    const files = fs.readdirSync(inputDir);
+    
+    for (const filename of files) {
+      if (skipFiles.includes(filename)) {
+        logger.info(`⭐️ Skipping ${filename} (matched --skip)`);
+        continue;
+      }
+      
+      if (!filename.endsWith('.yaml') && !filename.endsWith('.yml') && !filename.endsWith('.json')) {
+        continue;
+      }
+      
+      const baseName = path.basename(filename, path.extname(filename));
+      const serviceName = snakeCase(baseName);
+
+      console.log(`processing service: ${serviceName}`);
+
+      const specPath = path.join(inputDir, filename);
+      const spec = loadSpec(specPath);
+      
+      // Initialize resources object with defaultdict-like behavior
+      const resources = {};
+      
+      for (const [pathKey, pathItem] of Object.entries(spec.paths || {})) {
+        for (const [verb, operation] of Object.entries(pathItem)) {
+          if (typeof operation !== 'object' || operation === null) {
+            continue;
+          }
+          
+          const operationId = operation.operationId;
+          if (!operationId) {
+            continue;
+          }
+          
+          const manifestKey = `${filename}::${operationId}`;
+          const entry = manifest[manifestKey];
+          if (!entry) {
+            logger.error(`❌ ERROR: ${filename} → ${operationId} not found in manifest`);
+            return false;
+          }
+          
+          const resource = entry.stackql_resource_name;
+          const method = entry.stackql_method_name;
+          const sqlverb = entry.stackql_verb;
+          
+          // Initialize resource if it doesn't exist
+          if (!resources[resource]) {
+            resources[resource] = {
+              id: `${providerId}.${serviceName}.${resource}`,
+              name: resource,
+              title: resource.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase()),
+              methods: {},
+              sqlVerbs: { 
+                select: [], 
+                insert: [], 
+                update: [], 
+                delete: [], 
+                replace: [] 
+              }
+            };
+          }
+          
+          const pathRef = encodeRefPath(pathKey, verb);
+          const responseInfo = getSuccessResponseInfo(operation);
+          
+          const methodEntry = {
+            operation: { $ref: pathRef },
+            response: responseInfo
+          };
+
+          resources[resource].methods[method] = methodEntry;
+          if (sqlverb && sqlverb === 'exec') {
+            logger.info(`exec method skipped:  ${resource}.${method}`);
+          } else if (sqlverb && resources[resource].sqlVerbs[sqlverb]) {
+            resources[resource].sqlVerbs[sqlverb].push({
+              $ref: `#/components/x-stackQL-resources/${resource}/methods/${method}`
+            });
+          } else if (sqlverb) {
+            logger.warn(`⚠️ Unknown SQL verb '${sqlverb}' for ${resource}.${method}, skipping`);
+          }
+        }
+      }
+      
+      // Inject into spec
+      if (!spec.components) {
+        spec.components = {};
+      }
+      spec.components['x-stackQL-resources'] = resources;
+      
+      // Inject servers if provided
+      if (servers) {
+        try {
+          const serversJson = JSON.parse(servers);
+          spec.servers = serversJson;
+        } catch (error) {
+          logger.error(`❌ Failed to parse servers JSON: ${error.message}`);
+          return false;
+        }
+      }
+      
+      // Write enriched spec
+      const outputPath = path.join(servicesPath, filename);
+      writeSpec(outputPath, spec);
+      logger.info(`✅ Wrote enriched spec: ${outputPath}`);
+      
+      // Add providerService entry
+      const info = spec.info || {};
+      const specTitle = info.title || `${serviceName.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase())} API`;
+      const specDescription = info.description || `TODO: add description for ${serviceName}`;
+      
+      providerServices[serviceName] = {
+        id: `${serviceName}:${version}`,
+        name: serviceName,
+        preferred: true,
+        service: {
+          $ref: `${providerId}/${version}/services/${filename}`
+        },
+        title: specTitle,
+        version: version,
+        description: specDescription
+      };
+    }
+    
+    // Write provider.yaml
+    const providerYaml = {
+      id: providerId,
+      name: providerId,
+      version: version,
+      providerServices: providerServices,
+    };
+    
+    if (providerConfig) {
+      try {
+        const providerConfigJson = JSON.parse(providerConfig);
+        providerYaml.config = providerConfigJson;
+      } catch (error) {
+        logger.error(`❌ Failed to parse provider config JSON: ${error.message}`);
+        return false;
+      }
+    }
+    
+    writeSpec(path.join(outputDir, version, 'provider.yaml'), providerYaml);
+    logger.info(`📦 Wrote provider.yaml to ${outputDir}/${version}/provider.yaml`);
+    
+    return true;
+  } catch (error) {
+    logger.error(`Failed to generate provider: ${error.message}`);
+    return false;
+  }
+}

package/src/providerdev/index.js

@@ -0,0 +1,10 @@
+// src/providerdev/index.js
+import { split } from './split.js';
+import { generate } from './generate.js';
+import { analyze } from './analyze.js';
+
+export {
+  split,
+  generate,
+  analyze
+};