@stackql/provider-utils 0.2.3 → 0.3.0

package/README.md

@@ -154,9 +154,16 @@ components:
 ### 3. Run the Test
 
 ```bash
-node tests/docgen/test-docgen.js
+node tests/docgen/test-docgen.js snowflake
+node tests/docgen/test-docgen.js google
+node tests/docgen/test-docgen.js homebrew
 ```
 
+
+node tests/providerdev/test-split.js okta tests/providerdev/split-source/okta/management-minimal.yaml path
+node tests/providerdev/test-analyze.js okta
+node tests/providerdev/test-generate.js okta
+
 ## Using the Documentation Generator
 
 ### Basic Example

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackql/provider-utils",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "description": "Utilities for building StackQL providers from OpenAPI specifications.",
   "type": "module",
   "main": "./src/index.js",
@@ -36,6 +36,7 @@
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.1",
     "@stackql/deno-openapi-dereferencer": "npm:@jsr/stackql__deno-openapi-dereferencer@^0.3.1",
+    "csv-parser": "^3.2.0",
     "js-yaml": "^4.1.0",
     "pluralize": "^8.0.0"
   },

package/src/docgen/generator.js

@@ -13,6 +13,7 @@ export async function generateDocs(options) {
         providerDir,        // e.g., 'output/src/heroku/v00.00.00000'
         outputDir,          // e.g., 'website'
         providerDataDir,    // e.g., 'config/provider-data'
+        dereferenced = false,
     } = options;
 
     console.log(`documenting ${providerName}...`);
@@ -60,7 +61,7 @@ export async function generateDocs(options) {
         const filePath = path.join(serviceDir, file);
         totalServicesCount++;
         const serviceFolder = `${servicesDir}/${serviceName}`;
-        await createDocsForService(filePath, providerName, serviceName, serviceFolder);
+        await createDocsForService(filePath, providerName, serviceName, serviceFolder, dereferenced);
     }
 
     console.log(`Processed ${totalServicesCount} services`);
@@ -116,7 +117,7 @@ ${servicesToMarkdown(providerName, secondColumnServices)}
 }
 
 // Process each service sequentially
-async function createDocsForService(yamlFilePath, providerName, serviceName, serviceFolder) {
+async function createDocsForService(yamlFilePath, providerName, serviceName, serviceFolder, dereferenced = false) {
 
     const data = yaml.load(fs.readFileSync(yamlFilePath, 'utf8'));
 
@@ -126,12 +127,17 @@ async function createDocsForService(yamlFilePath, providerName, serviceName, ser
     const ignorePaths = ["$.components.x-stackQL-resources"];
     let dereferencedAPI;
 
-    try {
-        // dereferencedAPI = await deno_openapi_dereferencer.dereferenceApi(api, "$", ignorePaths);
-        dereferencedAPI = await SwaggerParser.dereference(api);
-        dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
-    } catch (error) {
-        console.error("error in dereferencing or flattening:", error);
+    if (dereferenced) {
+        // If API is already dereferenced, just use it as is
+        dereferencedAPI = api;
+    } else {
+        try {
+            // Only dereference and flatten if needed
+            dereferencedAPI = await SwaggerParser.dereference(api);
+            dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
+        } catch (error) {
+            console.error("error in dereferencing or flattening:", error);
+        }
     }
 
     // Create service directory
@@ -154,11 +160,21 @@ async function createDocsForService(yamlFilePath, providerName, serviceName, ser
             console.warn(`No 'id' defined for resource: ${resourceName} in service: ${serviceName}`);
             continue;
         }
-    
+
+        const resourceDescription = resourceData.description || '';
+
+        // Determine if it's a View or a Resource
+        let resourceType = "Resource"; // Default type
+        if (resourceData.config?.views?.select) {
+            resourceType = "View";
+        }
+        
         resources.push({
             name: resourceName,
+            description: resourceDescription,
+            type: resourceType,
             resourceData,
-            dereferencedAPI
+            dereferencedAPI,
         });
     }    
 
@@ -197,9 +213,7 @@ async function processResource(providerName, serviceFolder, serviceName, resourc
     const resourceIndexContent = await createResourceIndexContent(
         providerName, 
         serviceName, 
-        resource.name, 
-        resource.resourceData, 
-        resource.dereferencedAPI,
+        resource,
     );
     fs.writeFileSync(resourceIndexPath, resourceIndexContent);
 

package/src/docgen/helpers.js

@@ -397,28 +397,6 @@ function getHttpOperationInfo(dereferencedAPI, path, httpVerb, mediaType, openAP
     };
 }
 
-// function getHttpRespBody(schema, objectKey) {
-
-//     if (schema.type === 'array') {
-//         return {
-//          respProps: schema.items.properties || {},
-//          respDescription: schema.items.description || '',
-//         }
-//     } else if (schema.type === 'object') {
-//         return {
-//          respProps: schema.properties || {},
-//          respDescription: schema.description || '',
-//         };
-//     } else {
-//         return {
-//             respProps: {},
-//             respDescription: '',
-//         };
-//     }
-
-
-// }
-
 function getHttpRespBody(schema, objectKey) {
 
     if (schema.type === 'array') {
@@ -435,12 +413,20 @@ function getHttpRespBody(schema, objectKey) {
                 const parts = objectKey.split('[*]');
                 const complexObjectKey = parts[1].replace('.', '');
                 console.log(`Item of Interest : ${complexObjectKey}`);
-                const respProps = schema.properties.items.additionalProperties.properties[complexObjectKey].items.properties;
-                console.info(respProps);
-                const respDescription = schema.properties.items.additionalProperties.properties[complexObjectKey].items.description || schema.properties.items.description || '';
-                console.log(respDescription);
+
+                // Safe access to respProps
+                const respProps = schema?.properties?.items?.additionalProperties?.properties?.[complexObjectKey]?.items?.properties ?? {};
+
+                // Safe access to respDescription with fallbacks
+                const respDescription = 
+                    schema?.properties?.items?.additionalProperties?.properties?.[complexObjectKey]?.items?.description ?? 
+                    schema?.properties?.items?.description ?? 
+                    '';
+
+                // console.info(respProps);
+                // console.log(respDescription);
                 return {
-                    respProps: respProps || {},
+                    respProps: respProps,
                     respDescription: respDescription,
                 };
 
@@ -448,12 +434,20 @@ function getHttpRespBody(schema, objectKey) {
                 // simple object key
                 console.log(`Simple Object Key : ${objectKey}`);
                 const simpleObjectKey = objectKey.replace('$.', '');
-                const respProps = schema.properties[simpleObjectKey].items.properties;
-                console.info(respProps);
-                const respDescription = schema.properties[simpleObjectKey].items.description || schema.description || '';
-                console.log(respDescription);
+
+
+                const respProps = (schema?.properties?.[simpleObjectKey]?.items?.properties) ?? 
+                                (schema?.properties?.[simpleObjectKey]?.properties) ?? 
+                                {};
+
+                const respDescription = (schema?.properties?.[simpleObjectKey]?.items?.description) ?? 
+                                        (schema?.description) ?? 
+                                        '';
+
+                // console.info(respProps);
+                // console.log(respDescription);
                 return {
-                    respProps: respProps || {},
+                    respProps: respProps,
                     respDescription: respDescription,
                 };
             }

package/src/docgen/resource/fields.js

@@ -3,48 +3,54 @@ import {
     getSqlMethodsWithOrderedFields, 
     sanitizeHtml,
 } from '../helpers.js';
+import { docView } from './view.js';
 
 const mdCodeAnchor = "`";
 
-export function createFieldsSection(resourceData, dereferencedAPI) {
+export function createFieldsSection(resourceType, resourceData, dereferencedAPI) {
     let content = '## Fields\n\n';
 
-    content += 'The following fields are returned by `SELECT` queries:\n\n';
-
-    // Use the reusable function to get methods with ordered fields
-    const methods = getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, 'select');
+    if(resourceType === 'Resource'){
     
-    if (Object.keys(methods).length > 0) {
-        // Create the tabs and markdown content
-        const methodNames = Object.keys(methods);
-        
-        // Create the tab values array for the Tabs component
-        const tabValues = methodNames.map(methodName => {
-            return `{ label: '${methodName}', value: '${methodName}' }`;
-        }).join(',\n        ');
+        content += 'The following fields are returned by `SELECT` queries:\n\n';
+
+        // Use the reusable function to get methods with ordered fields
+        const methods = getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, 'select');
         
-        // Start building the Tabs component
-        content += `<Tabs
+        if (Object.keys(methods).length > 0) {
+            // Create the tabs and markdown content
+            const methodNames = Object.keys(methods);
+            
+            // Create the tab values array for the Tabs component
+            const tabValues = methodNames.map(methodName => {
+                return `{ label: '${methodName}', value: '${methodName}' }`;
+            }).join(',\n        ');
+            
+            // Start building the Tabs component
+            content += `<Tabs
     defaultValue="${methodNames[0]}"
     values={[
         ${tabValues}
     ]}
 >\n`;
         
-        // Create the TabItems with table content
-        for (const methodName of methodNames) {
-            const methodData = methods[methodName];
-            
-            // Start the TabItem
-            content += `<TabItem value="${methodName}">\n\n`;
-            
-            // Add the method description if available
-            if (methodData.respDescription && methodData.respDescription.trim().toUpperCase() !== 'OK') {
-                content += `${sanitizeHtml(methodData.respDescription)}\n\n`;
-            }
-            
-            // Add the table header
-            content += `<table>
+            // Create the TabItems with table content
+            for (const methodName of methodNames) {
+                const methodData = methods[methodName];
+                
+                // Start the TabItem
+                content += `<TabItem value="${methodName}">\n\n`;
+                
+                // Add the method description if available
+                if (methodData.respDescription 
+                    && methodData.respDescription.trim().toUpperCase() !== 'OK' 
+                    && methodData.respDescription.trim() !== 'Successful response'
+                ) {
+                    content += `${sanitizeHtml(methodData.respDescription)}\n\n`;
+                }
+                
+                // Add the table header
+                content += `<table>
 <thead>
     <tr>
     <th>Name</th>
@@ -54,28 +60,34 @@ export function createFieldsSection(resourceData, dereferencedAPI) {
 </thead>
 <tbody>`;            
 
-            // Add each property as a row in the table
-            for (const [propName, propData] of Object.entries(methodData.properties)) {
-                content += `\n<tr>
+                // Add each property as a row in the table
+                for (const [propName, propData] of Object.entries(methodData.properties)) {
+                    content += `\n<tr>
     <td><CopyableCode code="${propName}" /></td>
     <td><code>${propData.type}</code></td>
     <td>${sanitizeHtml(propData.description)}</td>
 </tr>`;
-            }
+                }
             
-            content += `\n</tbody>
+                content += `\n</tbody>
 </table>
 `;
 
-            // Close the TabItem
-            content += `</TabItem>\n`;
+                // Close the TabItem
+                content += `</TabItem>\n`;
+            }
+            
+            // Close the Tabs component
+            content += `</Tabs>\n`;
+        } else {
+            // no fields
+            content += `${mdCodeAnchor}SELECT${mdCodeAnchor} not supported for this resource, use ${mdCodeAnchor}SHOW METHODS${mdCodeAnchor} to view available operations for the resource.\n\n`;
         }
-        
-        // Close the Tabs component
-        content += `</Tabs>\n`;
+
     } else {
-        // no fields
-        content += `${mdCodeAnchor}SELECT${mdCodeAnchor} not supported for this resource, use ${mdCodeAnchor}SHOW METHODS${mdCodeAnchor} to view available operations for the resource.\n\n`;
+        // its a view
+        console.log(`processing view : ${resourceData.name}...`)
+        content += docView(resourceData);
     }
 
     return content;

package/src/docgen/resource/overview.js

@@ -3,7 +3,7 @@ import {
     getIndefiniteArticle, 
 } from '../helpers.js';
 
-export function createOverviewSection(resourceName, providerName, serviceName) {
+export function createOverviewSection(resourceName, resourceType, resourceDescription, providerName, serviceName) {
 
   let content = `--- 
 title: ${resourceName}
@@ -25,12 +25,16 @@ import CopyableCode from '@site/src/components/CopyableCode/CopyableCode';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-Creates, updates, deletes, gets or lists ${getIndefiniteArticle(resourceName)} <code>${resourceName}</code> resource.
+`;
+
+content += resourceDescription ? resourceDescription : `Creates, updates, deletes, gets or lists ${getIndefiniteArticle(resourceName)} <code>${resourceName}</code> resource.`;
+
+content += `
 
 ## Overview
 <table><tbody>
 <tr><td><b>Name</b></td><td><code>${resourceName}</code></td></tr>
-<tr><td><b>Type</b></td><td>Resource</td></tr>
+<tr><td><b>Type</b></td><td>${resourceType}</td></tr>
 <tr><td><b>Id</b></td><td><CopyableCode code="${providerName}.${serviceName}.${resourceName}" /></td></tr>
 </tbody></table>
 

package/src/docgen/resource/view.js

@@ -0,0 +1,130 @@
+// src/docgen/resource/view.js
+export function docView(resourceData) {
+
+    let content = '';
+
+    const fields = resourceData.config?.views?.fields ?? [];
+
+    if (fields.length === 0) {
+        content += `See the SQL Definition (view DDL) for fields returned by this view.\n\n`;
+    } else {
+        // Add the table
+        content += `The following fields are returned by this view:\n\n`;
+        content += `<table>
+<thead>
+    <tr>
+    <th>Name</th>
+    <th>Datatype</th>
+    <th>Description</th>
+    </tr>
+</thead>
+<tbody>`;
+
+        for (const field of fields) {
+            content += `
+<tr>
+    <td>${field.name}</td>
+    <td>${field.type}</td>
+    <td>${field.description}</td>
+</tr>`;
+        }
+
+        // Close the table
+        content += `
+</tbody>
+</table>\n\n`;
+    }
+
+    // Add required params section if exists
+    const requiredParams = resourceData.config?.views?.requiredParams ?? [];
+    if (requiredParams.length > 0) {
+        // add the table
+        content += `## Required Parameters\n\n`;
+        content += `The following parameters are required by this view:\n\n`;
+        content += `<table>
+<thead>
+    <tr>
+    <th>Name</th>
+    <th>Datatype</th>
+    <th>Description</th>
+    </tr>
+</thead>
+<tbody>`;
+
+        for (const param of requiredParams) {
+            content += `
+<tr>
+    <td>${param.name}</td>
+    <td>${param.type}</td>
+    <td>${param.description}</td>
+</tr>`;
+        }
+
+        // Close the table
+        content += `
+</tbody>
+</table>\n\n`;
+    }
+
+    // SQL Definition section
+    content += `## SQL Definition\n\n`;
+    
+    // Build array of dialect objects
+    const dialects = [];
+    let currentSelect = resourceData.config.views.select;
+    
+    // Add primary dialect
+    dialects.push({
+        name: extractDialectName(currentSelect.predicate),
+        ddl: currentSelect.ddl
+    });
+    
+    // Add fallback dialects
+    while (currentSelect.fallback) {
+        currentSelect = currentSelect.fallback;
+        dialects.push({
+            name: extractDialectName(currentSelect.predicate),
+            ddl: currentSelect.ddl
+        });
+    }
+    
+    // Create the tabbed interface
+    const tabValues = dialects.map(dialect => (
+`{ label: '${dialect.name}', value: '${dialect.name}' }`
+    )).join(',\n');
+    
+    content += `<Tabs
+defaultValue="${dialects[0].name}"
+values={[
+${tabValues}
+]}
+>\n`;
+    
+    // Create tab content
+    for (const dialect of dialects) {
+        content += `<TabItem value="${dialect.name}">\n\n`;
+        content += `\`\`\`sql
+${dialect.ddl}
+\`\`\`\n\n`;
+        content += `</TabItem>\n`;
+    }
+    
+    content += `</Tabs>\n`;
+
+    return content;
+
+}
+
+// Extract and format dialect name from predicate
+function extractDialectName(predicate) {
+    if (!predicate) {
+        return 'Default';
+    }
+    const dialectMatch = predicate.match(/sqlDialect\s*==\s*['"](.*?)['"]/);
+    if (!dialectMatch || !dialectMatch[1]) {
+        throw new Error(`Invalid dialect predicate: ${predicate}`);
+    }
+    
+    // Capitalize first letter of dialect name
+    return dialectMatch[1].charAt(0).toUpperCase() + dialectMatch[1].slice(1);
+}

package/src/docgen/resource-content.js

@@ -9,16 +9,14 @@ import { createExamplesSection } from './resource/examples.js';
 export async function createResourceIndexContent(
     providerName, 
     serviceName, 
-    resourceName, 
-    resourceData, 
-    dereferencedAPI,
+    resource,
 ) {
     // Generate each section of the documentation
-    const overviewContent = createOverviewSection(resourceName, providerName, serviceName);
-    const fieldsContent = createFieldsSection(resourceData, dereferencedAPI);
-    const methodsContent = createMethodsSection(resourceData, dereferencedAPI);
-    const paramsContent = createParamsSection(resourceData, dereferencedAPI);
-    const examplesContent = createExamplesSection(providerName, serviceName, resourceName, resourceData, dereferencedAPI);
+    const overviewContent = createOverviewSection(resource.name, resource.type, resource.description, providerName, serviceName);
+    const fieldsContent = createFieldsSection(resource.type, resource.resourceData, resource.dereferencedAPI);
+    const methodsContent = resource.type === 'Resource' ? createMethodsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const paramsContent = resource.type === 'Resource' ? createParamsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const examplesContent = resource.type === 'Resource' ? createExamplesSection(providerName, serviceName, resource.name, resource.resourceData, resource.dereferencedAPI) : '';
 
     // Combine all sections into the final content
     return `${overviewContent}${fieldsContent}${methodsContent}${paramsContent}${examplesContent}`;

package/src/index.js

@@ -1,4 +1,5 @@
-// src/index.js
+// @stackql/provider-utils/src/index.js
 
 // Main entry point for Node.js
-export * as docgen from './docgen/index.js';
+export * as docgen from './docgen/index.js';
+export * as providerdev from './providerdev/index.js';

package/src/logger.js

@@ -0,0 +1,52 @@
+// @stackql/provider-utils/src/logger.js
+
+// Simple logger implementation
+const logLevels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  debug: 3
+};
+
+let currentLevel = 'info';
+
+const logger = {
+  get level() {
+    return currentLevel;
+  },
+  
+  set level(newLevel) {
+    if (logLevels[newLevel] !== undefined) {
+      currentLevel = newLevel;
+    } else {
+      console.warn(`Invalid log level: ${newLevel}. Using 'info' instead.`);
+      currentLevel = 'info';
+    }
+  },
+  
+  error: (message) => {
+    if (logLevels[currentLevel] >= logLevels.error) {
+      console.error(`ERROR: ${message}`);
+    }
+  },
+  
+  warn: (message) => {
+    if (logLevels[currentLevel] >= logLevels.warn) {
+      console.warn(`WARNING: ${message}`);
+    }
+  },
+  
+  info: (message) => {
+    if (logLevels[currentLevel] >= logLevels.info) {
+      console.info(`INFO: ${message}`);
+    }
+  },
+  
+  debug: (message) => {
+    if (logLevels[currentLevel] >= logLevels.debug) {
+      console.debug(`DEBUG: ${message}`);
+    }
+  }
+};
+
+export default logger;