@stackql/provider-utils 0.2.2 → 0.2.4

package/README.md

@@ -154,7 +154,9 @@ components:
 ### 3. Run the Test
 
 ```bash
-node tests/docgen/test-docgen.js
+node tests/docgen/test-docgen.js snowflake
+node tests/docgen/test-docgen.js google
+node tests/docgen/test-docgen.js homebrew
 ```
 
 ## Using the Documentation Generator

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackql/provider-utils",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "Utilities for building StackQL providers from OpenAPI specifications.",
   "type": "module",
   "main": "./src/index.js",
@@ -35,7 +35,7 @@
   "license": "MIT",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.1",
-    "@stackql/deno-openapi-dereferencer": "npm:@jsr/stackql__deno-openapi-dereferencer@^0.3.0",
+    "@stackql/deno-openapi-dereferencer": "npm:@jsr/stackql__deno-openapi-dereferencer@^0.3.1",
     "js-yaml": "^4.1.0",
     "pluralize": "^8.0.0"
   },

package/src/docgen/generator.js

@@ -13,6 +13,7 @@ export async function generateDocs(options) {
         providerDir,        // e.g., 'output/src/heroku/v00.00.00000'
         outputDir,          // e.g., 'website'
         providerDataDir,    // e.g., 'config/provider-data'
+        dereferenced = false,
     } = options;
 
     console.log(`documenting ${providerName}...`);
@@ -60,7 +61,7 @@ export async function generateDocs(options) {
         const filePath = path.join(serviceDir, file);
         totalServicesCount++;
         const serviceFolder = `${servicesDir}/${serviceName}`;
-        await createDocsForService(filePath, providerName, serviceName, serviceFolder);
+        await createDocsForService(filePath, providerName, serviceName, serviceFolder, dereferenced);
     }
 
     console.log(`Processed ${totalServicesCount} services`);
@@ -116,7 +117,7 @@ ${servicesToMarkdown(providerName, secondColumnServices)}
 }
 
 // Process each service sequentially
-async function createDocsForService(yamlFilePath, providerName, serviceName, serviceFolder) {
+async function createDocsForService(yamlFilePath, providerName, serviceName, serviceFolder, dereferenced = false) {
 
     const data = yaml.load(fs.readFileSync(yamlFilePath, 'utf8'));
 
@@ -126,12 +127,17 @@ async function createDocsForService(yamlFilePath, providerName, serviceName, ser
     const ignorePaths = ["$.components.x-stackQL-resources"];
     let dereferencedAPI;
 
-    try {
-        // dereferencedAPI = await deno_openapi_dereferencer.dereferenceApi(api, "$", ignorePaths);
-        dereferencedAPI = await SwaggerParser.dereference(api);
-        dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
-    } catch (error) {
-        console.error("error in dereferencing or flattening:", error);
+    if (dereferenced) {
+        // If API is already dereferenced, just use it as is
+        dereferencedAPI = api;
+    } else {
+        try {
+            // Only dereference and flatten if needed
+            dereferencedAPI = await SwaggerParser.dereference(api);
+            dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
+        } catch (error) {
+            console.error("error in dereferencing or flattening:", error);
+        }
     }
 
     // Create service directory
@@ -154,11 +160,21 @@ async function createDocsForService(yamlFilePath, providerName, serviceName, ser
             console.warn(`No 'id' defined for resource: ${resourceName} in service: ${serviceName}`);
             continue;
         }
-    
+
+        const resourceDescription = resourceData.description || '';
+
+        // Determine if it's a View or a Resource
+        let resourceType = "Resource"; // Default type
+        if (resourceData.config?.views?.select) {
+            resourceType = "View";
+        }
+        
         resources.push({
             name: resourceName,
+            description: resourceDescription,
+            type: resourceType,
             resourceData,
-            dereferencedAPI
+            dereferencedAPI,
         });
     }    
 
@@ -197,9 +213,7 @@ async function processResource(providerName, serviceFolder, serviceName, resourc
     const resourceIndexContent = await createResourceIndexContent(
         providerName, 
         serviceName, 
-        resource.name, 
-        resource.resourceData, 
-        resource.dereferencedAPI,
+        resource,
     );
     fs.writeFileSync(resourceIndexPath, resourceIndexContent);
 

package/src/docgen/helpers.js

@@ -405,18 +405,64 @@ function getHttpRespBody(schema, objectKey) {
          respDescription: schema.items.description || '',
         }
     } else if (schema.type === 'object') {
-        return {
-         respProps: schema.properties || {},
-         respDescription: schema.description || '',
-        };
+        if(objectKey){
+            // if objectKey contains [*] print something
+            if (objectKey.includes('[*]')) {
+                // complex object key
+                console.log(`Complex Object Key : ${objectKey}`);
+                const parts = objectKey.split('[*]');
+                const complexObjectKey = parts[1].replace('.', '');
+                console.log(`Item of Interest : ${complexObjectKey}`);
+
+                // Safe access to respProps
+                const respProps = schema?.properties?.items?.additionalProperties?.properties?.[complexObjectKey]?.items?.properties ?? {};
+
+                // Safe access to respDescription with fallbacks
+                const respDescription = 
+                    schema?.properties?.items?.additionalProperties?.properties?.[complexObjectKey]?.items?.description ?? 
+                    schema?.properties?.items?.description ?? 
+                    '';
+
+                // console.info(respProps);
+                // console.log(respDescription);
+                return {
+                    respProps: respProps,
+                    respDescription: respDescription,
+                };
+
+            } else {
+                // simple object key
+                console.log(`Simple Object Key : ${objectKey}`);
+                const simpleObjectKey = objectKey.replace('$.', '');
+
+
+                const respProps = (schema?.properties?.[simpleObjectKey]?.items?.properties) ?? 
+                                (schema?.properties?.[simpleObjectKey]?.properties) ?? 
+                                {};
+
+                const respDescription = (schema?.properties?.[simpleObjectKey]?.items?.description) ?? 
+                                        (schema?.description) ?? 
+                                        '';
+
+                // console.info(respProps);
+                // console.log(respDescription);
+                return {
+                    respProps: respProps,
+                    respDescription: respDescription,
+                };
+            }
+        } else {
+                return {
+                    respProps: schema.properties || {},
+                    respDescription: schema.description || '',
+                };
+        }
     } else {
         return {
             respProps: {},
             respDescription: '',
         };
     }
-
-
 }
 
 function getHttpOperationParams(dereferencedAPI, path, httpVerb) {

package/src/docgen/resource/fields.js

@@ -3,48 +3,54 @@ import {
     getSqlMethodsWithOrderedFields, 
     sanitizeHtml,
 } from '../helpers.js';
+import { docView } from './view.js';
 
 const mdCodeAnchor = "`";
 
-export function createFieldsSection(resourceData, dereferencedAPI) {
+export function createFieldsSection(resourceType, resourceData, dereferencedAPI) {
     let content = '## Fields\n\n';
 
-    content += 'The following fields are returned by `SELECT` queries:\n\n';
-
-    // Use the reusable function to get methods with ordered fields
-    const methods = getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, 'select');
+    if(resourceType === 'Resource'){
     
-    if (Object.keys(methods).length > 0) {
-        // Create the tabs and markdown content
-        const methodNames = Object.keys(methods);
-        
-        // Create the tab values array for the Tabs component
-        const tabValues = methodNames.map(methodName => {
-            return `{ label: '${methodName}', value: '${methodName}' }`;
-        }).join(',\n        ');
+        content += 'The following fields are returned by `SELECT` queries:\n\n';
+
+        // Use the reusable function to get methods with ordered fields
+        const methods = getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, 'select');
         
-        // Start building the Tabs component
-        content += `<Tabs
+        if (Object.keys(methods).length > 0) {
+            // Create the tabs and markdown content
+            const methodNames = Object.keys(methods);
+            
+            // Create the tab values array for the Tabs component
+            const tabValues = methodNames.map(methodName => {
+                return `{ label: '${methodName}', value: '${methodName}' }`;
+            }).join(',\n        ');
+            
+            // Start building the Tabs component
+            content += `<Tabs
     defaultValue="${methodNames[0]}"
     values={[
         ${tabValues}
     ]}
 >\n`;
         
-        // Create the TabItems with table content
-        for (const methodName of methodNames) {
-            const methodData = methods[methodName];
-            
-            // Start the TabItem
-            content += `<TabItem value="${methodName}">\n\n`;
-            
-            // Add the method description if available
-            if (methodData.respDescription && methodData.respDescription.trim().toUpperCase() !== 'OK') {
-                content += `${sanitizeHtml(methodData.respDescription)}\n\n`;
-            }
-            
-            // Add the table header
-            content += `<table>
+            // Create the TabItems with table content
+            for (const methodName of methodNames) {
+                const methodData = methods[methodName];
+                
+                // Start the TabItem
+                content += `<TabItem value="${methodName}">\n\n`;
+                
+                // Add the method description if available
+                if (methodData.respDescription 
+                    && methodData.respDescription.trim().toUpperCase() !== 'OK' 
+                    && methodData.respDescription.trim() !== 'Successful response'
+                ) {
+                    content += `${sanitizeHtml(methodData.respDescription)}\n\n`;
+                }
+                
+                // Add the table header
+                content += `<table>
 <thead>
     <tr>
     <th>Name</th>
@@ -54,28 +60,34 @@ export function createFieldsSection(resourceData, dereferencedAPI) {
 </thead>
 <tbody>`;            
 
-            // Add each property as a row in the table
-            for (const [propName, propData] of Object.entries(methodData.properties)) {
-                content += `\n<tr>
+                // Add each property as a row in the table
+                for (const [propName, propData] of Object.entries(methodData.properties)) {
+                    content += `\n<tr>
     <td><CopyableCode code="${propName}" /></td>
     <td><code>${propData.type}</code></td>
     <td>${sanitizeHtml(propData.description)}</td>
 </tr>`;
-            }
+                }
             
-            content += `\n</tbody>
+                content += `\n</tbody>
 </table>
 `;
 
-            // Close the TabItem
-            content += `</TabItem>\n`;
+                // Close the TabItem
+                content += `</TabItem>\n`;
+            }
+            
+            // Close the Tabs component
+            content += `</Tabs>\n`;
+        } else {
+            // no fields
+            content += `${mdCodeAnchor}SELECT${mdCodeAnchor} not supported for this resource, use ${mdCodeAnchor}SHOW METHODS${mdCodeAnchor} to view available operations for the resource.\n\n`;
         }
-        
-        // Close the Tabs component
-        content += `</Tabs>\n`;
+
     } else {
-        // no fields
-        content += `${mdCodeAnchor}SELECT${mdCodeAnchor} not supported for this resource, use ${mdCodeAnchor}SHOW METHODS${mdCodeAnchor} to view available operations for the resource.\n\n`;
+        // its a view
+        console.log(`processing view : ${resourceData.name}...`)
+        content += docView(resourceData);
     }
 
     return content;

package/src/docgen/resource/overview.js

@@ -3,7 +3,7 @@ import {
     getIndefiniteArticle, 
 } from '../helpers.js';
 
-export function createOverviewSection(resourceName, providerName, serviceName) {
+export function createOverviewSection(resourceName, resourceType, resourceDescription, providerName, serviceName) {
 
   let content = `--- 
 title: ${resourceName}
@@ -25,12 +25,16 @@ import CopyableCode from '@site/src/components/CopyableCode/CopyableCode';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-Creates, updates, deletes, gets or lists ${getIndefiniteArticle(resourceName)} <code>${resourceName}</code> resource.
+`;
+
+content += resourceDescription ? resourceDescription : `Creates, updates, deletes, gets or lists ${getIndefiniteArticle(resourceName)} <code>${resourceName}</code> resource.`;
+
+content += `
 
 ## Overview
 <table><tbody>
 <tr><td><b>Name</b></td><td><code>${resourceName}</code></td></tr>
-<tr><td><b>Type</b></td><td>Resource</td></tr>
+<tr><td><b>Type</b></td><td>${resourceType}</td></tr>
 <tr><td><b>Id</b></td><td><CopyableCode code="${providerName}.${serviceName}.${resourceName}" /></td></tr>
 </tbody></table>
 

package/src/docgen/resource/view.js

@@ -0,0 +1,130 @@
+// src/docgen/resource/view.js
+export function docView(resourceData) {
+
+    let content = '';
+
+    const fields = resourceData.config?.views?.fields ?? [];
+
+    if (fields.length === 0) {
+        content += `See the SQL Definition (view DDL) for fields returned by this view.\n\n`;
+    } else {
+        // Add the table
+        content += `The following fields are returned by this view:\n\n`;
+        content += `<table>
+<thead>
+    <tr>
+    <th>Name</th>
+    <th>Datatype</th>
+    <th>Description</th>
+    </tr>
+</thead>
+<tbody>`;
+
+        for (const field of fields) {
+            content += `
+<tr>
+    <td>${field.name}</td>
+    <td>${field.type}</td>
+    <td>${field.description}</td>
+</tr>`;
+        }
+
+        // Close the table
+        content += `
+</tbody>
+</table>\n\n`;
+    }
+
+    // Add required params section if exists
+    const requiredParams = resourceData.config?.views?.requiredParams ?? [];
+    if (requiredParams.length > 0) {
+        // add the table
+        content += `## Required Parameters\n\n`;
+        content += `The following parameters are required by this view:\n\n`;
+        content += `<table>
+<thead>
+    <tr>
+    <th>Name</th>
+    <th>Datatype</th>
+    <th>Description</th>
+    </tr>
+</thead>
+<tbody>`;
+
+        for (const param of requiredParams) {
+            content += `
+<tr>
+    <td>${param.name}</td>
+    <td>${param.type}</td>
+    <td>${param.description}</td>
+</tr>`;
+        }
+
+        // Close the table
+        content += `
+</tbody>
+</table>\n\n`;
+    }
+
+    // SQL Definition section
+    content += `## SQL Definition\n\n`;
+    
+    // Build array of dialect objects
+    const dialects = [];
+    let currentSelect = resourceData.config.views.select;
+    
+    // Add primary dialect
+    dialects.push({
+        name: extractDialectName(currentSelect.predicate),
+        ddl: currentSelect.ddl
+    });
+    
+    // Add fallback dialects
+    while (currentSelect.fallback) {
+        currentSelect = currentSelect.fallback;
+        dialects.push({
+            name: extractDialectName(currentSelect.predicate),
+            ddl: currentSelect.ddl
+        });
+    }
+    
+    // Create the tabbed interface
+    const tabValues = dialects.map(dialect => (
+`{ label: '${dialect.name}', value: '${dialect.name}' }`
+    )).join(',\n');
+    
+    content += `<Tabs
+defaultValue="${dialects[0].name}"
+values={[
+${tabValues}
+]}
+>\n`;
+    
+    // Create tab content
+    for (const dialect of dialects) {
+        content += `<TabItem value="${dialect.name}">\n\n`;
+        content += `\`\`\`sql
+${dialect.ddl}
+\`\`\`\n\n`;
+        content += `</TabItem>\n`;
+    }
+    
+    content += `</Tabs>\n`;
+
+    return content;
+
+}
+
+// Extract and format dialect name from predicate
+function extractDialectName(predicate) {
+    if (!predicate) {
+        return 'Default';
+    }
+    const dialectMatch = predicate.match(/sqlDialect\s*==\s*['"](.*?)['"]/);
+    if (!dialectMatch || !dialectMatch[1]) {
+        throw new Error(`Invalid dialect predicate: ${predicate}`);
+    }
+    
+    // Capitalize first letter of dialect name
+    return dialectMatch[1].charAt(0).toUpperCase() + dialectMatch[1].slice(1);
+}

package/src/docgen/resource-content.js

@@ -9,16 +9,14 @@ import { createExamplesSection } from './resource/examples.js';
 export async function createResourceIndexContent(
     providerName, 
     serviceName, 
-    resourceName, 
-    resourceData, 
-    dereferencedAPI,
+    resource,
 ) {
     // Generate each section of the documentation
-    const overviewContent = createOverviewSection(resourceName, providerName, serviceName);
-    const fieldsContent = createFieldsSection(resourceData, dereferencedAPI);
-    const methodsContent = createMethodsSection(resourceData, dereferencedAPI);
-    const paramsContent = createParamsSection(resourceData, dereferencedAPI);
-    const examplesContent = createExamplesSection(providerName, serviceName, resourceName, resourceData, dereferencedAPI);
+    const overviewContent = createOverviewSection(resource.name, resource.type, resource.description, providerName, serviceName);
+    const fieldsContent = createFieldsSection(resource.type, resource.resourceData, resource.dereferencedAPI);
+    const methodsContent = resource.type === 'Resource' ? createMethodsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const paramsContent = resource.type === 'Resource' ? createParamsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const examplesContent = resource.type === 'Resource' ? createExamplesSection(providerName, serviceName, resource.name, resource.resourceData, resource.dereferencedAPI) : '';
 
     // Combine all sections into the final content
     return `${overviewContent}${fieldsContent}${methodsContent}${paramsContent}${examplesContent}`;