@stackql/provider-utils 0.5.4 → 0.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackql/provider-utils",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Utilities for building StackQL providers from OpenAPI specifications.",
   "type": "module",
   "main": "./src/index.js",

package/src/docgen/generator.js

@@ -3,7 +3,7 @@
 import fs from 'fs';
 import path from 'path';
 import yaml from 'js-yaml';
-import { createResourceIndexContent } from './resource-content.js';
+import { createResourceIndexContent, createResourceIndexContentv2 } from './resource-content.js';
 import SwaggerParser from '@apidevtools/swagger-parser';
 import * as deno_openapi_dereferencer from "@stackql/deno-openapi-dereferencer";
 
@@ -284,6 +284,222 @@ function generateResourceLinks(providerName, serviceName, resources) {
     return resourceLinks.join('<br />\n');
 }
 
+export async function generateDocsv2(options) {
+    const {
+        providerName,
+        providerDir,        // e.g., 'output/src/heroku/v00.00.00000'
+        outputDir,          // e.g., 'website'
+        providerDataDir,    // e.g., 'config/provider-data'
+        dereferenced = false,
+    } = options;
+
+    console.log(`documenting ${providerName} (v2)...`);
+
+    const docsDir = path.join(outputDir, `docs`);
+    const servicesDir = path.join(docsDir, `services`);
+
+    // Remove directory if it exists, then create it fresh
+    fs.existsSync(docsDir) && fs.rmSync(docsDir, { recursive: true, force: true });
+    fs.mkdirSync(servicesDir, { recursive: true });
+
+    // Check for provider data files
+    console.log(providerDataDir);
+    try {
+        const files = fs.readdirSync(providerDataDir);
+        console.log('Files in providerDataDir:', files);
+    } catch (err) {
+        console.error('Error reading providerDataDir:', err.message);
+    }
+
+    const headerContent1Path = path.join(providerDataDir, 'headerContent1.txt');
+    const headerContent2Path = path.join(providerDataDir, 'headerContent2.txt');
+
+    if (!fs.existsSync(headerContent1Path) || !fs.existsSync(headerContent2Path)) {
+        throw new Error(`Missing headerContent1.txt or headerContent2.txt in ${providerDataDir}`);
+    }
+
+    const headerContent1 = fs.readFileSync(headerContent1Path, 'utf8');
+    const headerContent2 = fs.readFileSync(headerContent2Path, 'utf8');
+
+    // Initialize counters
+    let servicesForIndex = [];
+    let totalServicesCount = 0;
+    let totalResourcesCount = 0;
+
+    // Process services
+    const serviceDir = path.join(providerDir, 'services');
+    console.log(`Processing services in ${serviceDir}...`);
+    const serviceFiles = fs.readdirSync(serviceDir).filter(file => path.extname(file) === '.yaml');
+
+    for (const file of serviceFiles) {
+        const serviceName = path.basename(file, '.yaml').replace(/-/g, '_');
+        console.log(`Processing service: ${serviceName}`);
+        servicesForIndex.push(serviceName);
+        const filePath = path.join(serviceDir, file);
+        totalServicesCount++;
+        const serviceFolder = `${servicesDir}/${serviceName}`;
+        await createDocsForServicev2(filePath, providerName, serviceName, serviceFolder, dereferenced);
+    }
+
+    console.log(`Processed ${totalServicesCount} services`);
+
+    // Count total resources
+    totalResourcesCount = fs.readdirSync(`${servicesDir}`, { withFileTypes: true })
+        .filter(dirent => dirent.isDirectory())
+        .map(dirent => fs.readdirSync(`${servicesDir}/${dirent.name}`).length)
+        .reduce((a, b) => a + b, 0);
+
+    console.log(`Processed ${totalResourcesCount} resources`);
+
+    // Create provider index
+    servicesForIndex = [...new Set(servicesForIndex)];
+    servicesForIndex.sort();
+
+    const half = Math.ceil(servicesForIndex.length / 2);
+    const firstColumnServices = servicesForIndex.slice(0, half);
+    const secondColumnServices = servicesForIndex.slice(half);
+
+    const indexContent = `${headerContent1}
+
+:::info[Provider Summary]
+
+total services: __${totalServicesCount}__
+total resources: __${totalResourcesCount}__
+
+:::
+
+${headerContent2}
+
+## Services
+<div class="row">
+<div class="providerDocColumn">
+${servicesToMarkdown(providerName, firstColumnServices)}
+</div>
+<div class="providerDocColumn">
+${servicesToMarkdown(providerName, secondColumnServices)}
+</div>
+</div>
+`;
+
+    // Write index
+    const indexPath = path.join(docsDir, 'index.md');
+    fs.writeFileSync(indexPath, indexContent);
+    console.log(`Index file created at ${indexPath}`);
+
+    return {
+        totalServices: totalServicesCount,
+        totalResources: totalResourcesCount,
+        outputPath: docsDir
+    };
+}
+
+// v2 service processing - uses SchemaTable for collapsible nested fields
+async function createDocsForServicev2(yamlFilePath, providerName, serviceName, serviceFolder, dereferenced = false) {
+
+    const data = yaml.load(fs.readFileSync(yamlFilePath, 'utf8'));
+
+    // Create a new SwaggerParser instance
+    let parser = new SwaggerParser();
+    const api = await parser.parse(yamlFilePath);
+    const ignorePaths = ["$.components.x-stackQL-resources"];
+    let dereferencedAPI;
+
+    if (dereferenced) {
+        // If API is already dereferenced, just use it as is
+        dereferencedAPI = api;
+    } else {
+        try {
+            // Only dereference and flatten if needed
+            dereferencedAPI = await SwaggerParser.dereference(api);
+            dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
+        } catch (error) {
+            console.error("error in dereferencing or flattening:", error);
+        }
+    }
+
+    // Create service directory
+    if (!fs.existsSync(serviceFolder)) {
+        fs.mkdirSync(serviceFolder, { recursive: true });
+    }
+
+    const resourcesObj = data.components['x-stackQL-resources'];
+
+    if (!resourcesObj) {
+        console.warn(`No resources found in ${yamlFilePath}`);
+        return;
+    }
+
+    const resources = [];
+    for (let resourceName in resourcesObj) {
+
+        let resourceData = resourcesObj[resourceName];
+        if (!resourceData.id) {
+            console.warn(`No 'id' defined for resource: ${resourceName} in service: ${serviceName}`);
+            continue;
+        }
+
+        const resourceDescription = resourceData.description || '';
+
+        // Determine if it's a View or a Resource
+        let resourceType = "Resource"; // Default type
+        if (resourceData.config?.views?.select) {
+            resourceType = "View";
+        }
+
+        resources.push({
+            name: resourceName,
+            description: resourceDescription,
+            type: resourceType,
+            resourceData,
+            dereferencedAPI,
+        });
+    }
+
+    // Process service index
+    const serviceIndexPath = path.join(serviceFolder, 'index.md');
+    const serviceIndexContent = await createServiceIndexContent(providerName, serviceName, resources);
+    fs.writeFileSync(serviceIndexPath, serviceIndexContent);
+
+    // Split into columns and process resources one by one
+    const halfLength = Math.ceil(resources.length / 2);
+    const firstColumn = resources.slice(0, halfLength);
+    const secondColumn = resources.slice(halfLength);
+
+    // Process each resource in first column
+    for (const resource of firstColumn) {
+        await processResourcev2(providerName, serviceFolder, serviceName, resource);
+    }
+
+    // Process each resource in second column
+    for (const resource of secondColumn) {
+        await processResourcev2(providerName, serviceFolder, serviceName, resource);
+    }
+
+    console.log(`Generated documentation (v2) for ${serviceName}`);
+}
+
+async function processResourcev2(providerName, serviceFolder, serviceName, resource) {
+    console.log(`Processing resource (v2): ${resource.name}`);
+
+    const resourceFolder = path.join(serviceFolder, resource.name);
+    if (!fs.existsSync(resourceFolder)) {
+        fs.mkdirSync(resourceFolder, { recursive: true });
+    }
+
+    const resourceIndexPath = path.join(resourceFolder, 'index.md');
+    const resourceIndexContent = await createResourceIndexContentv2(
+        providerName,
+        serviceName,
+        resource,
+    );
+    fs.writeFileSync(resourceIndexPath, resourceIndexContent);
+
+    // After writing the file, force garbage collection if available (optional)
+    if (global.gc) {
+        global.gc();
+    }
+}
+
 // Function to convert services to markdown links
 function servicesToMarkdown(providerName, servicesList) {
     return servicesList.map(service => `<a href="/services/${service}/">${service}</a><br />`).join('\n');

package/src/docgen/helpers.js

@@ -90,7 +90,7 @@ export function getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, sq
         // Get all SQL verb methods
         const allSqlMethodNames = new Set();
         const sqlVerbTypes = ['select', 'insert', 'update', 'delete', 'replace'];
-        
+
         for (const verb of sqlVerbTypes) {
             if (resourceData.sqlVerbs[verb] && resourceData.sqlVerbs[verb].length > 0) {
                 for (const method of resourceData.sqlVerbs[verb]) {
@@ -99,32 +99,32 @@ export function getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, sq
                 }
             }
         }
-        
+
         // Process each method that's not in any SQL verb
         for (const [methodName, methodData] of Object.entries(resourceData.methods)) {
             if (!allSqlMethodNames.has(methodName)) {
                 const { path, httpVerb, mediaType, openAPIDocKey } = methodData.operation;
                 let resolvedPath = path;
                 let resolvedVerb = httpVerb;
-                
+
                 // If operation uses $ref, resolve it
                 if (methodData.operation.$ref) {
                     const refPath = methodData.operation.$ref;
-                    
+
                     // Extract the path and verb from the $ref
                     // The path format is typically '#/paths/~1api~1v2~1accounts~1{name}:undrop/post'
                     const pathMatch = refPath.match(/#\/paths\/(.+)\/([^/]+)$/);
-                    
+
                     if (pathMatch && pathMatch.length === 3) {
                         // Replace the escaped characters in the path
                         let path = pathMatch[1]
                             .replace(/~1/g, '/') // Replace ~1 with /
                             .replace(/~0/g, '~') // Replace ~0 with ~ if needed
-                        
+
                         // Don't modify path parts with special characters like ':undrop'
                         resolvedPath = path;
                         resolvedVerb = pathMatch[2];
-                        
+
                         console.log(`Resolved path: ${resolvedPath}, verb: ${resolvedVerb}`);
                     } else {
                         console.warn(`Could not parse $ref path: ${refPath}`);
@@ -135,20 +135,20 @@ export function getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, sq
 
                 // Get response and params using the same function as for SQL verbs
                 const { respProps, respDescription, opDescription, requestBody } = getHttpOperationInfo(
-                    dereferencedAPI, 
-                    resolvedPath, 
-                    resolvedVerb, 
-                    methodData.response.mediaType || '', 
+                    dereferencedAPI,
+                    resolvedPath,
+                    resolvedVerb,
+                    methodData.response.mediaType || '',
                     methodData.response.openAPIDocKey || '200',
                     ''
                 );
-                
+
                 const { requiredParams, optionalParams } = getHttpOperationParams(
-                    dereferencedAPI, 
-                    resolvedPath, 
+                    dereferencedAPI,
+                    resolvedPath,
                     resolvedVerb
                 );
-                
+
                 // Initialize the method with the same structure as SQL methods
                 methods[methodName] = {
                     opDescription,
@@ -157,16 +157,17 @@ export function getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, sq
                     requiredParams: requiredParams || {},
                     optionalParams: optionalParams || {},
                     requestBody: requestBody || {},
+                    rawRespProps: respProps,
                 };
-                
+
                 // Format and sort the properties using our helper functions
                 const allProperties = formatProperties(respProps);
                 sortAndAddProperties(methods[methodName], allProperties);
-                
+
                 console.info(`Processed exec method: ${methodName}`);
             }
         }
-        
+
         return methods;
     }
 
@@ -180,22 +181,23 @@ export function getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, sq
         const {requiredParams, optionalParams} = getHttpOperationParams(dereferencedAPI, path, httpVerb);
 
         // Initialize the method object with description and params
-        methods[methodName] = { 
+        methods[methodName] = {
             opDescription,
             respDescription,
             properties: {},
             requiredParams: requiredParams || {},
             optionalParams: optionalParams || {},
             requestBody: requestBody || {},
+            rawRespProps: respProps,
         };
-        
+
         // Format and sort the properties using our helper functions
         const allProperties = formatProperties(respProps);
         sortAndAddProperties(methods[methodName], allProperties);
-        
+
         console.info(`Processed method: ${methodName}`);
     }
-    
+
     return methods;
 }
 
@@ -569,6 +571,81 @@ function getHttpRespBody(schema, objectKey) {
     }
 }
 
+/**
+ * Recursively generates a nested JSON schema tree from response properties,
+ * suitable for use with the SchemaTable React component.
+ * Each node has: { name, type, description, children? }
+ * @param {Object} respProps - The raw response properties from the dereferenced API
+ * @param {number} depth - Current recursion depth
+ * @param {number} maxDepth - Maximum recursion depth to prevent infinite loops
+ * @returns {Array} Array of schema field objects
+ */
+export function generateSchemaJsonFromProps(respProps, depth = 0, maxDepth = 4) {
+    if (depth >= maxDepth || !respProps || typeof respProps !== 'object') return [];
+
+    return Object.entries(respProps).map(([propName, prop]) => {
+        if (!prop) return null;
+
+        let propType = prop.type || 'object';
+        let propDesc = sanitizeHtml(prop.description || '');
+
+        // Add format info to type string if available
+        if (prop.format) {
+            propType += ` (${prop.format})`;
+        }
+
+        let children = [];
+
+        // Handle arrays - get children from items
+        if (prop.type === 'array' && prop.items) {
+            if (prop.items.properties) {
+                children = generateSchemaJsonFromProps(prop.items.properties, depth + 1, maxDepth);
+            }
+        }
+
+        // Handle objects - get children from properties
+        if (prop.type === 'object' && prop.properties) {
+            children = generateSchemaJsonFromProps(prop.properties, depth + 1, maxDepth);
+        }
+
+        return {
+            name: propName,
+            type: propType,
+            description: propDesc,
+            ...(children.length > 0 && { children })
+        };
+    }).filter(Boolean);
+}
+
+/**
+ * Sorts an array of schema field objects using the standard field priority:
+ * 1. Exact 'id' and 'name' fields
+ * 2. Fields ending with '_id'
+ * 3. Fields ending with '_name'
+ * 4. All other fields (alphabetical)
+ * @param {Array} fields - Array of schema field objects with 'name' property
+ * @returns {Array} Sorted array of schema field objects
+ */
+export function sortSchemaFields(fields) {
+    if (!fields || fields.length === 0) return [];
+
+    const exactIdName = fields.filter(f => f.name === 'id' || f.name === 'name');
+    const idSuffix = fields.filter(f => f.name !== 'id' && f.name.endsWith('_id'));
+    const nameSuffix = fields.filter(f => f.name !== 'name' && f.name.endsWith('_name'));
+    const others = fields.filter(f =>
+        !exactIdName.includes(f) &&
+        !idSuffix.includes(f) &&
+        !nameSuffix.includes(f)
+    );
+
+    return [
+        ...exactIdName.sort((a, b) => a.name.localeCompare(b.name)),
+        ...idSuffix.sort((a, b) => a.name.localeCompare(b.name)),
+        ...nameSuffix.sort((a, b) => a.name.localeCompare(b.name)),
+        ...others.sort((a, b) => a.name.localeCompare(b.name)),
+    ];
+}
+
 function getHttpOperationParams(dereferencedAPI, path, httpVerb) {
     const requiredParams = {};
     const optionalParams = {};

package/src/docgen/index.js

@@ -1,5 +1,5 @@
 // src/docgen/index.js
 
 // Export all documentation generation functions
-export { generateDocs } from './generator.js';
-export { createResourceIndexContent } from './resource-content.js';
+export { generateDocs, generateDocsv2 } from './generator.js';
+export { createResourceIndexContent, createResourceIndexContentv2 } from './resource-content.js';

package/src/docgen/resource/fields.js

@@ -1,7 +1,9 @@
 // src/docgen/resource/fields.js
-import { 
-    getSqlMethodsWithOrderedFields, 
+import {
+    getSqlMethodsWithOrderedFields,
     sanitizeHtml,
+    generateSchemaJsonFromProps,
+    sortSchemaFields,
 } from '../helpers.js';
 import { docView } from './view.js';
 
@@ -100,3 +102,62 @@ export function createFieldsSection(resourceType, resourceData, dereferencedAPI)
     return content;
 }
 
+export function createFieldsSectionv2(resourceType, resourceData, dereferencedAPI) {
+    let content = '## Fields\n\n';
+
+    if (resourceType === 'Resource') {
+
+        content += 'The following fields are returned by `SELECT` queries:\n\n';
+
+        // Use the reusable function to get methods with ordered fields
+        const methods = getSqlMethodsWithOrderedFields(resourceData, dereferencedAPI, 'select');
+
+        if (Object.keys(methods).length > 0) {
+            const methodNames = Object.keys(methods);
+
+            const tabValues = methodNames.map(methodName => {
+                return `{ label: '${methodName}', value: '${methodName}' }`;
+            }).join(',\n        ');
+
+            content += `<Tabs
+    defaultValue="${methodNames[0]}"
+    values={[
+        ${tabValues}
+    ]}
+>\n`;
+
+            for (const methodName of methodNames) {
+                const methodData = methods[methodName];
+
+                content += `<TabItem value="${methodName}">\n\n`;
+
+                // Add the method description if available and not in the meaningless list
+                if (methodData.respDescription &&
+                    !meaninglessDescriptions.includes(methodData.respDescription.trim().toLowerCase()) &&
+                    methodData.respDescription.trim().length > 0) {
+                    content += `${sanitizeHtml(methodData.respDescription)}\n\n`;
+                }
+
+                // Build nested schema JSON from raw response props and sort fields
+                const schemaFields = generateSchemaJsonFromProps(methodData.rawRespProps);
+                const sortedFields = sortSchemaFields(schemaFields);
+
+                content += `<SchemaTable fields={${JSON.stringify(sortedFields, null, 2)}} />\n`;
+
+                content += `</TabItem>\n`;
+            }
+
+            content += `</Tabs>\n`;
+        } else {
+            content += `${mdCodeAnchor}SELECT${mdCodeAnchor} not supported for this resource, use ${mdCodeAnchor}SHOW METHODS${mdCodeAnchor} to view available operations for the resource.\n\n`;
+        }
+
+    } else {
+        // its a view
+        console.log(`processing view : ${resourceData.name}...`)
+        content += docView(resourceData);
+    }
+
+    return content;
+}
+

package/src/docgen/resource/overview.js

@@ -41,3 +41,43 @@ content += `
 `;
     return content;
 }
+
+export function createOverviewSectionv2(resourceName, resourceType, resourceDescription, providerName, serviceName) {
+
+  let content = `---
+title: ${resourceName}
+hide_title: false
+hide_table_of_contents: false
+keywords:
+  - ${resourceName}
+  - ${serviceName}
+  - ${providerName}
+  - infrastructure-as-code
+  - configuration-as-data
+  - cloud inventory
+description: Query, deploy and manage ${providerName} resources using SQL
+custom_edit_url: null
+image: /img/stackql-${providerName}-provider-featured-image.png
+---
+
+import CopyableCode from '@site/src/components/CopyableCode/CopyableCode';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import SchemaTable from '@site/src/components/SchemaTable/SchemaTable';
+
+`;
+
+content += resourceDescription ? resourceDescription : `Creates, updates, deletes, gets or lists ${getIndefiniteArticle(resourceName)} <code>${resourceName}</code> resource.`;
+
+content += `
+
+## Overview
+<table><tbody>
+<tr><td><b>Name</b></td><td><code>${resourceName}</code></td></tr>
+<tr><td><b>Type</b></td><td>${resourceType}</td></tr>
+<tr><td><b>Id</b></td><td><CopyableCode code="${providerName}.${serviceName}.${resourceName}" /></td></tr>
+</tbody></table>
+
+`;
+    return content;
+}

package/src/docgen/resource-content.js

@@ -1,14 +1,14 @@
 // src/docgen/resource-content.js
 
-import { createOverviewSection } from './resource/overview.js';
-import { createFieldsSection } from './resource/fields.js';
+import { createOverviewSection, createOverviewSectionv2 } from './resource/overview.js';
+import { createFieldsSection, createFieldsSectionv2 } from './resource/fields.js';
 import { createMethodsSection } from './resource/methods.js';
 import { createParamsSection } from './resource/parameters.js';
 import { createExamplesSection } from './resource/examples.js';
- 
+
 export async function createResourceIndexContent(
-    providerName, 
-    serviceName, 
+    providerName,
+    serviceName,
     resource,
 ) {
     // Generate each section of the documentation
@@ -21,3 +21,19 @@ export async function createResourceIndexContent(
     // Combine all sections into the final content
     return `${overviewContent}${fieldsContent}${methodsContent}${paramsContent}${examplesContent}`;
 }
+
+export async function createResourceIndexContentv2(
+    providerName,
+    serviceName,
+    resource,
+) {
+    // Generate each section of the documentation (v2 uses SchemaTable for fields)
+    const overviewContent = createOverviewSectionv2(resource.name, resource.type, resource.description, providerName, serviceName);
+    const fieldsContent = createFieldsSectionv2(resource.type, resource.resourceData, resource.dereferencedAPI);
+    const methodsContent = resource.type === 'Resource' ? createMethodsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const paramsContent = resource.type === 'Resource' ? createParamsSection(resource.resourceData, resource.dereferencedAPI) : '';
+    const examplesContent = resource.type === 'Resource' ? createExamplesSection(providerName, serviceName, resource.name, resource.resourceData, resource.dereferencedAPI) : '';
+
+    // Combine all sections into the final content
+    return `${overviewContent}${fieldsContent}${methodsContent}${paramsContent}${examplesContent}`;
+}