@stackql/provider-utils 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 StackQL Studios
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,320 @@
+# StackQL Provider Utils
+
+A comprehensive toolkit for transforming OpenAPI specs into StackQL providers. Includes parsing, mapping, validation, testing, and documentation generation utilities.
+
+## Table of Contents
+
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+- [Local Development Setup](#local-development-setup)
+- [Testing with Node.js](#testing-with-nodejs)
+- [Using the Documentation Generator](#using-the-documentation-generator)
+- [API Reference](#api-reference)
+- [Contributing](#contributing)
+
+## Prerequisites
+
+### For Node.js
+- Node.js >= 20
+- npm or yarn
+- StackQL server (for documentation generation)
+
+### Installing StackQL
+
+Download and install StackQL from [stackql.io/downloads](https://stackql.io/downloads)
+
+```bash
+# macOS
+brew install stackql
+
+# Linux
+curl -L https://bit.ly/stackql-zip -O && unzip stackql-zip
+
+# Windows
+# Download from https://stackql.io/downloads
+```
+
+## Installation
+
+### For Node.js Projects
+
+```bash
+npm install @stackql/provider-utils
+# or
+yarn add @stackql/provider-utils
+```
+
+## Local Development Setup
+
+1. Clone the repository:
+```bash
+git clone https://github.com/stackql/stackql-provider-utils.git
+cd stackql-provider-utils
+```
+
+2. Install dependencies (Node.js):
+```bash
+npm install
+```
+
+## Testing with Node.js
+
+### 1. Create a Test Script
+
+Create a file `test-docgen.js`:
+
+```javascript
+import { docgen } from './src/index.js';
+
+// Test the documentation generator
+async function testDocGen() {
+    try {
+        const result = await docgen.generateDocs({
+            providerName: 'myservice',
+            providerDir: './test-data/output/src/myservice/v00.00.00000',
+            outputDir: './test-output',
+            providerDataDir: './test-data/provider-data',
+            stackqlConfig: {
+                host: 'localhost',
+                port: 5444,
+                user: 'stackql',
+                database: 'stackql'
+            }
+        });
+        
+        console.log('Documentation generated successfully:', result);
+    } catch (error) {
+        console.error('Error generating documentation:', error);
+    }
+}
+
+testDocGen();
+```
+
+### 2. Set Up Test Data
+
+Create the required directory structure:
+
+```bash
+mkdir -p test-data/output/src/myservice/v00.00.00000/services
+mkdir -p test-data/provider-data
+```
+
+Add test files:
+
+`test-data/provider-data/headerContent1.txt`:
+```
+---
+title: myservice
+hide_title: false
+hide_table_of_contents: false
+keywords:
+  - myservice
+  - stackql
+  - infrastructure-as-code
+  - configuration-as-data
+description: Query and manage myservice resources using SQL
+---
+
+# myservice Provider
+
+The myservice provider for StackQL allows you to query, deploy, and manage myservice resources using SQL.
+```
+
+`test-data/provider-data/headerContent2.txt`:
+```
+See the [myservice provider documentation](https://myservice.com/docs) for more information.
+```
+
+`test-data/output/src/myservice/v00.00.00000/services/example.yaml`:
+```yaml
+openapi: 3.0.0
+info:
+  title: Example Service
+  version: 1.0.0
+paths:
+  /examples:
+    get:
+      operationId: listExamples
+      responses:
+        '200':
+          description: Success
+components:
+  x-stackQL-resources:
+    examples:
+      id: myservice.example.examples
+      name: examples
+      title: Examples
+      methods:
+        list:
+          operation:
+            $ref: '#/paths/~1examples/get'
+          response:
+            mediaType: application/json
+            openAPIDocKey: '200'
+      sqlVerbs:
+        select:
+          - $ref: '#/components/x-stackQL-resources/examples/methods/list'
+```
+
+### 3. Start StackQL Server
+
+```bash
+# In a separate terminal
+stackql srv \
+  --pgsrv.port=5444 \
+  --pgsrv.tls=false \
+  --loglevel=INFO
+```
+
+### 4. Run the Test
+
+```bash
+sh start-stackql-server.sh tests/docgen
+node tests/docgen/test-docgen.js
+```
+
+## Using the Documentation Generator
+
+### Basic Example
+
+```javascript
+import { docgen } from '@stackql/provider-utils';
+
+const options = {
+    providerName: 'github',
+    providerDir: './output/src/github/v00.00.00000',
+    outputDir: './docs',
+    providerDataDir: './config/provider-data',
+    stackqlConfig: {
+        host: 'localhost',
+        port: 5444,
+        user: 'stackql',
+        database: 'stackql'
+    }
+};
+
+const result = await docgen.generateDocs(options);
+console.log(`Generated docs for ${result.totalServices} services and ${result.totalResources} resources`);
+console.log(`Output location: ${result.outputPath}`);
+```
+
+### Options
+
+| Option | Type | Description | Default |
+|--------|------|-------------|---------|
+| `providerName` | string | Name of the provider (e.g., 'github', 'aws') | Required |
+| `providerDir` | string | Path to provider spec directory | Required |
+| `outputDir` | string | Directory for generated documentation | Required |
+| `providerDataDir` | string | Directory containing provider header files | Required |
+| `stackqlConfig` | object | StackQL server connection configuration | See below |
+
+#### StackQL Config Options
+
+```javascript
+{
+    host: 'localhost',  // StackQL server host
+    port: 5444,         // StackQL server port
+    user: 'stackql',    // Database user
+    database: 'stackql' // Database name
+}
+```
+
+## Directory Structure Requirements
+
+### Provider Data Directory
+```
+provider-data/
+├── headerContent1.txt    # Provider introduction
+├── headerContent2.txt    # Additional provider info
+└── stackql-provider-registry.mdx (optional)
+```
+
+### Provider Spec Directory
+```
+output/src/{provider}/v00.00.00000/
+├── provider.yaml
+└── services/
+    ├── service1.yaml
+    ├── service2.yaml
+    └── ...
+```
+
+### Generated Output
+```
+docs/{provider}-docs/
+├── index.md
+├── stackql-provider-registry.mdx
+└── providers/
+    └── {provider}/
+        └── {service}/
+            ├── index.md
+            └── {resource}/
+                └── index.md
+```
+
+## Troubleshooting
+
+### StackQL Server Connection Issues
+- Ensure StackQL server is running: `stackql srv --pgsrv.port=5444`
+- Check if port 5444 is available
+- Verify connection settings in `stackqlConfig`
+
+### Missing Provider Data
+- Ensure `headerContent1.txt` and `headerContent2.txt` exist in provider data directory
+- Check file permissions
+
+### Empty Documentation
+- Verify provider specs have `x-stackQL-resources` components
+- Check that resources have proper method definitions
+
+## API Reference
+
+### `docgen.generateDocs(options)`
+
+Generates documentation for a StackQL provider.
+
+**Parameters:**
+- `options` (Object): Configuration options
+
+**Returns:**
+- Promise<Object>: Result object containing:
+  - `totalServices`: Number of services processed
+  - `totalResources`: Number of resources documented
+  - `outputPath`: Path to generated documentation
+
+**Example:**
+```javascript
+const result = await docgen.generateDocs({
+    providerName: 'aws',
+    providerDir: './providers/src/aws/v00.00.00000',
+    outputDir: './documentation',
+    providerDataDir: './config/aws',
+    stackqlConfig: {
+        host: 'localhost',
+        port: 5444
+    }
+});
+```
+
+### `docgen.createResourceIndexContent(...)`
+
+Creates markdown content for a single resource. This is a lower-level function used internally by `generateDocs`.
+
+## Contributing
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+## License
+
+MIT
+
+## Support
+
+- [StackQL Documentation](https://stackql.io/docs)
+- [GitHub Issues](https://github.com/stackql/stackql-provider-utils/issues)
+- [StackQL Discord](https://discord.gg/stackql)

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@stackql/provider-utils",
+  "version": "0.1.0",
+  "description": "Utilities for building StackQL providers from OpenAPI specifications.",
+  "type": "module",
+  "main": "./src/index.js",
+  "exports": {
+    ".": {
+      "import": "./src/index.js"
+    },
+    "./docgen": {
+      "import": "./src/docgen/index.js"
+    }
+  },
+  "files": [
+    "src/",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "jest",
+    "lint": "eslint src"
+  },
+  "keywords": [
+    "stackql",
+    "openapi",
+    "infrastructure-as-code",
+    "sql",
+    "cloud",
+    "provider",
+    "documentation"
+  ],
+  "author": "StackQL Studios",
+  "license": "MIT",
+  "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.1",
+    "@stackql/deno-openapi-dereferencer": "npm:@jsr/stackql__deno-openapi-dereferencer@^0.3.0",
+    "js-yaml": "^4.1.0",
+    "pluralize": "^8.0.0"
+  },
+  "devDependencies": {
+    "eslint": "^8.0.0",
+    "jest": "^29.0.0"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/stackql/stackql-provider-utils.git"
+  }
+}

package/src/docgen/generator.js

@@ -0,0 +1,298 @@
+// src/docgen/generator.js
+
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import { createResourceIndexContent } from './resource-content.js';
+import SwaggerParser from '@apidevtools/swagger-parser';
+import * as deno_openapi_dereferencer from "@stackql/deno-openapi-dereferencer";
+
+export async function generateDocs(options) {
+    const {
+        providerName,
+        providerDir,        // e.g., 'output/src/heroku/v00.00.00000'
+        outputDir,          // e.g., 'docs'
+        providerDataDir,    // e.g., 'config/provider-data'
+        // stackqlConfig = {
+        //     host: 'localhost',
+        //     port: 5444,
+        //     user: 'stackql',
+        //     database: 'stackql'
+        // }
+    } = options;
+
+    console.log(`documenting ${providerName}...`);
+
+    const docsDir = path.join(outputDir, `${providerName}-docs`);
+   
+    // Clean existing docs
+    fs.existsSync(`${docsDir}/index.md`) && fs.unlinkSync(`${docsDir}/index.md`);
+    fs.existsSync(`${docsDir}/providers`) && fs.rmSync(`${docsDir}/providers`, { recursive: true, force: true });
+
+    // Check for provider data files
+console.log(providerDataDir);
+try {
+    const files = fs.readdirSync(providerDataDir);
+    console.log('Files in providerDataDir:', files);
+} catch (err) {
+    console.error('Error reading providerDataDir:', err.message);
+}
+
+
+
+
+    const headerContent1Path = path.join(providerDataDir, 'headerContent1.txt');
+    const headerContent2Path = path.join(providerDataDir, 'headerContent2.txt');
+    const mdxPath = path.join(providerDataDir, 'stackql-provider-registry.mdx');
+
+    if (!fs.existsSync(headerContent1Path) || !fs.existsSync(headerContent2Path)) {
+        throw new Error(`Missing headerContent1.txt or headerContent2.txt in ${providerDataDir}`);
+    }
+
+    const headerContent1 = fs.readFileSync(headerContent1Path, 'utf8');
+    const headerContent2 = fs.readFileSync(headerContent2Path, 'utf8');
+
+    // Initialize counters
+    let servicesForIndex = [];
+    let totalServicesCount = 0;
+    let totalResourcesCount = 0;
+
+    // Process services
+    const serviceDir = path.join(providerDir, 'services');
+    console.log(`Processing services in ${serviceDir}...`);
+    const serviceFiles = fs.readdirSync(serviceDir).filter(file => path.extname(file) === '.yaml');
+
+    for (const file of serviceFiles) {
+        const serviceName = path.basename(file, '.yaml').replace(/-/g, '_');
+        console.log(`Processing service: ${serviceName}`);
+        servicesForIndex.push(serviceName);
+        const filePath = path.join(serviceDir, file);
+        totalServicesCount++;
+        const serviceFolder = `${docsDir}/providers/${providerName}/${serviceName}`;
+        await createDocsForService(filePath, providerName, serviceName, serviceFolder);
+    }
+
+    console.log(`Processed ${totalServicesCount} services`);
+
+    // Count total resources
+    totalResourcesCount = fs.readdirSync(`${docsDir}/providers/${providerName}`, { withFileTypes: true })
+        .filter(dirent => dirent.isDirectory())
+        .map(dirent => fs.readdirSync(`${docsDir}/providers/${providerName}/${dirent.name}`).length)
+        .reduce((a, b) => a + b, 0);
+
+    console.log(`Processed ${totalResourcesCount} resources`);
+
+    // Create provider index
+    servicesForIndex = [...new Set(servicesForIndex)];
+    servicesForIndex.sort();
+
+    const half = Math.ceil(servicesForIndex.length / 2);
+    const firstColumnServices = servicesForIndex.slice(0, half);
+    const secondColumnServices = servicesForIndex.slice(half);
+
+    const indexContent = `${headerContent1}
+
+:::info Provider Summary
+
+<div class="row">
+<div class="providerDocColumn">
+<span>total services:&nbsp;<b>${totalServicesCount}</b></span><br />
+<span>total resources:&nbsp;<b>${totalResourcesCount}</b></span><br />
+</div>
+</div>
+
+:::
+
+${headerContent2}
+
+## Services
+<div class="row">
+<div class="providerDocColumn">
+${servicesToMarkdown(providerName, firstColumnServices)}
+</div>
+<div class="providerDocColumn">
+${servicesToMarkdown(providerName, secondColumnServices)}
+</div>
+</div>
+`;
+
+    // Write index
+    const indexPath = path.join(docsDir, 'index.md');
+    fs.writeFileSync(indexPath, indexContent);
+    console.log(`Index file created at ${indexPath}`);
+
+    // Copy MDX file if exists
+    if (fs.existsSync(mdxPath)) {
+        fs.copyFileSync(mdxPath, path.join(docsDir, 'stackql-provider-registry.mdx'));
+        console.log(`MDX file copied`);
+    }
+
+    return {
+        totalServices: totalServicesCount,
+        totalResources: totalResourcesCount,
+        outputPath: docsDir
+    };
+}
+
+// Process each service sequentially
+async function createDocsForService(yamlFilePath, providerName, serviceName, serviceFolder) {
+
+    const data = yaml.load(fs.readFileSync(yamlFilePath, 'utf8'));
+
+    // Create a new SwaggerParser instance
+    let parser = new SwaggerParser();
+    const api = await parser.parse(yamlFilePath); 
+    const ignorePaths = ["$.components.x-stackQL-resources"];
+    let dereferencedAPI;
+
+    try {
+        dereferencedAPI = await deno_openapi_dereferencer.dereferenceApi(api, "$", ignorePaths);
+        dereferencedAPI = await deno_openapi_dereferencer.flattenAllOf(dereferencedAPI);
+    } catch (error) {
+        console.error("error in dereferencing or flattening:", error);
+    }
+
+    // Create service directory
+    if (!fs.existsSync(serviceFolder)) {
+        fs.mkdirSync(serviceFolder, { recursive: true });
+    }
+
+    const resourcesObj = data.components['x-stackQL-resources'];
+
+    if (!resourcesObj) {
+        console.warn(`No resources found in ${yamlFilePath}`);
+        return;
+    }
+
+    const resources = [];
+    for (let resourceName in resourcesObj) {
+    
+        let resourceData = resourcesObj[resourceName];
+        if (!resourceData.id) {
+            console.warn(`No 'id' defined for resource: ${resourceName} in service: ${serviceName}`);
+            continue;
+        }
+    
+        resources.push({
+            name: resourceName,
+            resourceData,
+            dereferencedAPI
+        });
+    }    
+
+    // Process service index
+    const serviceIndexPath = path.join(serviceFolder, 'index.md');
+    const serviceIndexContent = await createServiceIndexContent(providerName, serviceName, resources);
+    fs.writeFileSync(serviceIndexPath, serviceIndexContent);
+
+    // Split into columns and process resources one by one
+    const halfLength = Math.ceil(resources.length / 2);
+    const firstColumn = resources.slice(0, halfLength);
+    const secondColumn = resources.slice(halfLength);
+
+    // Process each resource in first column
+    for (const resource of firstColumn) {
+        await processResource(providerName, serviceFolder, serviceName, resource);
+    }
+
+    // Process each resource in second column
+    for (const resource of secondColumn) {
+        await processResource(providerName, serviceFolder, serviceName, resource);
+    }
+
+    console.log(`Generated documentation for ${serviceName}`);
+}
+
+async function processResource(providerName, serviceFolder, serviceName, resource) {
+    console.log(`Processing resource: ${resource.name}`);
+    
+    const resourceFolder = path.join(serviceFolder, resource.name);
+    if (!fs.existsSync(resourceFolder)) {
+        fs.mkdirSync(resourceFolder, { recursive: true });
+    }
+
+    const resourceIndexPath = path.join(resourceFolder, 'index.md');
+    const resourceIndexContent = await createResourceIndexContent(
+        providerName, 
+        serviceName, 
+        resource.name, 
+        resource.resourceData, 
+        resource.dereferencedAPI,
+    );
+    fs.writeFileSync(resourceIndexPath, resourceIndexContent);
+
+    // After writing the file, force garbage collection if available (optional)
+    if (global.gc) {
+        global.gc();
+    }
+}
+
+async function createServiceIndexContent(providerName, serviceName, resources) {
+    const totalResources = resources.length; // Calculate the total resources
+
+    // Sort resources alphabetically by name
+    resources.sort((a, b) => a.name.localeCompare(b.name));
+
+    const halfLength = Math.ceil(totalResources / 2);
+    const firstColumnResources = resources.slice(0, halfLength);
+    const secondColumnResources = resources.slice(halfLength);
+
+    // Generate the HTML for resource links in the first column
+    const firstColumnLinks = generateResourceLinks(providerName, serviceName, firstColumnResources);
+
+    // Generate the HTML for resource links in the second column
+    const secondColumnLinks = generateResourceLinks(providerName, serviceName, secondColumnResources);
+
+    // Create the markdown content for the service index
+    // You can customize this content as needed
+    return `---
+title: ${serviceName}
+hide_title: false
+hide_table_of_contents: false
+keywords:
+  - ${serviceName}
+  - ${providerName}
+  - stackql
+  - infrastructure-as-code
+  - configuration-as-data
+  - cloud inventory
+description: Query, deploy and manage ${providerName} resources using SQL
+custom_edit_url: null
+image: /img/providers/${providerName}/stackql-${providerName}-provider-featured-image.png
+---
+
+${serviceName} service documentation.
+
+:::info Service Summary
+
+<div class="row">
+<div class="providerDocColumn">
+<span>total resources:&nbsp;<b>${totalResources}</b></span><br />
+</div>
+</div>
+
+:::
+
+## Resources
+<div class="row">
+<div class="providerDocColumn">
+${firstColumnLinks}
+</div>
+<div class="providerDocColumn">
+${secondColumnLinks}
+</div>
+</div>`;
+}
+
+function generateResourceLinks(providerName, serviceName, resources) {
+    // Generate resource links for the service index
+    const resourceLinks = resources.map((resource) => {
+        return `<a href="/providers/${providerName}/${serviceName}/${resource.name}/">${resource.name}</a>`;
+    });
+    return resourceLinks.join('<br />\n');
+}
+
+// Function to convert services to markdown links
+function servicesToMarkdown(providerName, servicesList) {
+    return servicesList.map(service => `<a href="/providers/${providerName}/${service}/">${service}</a><br />`).join('\n');
+}