@itentialopensource/adapter-openstack_cinder 0.1.1

package/swagger/openstack_api_cinder.js

@@ -0,0 +1,602 @@
+/* eslint-disable */
+const fs = require('fs');
+const util = require('util');
+const parse5 = require('parse5');
+const htmlparser2 = require("htmlparser2");
+const domutils = require("domutils");
+const { Parser } = require("htmlparser2");
+const { DomHandler } = require("domhandler");
+const toJsonSchema = require('to-json-schema');
+const { stringify } = require('yaml')
+
+const inspect = obj => util.inspect(obj, { depth: Infinity });
+
+const htmlFile = fs.readFileSync('/home/bota/IAP/apps/node_practice/openstack_API_docs/BlockStorageAPIV3-cinder-documentation.html', { encoding: 'utf-8' })
+const outputFileName = 'cinder-openapi.json'
+
+const changeToCamelCase = (s) => {
+    let operationId = "";
+    s.split(" ").forEach((word, index) => {
+        let firstLetter;
+        if (index === 0) {
+            firstLetter = word[0].toLowerCase()
+        } else {
+            firstLetter = word[0].toUpperCase()
+        }
+        operationId += firstLetter + word.slice(1);
+    })
+    return operationId;
+}
+
+const getTag = (node) => {
+    const siblings = domutils.getSiblings(node);
+    const h2 = domutils.filter(elem => elem.name === 'h2', siblings);
+    let tag = '';
+    if (h2.length !== 1) {
+        console.error('Can not obtain tag')
+        return tag;
+    }
+    const tagNode = h2[0];
+    const tagNodetext = domutils.getChildren(tagNode).filter(elem => elem.type === 'text', tagNode);
+    tag = domutils.innerText(tagNodetext).trim();
+    return tag;
+}
+
+const getRestMethod = (node) => {
+    const n = domutils.findOne((e => e.name === 'span' && e.attribs.class.split(' ').find(item => item === 'label')), [node]);
+    const restMethod = domutils.innerText(n).trim().toLocaleLowerCase();
+    return restMethod;
+}
+
+const getPath = (node) => {
+    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'endpoint-container'), [node]);
+    const childs = domutils.getElementsByTagName('div', domutils.getChildren(n), false);
+    const pathNode = childs[0];
+    const path = domutils.innerText(pathNode).trim();
+    return path;
+}
+
+const getDescription = (node) => {
+    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'endpoint-container'), [node]);
+    const childs = domutils.getElementsByTagName('div', domutils.getChildren(n), false);
+    const descriptionNode = childs[1];
+    const path = domutils.innerText(descriptionNode).trim();
+    return path;
+}
+
+const getDetailsLink = (node) => {
+    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'operation'), [node]);
+    const childs = domutils.getElementsByTagName('a', domutils.getChildren(n), true);
+    if (childs.length !== 1) {
+        return 'No link found';
+    }
+    const detailsLinkNode = childs[0];
+    const detailsLink = detailsLinkNode.attribs.href;
+    return detailsLink;
+}
+
+const getRequestSchema = (section) => {
+    let requestSchema = {}
+    if(!section) return requestSchema;
+    let exampleNode = domutils.findOne((elem) => elem.name === 'div' && (elem.attribs.class && elem.attribs.class.startsWith('highlight-javascript')), [section], true);
+    if (exampleNode) {
+        // console.log('exampleNode 1', exampleNode, domutils.textContent(exampleNode).trim());
+        // console.log('exampleNode 1 CONTENT', domutils.textContent(exampleNode).trim());
+        const s = domutils.textContent(exampleNode).trim();
+        try {
+            let o = JSON.parse(s);
+            requestSchema = toJsonSchema(o);
+        } catch (err) { }
+
+        // const requestSchema = stringify(schema);
+        // console.log('YAML schema', requestSchema)
+    }
+    console.log('example', JSON.stringify(requestSchema));
+    return requestSchema;
+}
+
+const getRequest = (node) => {
+    const requestParameters = [];
+    let requestSchema = {}
+    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), false);
+
+    let requestSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Request');
+    });
+
+    let requestExampleSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Request Example');
+    });
+
+    if (!requestSection && !requestExampleSection) return {requestParameters, requestSchema};
+    
+    requestSchema = getRequestSchema(requestExampleSection);
+
+    const subSections = domutils.getElementsByTagName('section', domutils.getChildren(requestSection), false);
+    let requestParamtersSection = subSections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Parameters');
+    });
+
+    // TODO Add support for multiple requests/responses
+    const table = domutils.getElementsByTagName('tbody', requestParamtersSection || requestSection);
+    const allRows = domutils.getElementsByTagName('tr', table);
+
+    allRows.forEach((row) => {
+        let parameter = {};
+        let columns = domutils.getElementsByTagName('p', row);
+        columns.forEach((c, index) => {
+            let t = domutils.innerText(c).replace(/[\r\n]\s+/g, ' ').trim();
+            switch (index) {
+                case 0:
+                    parameter.name = t;
+                    break;
+                case 1:
+                    parameter.in = t;
+                    break;
+                case 2:
+                    parameter.type = t;
+                    break;
+                case 3:
+                    parameter.description = t;
+                    break;
+            }
+        });
+        requestParameters.push(parameter);
+    });
+
+    return { requestParameters, requestSchema };
+}
+
+const getResponse = (node) => {
+    let response = [];
+    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), false);
+
+    let responseSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Response');
+    });
+    if (!responseSection) return response;
+
+    const subSections = domutils.getElementsByTagName('section', domutils.getChildren(responseSection), false);
+    let responseParamtersSection = subSections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Parameters');
+    });
+    if (!responseParamtersSection) return response
+    // TODO Add support for multiple requests/responses
+    const table = domutils.getElementsByTagName('tbody', responseParamtersSection);
+    const allRows = domutils.getElementsByTagName('tr', table);
+
+    allRows.forEach((row) => {
+        let parameter = {};
+        let columns = domutils.getElementsByTagName('p', row);
+        columns.forEach((c, index) => {
+            let t = domutils.innerText(c).replace(/[\r\n]\s+/g, ' ').trim();
+            switch (index) {
+                case 0:
+                    parameter.name = t;
+                    break;
+                case 1:
+                    parameter.in = t;
+                    break;
+                case 2:
+                    parameter.type = t;
+                    break;
+                case 3:
+                    parameter.description = t;
+                    break;
+            }
+        });
+        response.push(parameter);
+    });
+    return response;
+}
+
+const getStatusCodes = (node) => {
+    let statusCodes = [];
+    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), true);
+
+    let responseSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Response');
+    });
+
+    let statusCodesSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Status Codes');
+    });
+
+    let statusCodesSuccessSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Success');
+    });
+
+    let statusCodesErrorSection = sections.find(s => {
+        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
+        return domutils.textContent(textNodes).includes('Error');
+    });
+
+    if (statusCodesSuccessSection) {
+        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesSuccessSection));
+        rows.forEach((r, index) => {
+            if (index % 2 === 0) {
+                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
+            }
+        })
+    }
+    if (statusCodesErrorSection) {
+        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesErrorSection));
+        rows.forEach((r, index) => {
+            if (index % 2 === 0) {
+                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
+            }
+        })
+    }
+
+    if (statusCodesSection && !statusCodesSuccessSection && !statusCodesErrorSection) {
+        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesSection));
+        rows.forEach((r, index) => {
+            if (index % 2 === 0) {
+                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
+            }
+        })
+    }
+
+    if (statusCodes.length === 0) {
+        statusCodes.push({ code: "200 - OK", reason: "Request was successful." })
+    }
+    return statusCodes;
+}
+
+const extractData = (detailControlNodes, apiDetailControlNodes) => {
+    const N = detailControlNodes.length;
+    let data = [];
+    console.log("Operations found:");
+    for (let i = 0; i < N; i += 1) {
+        console.log('\n======= OPERATION ========\n');
+        const node = detailControlNodes[i];
+        const detailsNode = apiDetailControlNodes[i];
+
+        const tag = getTag(node);
+        const restMethod = getRestMethod(node);
+        const path = getPath(node);
+        const description = getDescription(node);
+        const detailsLink = getDetailsLink(node);
+        console.log(`> Operation:\n| ${restMethod} | ${path} | ${description} | ${detailsLink} |`);
+        const { requestParameters, requestSchema } = getRequest(detailsNode);
+        console.log(`> Request parameters:\n|${inspect(requestParameters)}|${inspect(requestSchema)}`);
+        const response = getResponse(detailsNode);
+        console.log(`> Response parameters:\n|${inspect(response)}|`);
+        const statusCodes = getStatusCodes(detailsNode);
+        console.log(`> Response status codes:\n|${inspect(statusCodes)}|`);
+        data.push({ restMethod, path, description, detailsLink, response, requestParameters, requestSchema, statusCodes, tag });
+    }
+    return data;
+}
+
+const saveToFile = (obj) => {
+    fs.writeFileSync(outputFileName, JSON.stringify(obj));
+    console.log(`File '${outputFileName}' saved.`)
+}
+
+const getBaseOpenApiObj = () => {
+    const openApiObj = {
+        openapi: "3.0.0",
+        info: {
+            version: "1.0",
+            title: "Openstack Cinder API"
+        },
+        servers: [
+            {
+                url: 'http://{host}:{port}',
+                description: 'Block Storage service (cinder) administrative endpoint',
+                variables: {
+                    host: {
+                        default: "localhost"
+                    },
+                    port: {
+                        default: "8776"
+                    }
+                }
+            }
+        ],
+        paths: {
+        },
+        components: { schemas: {}, responses: {}, parameters: {}, examples: {}, requestBodies: {}, headers: {}, securitySchemes: {}, links: {}, callbacks: {} },
+        security: [],
+        tags: [],
+        externalDocs: {
+            description: 'Openstack Cinder Block Storage API Operations',
+            url: 'https://docs.openstack.org/api-ref/block-storage/v3/index.html?expanded=#'
+        }
+    }
+
+    openApiObj.components.securitySchemes = {
+        token: {
+            type: "apiKey",
+            description: 'Token authentication',
+            name: "X-Auth-Token",
+            in: "header"
+        }
+    }
+
+    openApiObj.security = [{ "token": [] }];
+
+    return openApiObj;
+}
+
+const buildQueryParameters = (operation) => {
+    let params = []
+    let queryParams = operation.filter((item) => item.in === 'query')
+    queryParams.forEach((p) => {
+        isOptional = false
+        if (p.name.indexOf('Optional') !== -1) {
+            isOptional = true;
+            p.name = p.name.split(' ')[0].trim();
+        }
+        if (p.type.includes('key-only')) {
+            params.push({
+                "name": p.name,
+                "in": p.in,
+                "required": !isOptional,
+                "schema": {
+                    "type": 'boolean',
+                    "description": p.description
+                },
+                "allowEmptyValue": true
+            })
+        } else {
+            if (p.type.includes('bool')) p.type = 'boolean'
+            params.push({
+                "name": p.name,
+                "in": p.in,
+                "required": !isOptional,
+                "schema": {
+                    "type": p.type,
+                    "description": p.description
+                }
+            })
+        }
+    });
+    return params;
+}
+
+const buildHeaderParameters = (operation) => {
+    let params = []
+    let headerParams = operation.filter((item) => item.in === 'header')
+
+    headerParams.forEach((p) => {
+        isOptional = false
+        if (p.name.indexOf('Optional') !== -1) {
+            isOptional = true;
+            p.name = p.name.split(' ')[0].trim();
+        }
+        params.push({
+            "name": p.name,
+            "in": p.in,
+            "required": !isOptional,
+            "schema": {
+                "type": p.type,
+                "description": p.description
+            }
+        })
+    });
+    return params;
+}
+
+const buildPathParameters = (operation) => {
+    let params = []
+    let pathParams = operation.filter((item) => item.in === 'path')
+    pathParams.forEach((p) => {
+        isOptional = false
+        if (p.name.indexOf('Optional') !== -1) {
+            isOptional = true;
+            p.name = p.name.split(' ')[0].trim();
+        }
+        params.push({
+            "name": p.name,
+            "in": p.in,
+            "required": !isOptional,
+            "schema": {
+                "type": p.type,
+                "description": p.description
+            }
+        })
+    });
+    return params;
+}
+
+const buildRequestBody = (request, requestSchema) => {
+    // let bodyParams = request.filter((item) => item.in === 'body')
+    let requestBody = {
+        "content": {
+            "application/json": {
+                "schema": requestSchema
+            }
+        }
+    };
+
+    isBodyPresent = Object.keys(requestSchema).length > 0;
+    // required = [];
+    // bodyParams.forEach((bodyParam) => {
+    //     let propertyName = bodyParam.name.split(' ')[0];
+    //     if (!bodyParam.name.includes('Optional')) {
+    //         required.push(propertyName);
+    //     }
+    //     requestBody.content["application/json"].schema.properties[propertyName] = {};
+    //     if (bodyParam.type === 'list') {
+    //         bodyParam.type = 'array';
+    //     }
+    //     requestBody.content["application/json"].schema.properties[propertyName].type = bodyParam.type
+    //     if (bodyParam.type === 'array') {
+    //         let type = '';
+    //         if (bodyParam.description.includes('object')) {
+    //             type = 'object';
+    //         } else if (bodyParam.description.includes('string')) {
+    //             type = 'string';
+    //         } else {
+    //             type = 'string';
+    //         }
+    //         requestBody.content["application/json"].schema.properties[propertyName].items = {
+    //             type
+    //         }
+    //     }
+    //     requestBody.content["application/json"].schema.properties[propertyName].description = bodyParam.description
+    // });
+
+    // if (required.length > 0) {
+    //     requestBody.content["application/json"].schema.required = required;
+    // }
+
+    return { isBodyPresent, requestBody };
+}
+
+const buildResponseBody = (operation) => {
+    const responseBody = {
+        "headers": {},
+        "content": {
+            "application/json": {
+                "schema": {
+                    "type": "object",
+                    "properties": {}
+                }
+            }
+        }
+    };
+
+    let headerParams = operation.filter((item) => item.in === 'header');
+    headerParams.forEach((headerParam) => {
+        responseBody.headers[headerParam.name] = {
+            "description": headerParam.description,
+            "schema": {
+                "type": headerParam.type
+            }
+        };
+    });
+
+    let bodyParams = operation.filter((item) => item.in === 'body')
+    isBodyPresent = bodyParams.length > 0;
+    const required = new Set();
+    bodyParams.forEach((bodyParam) => {
+        let propertyName = bodyParam.name.split(' ')[0];
+        if (!bodyParam.name.includes('Optional')) {
+            required.add(propertyName);
+        }
+        responseBody.content["application/json"].schema.properties[propertyName] = {};
+        if (bodyParam.type === 'list') {
+            bodyParam.type = 'array';
+        }
+        responseBody.content["application/json"].schema.properties[propertyName].type = bodyParam.type
+        if (bodyParam.type === 'array') {
+            let type = '';
+            if (bodyParam.description.includes('object')) {
+                type = 'object';
+            } else if (bodyParam.description.includes('string')) {
+                type = 'string';
+            } else {
+                type = 'string';
+            }
+            responseBody.content["application/json"].schema.properties[propertyName].items = {
+                type
+            }
+        }
+        responseBody.content["application/json"].schema.properties[propertyName].description = bodyParam.description
+    });
+
+    if (required.size > 0) {
+        responseBody.content["application/json"].schema.required = [...required];
+    }
+
+    return { isBodyPresent, responseBody };
+}
+
+const buildResponses = (statusCodes, response) => {
+    const responses = {}
+    statusCodes.forEach((item) => {
+        let code = item.code.split(' ')[0];
+        responses[code] = { "description": item.reason }
+    });
+    // const { isBodyPresent, responseBody } = buildResponseBody(response);
+    // const successfullResponse = Object.keys(responses).find(k => k.startsWith('2'));
+    // console.log('successfullResponse', successfullResponse);
+    // if (isBodyPresent) {
+    //     console.log('responseBody', responseBody);
+    //     Object.assign(responses[successfullResponse], responseBody);
+    // }
+    return responses;
+}
+
+const createOpenApiFile = (data) => {
+    const openApiObj = getBaseOpenApiObj();
+
+    const allEndpoints = new Set()
+    data.forEach((item) => allEndpoints.add(item.path))
+    console.log('\nFound:', allEndpoints.size, 'endpoints\n');
+
+    allEndpoints.forEach(endpoint => {
+        console.log('Processing endpoint', endpoint);
+
+        allEndpointOperations = data.filter(item => item.path === endpoint)
+        console.log(`Found ${allEndpointOperations.length} operations\n`);
+
+        openApiObj.paths[endpoint] = {};
+
+        let pathOpenAPI = openApiObj.paths[endpoint];
+
+        allEndpointOperations.forEach(op => {
+            if (pathOpenAPI[op.restMethod]) {
+                console.error('More then 1 operation for same (URIpath, restMethod) set, skipping...');
+                return;
+            }
+            pathOpenAPI[op.restMethod] = { operationId: changeToCamelCase(op.description) };
+            pathOpenAPI[op.restMethod].summary = op.description;
+            pathOpenAPI[op.restMethod].externalDocs = { url: op.detailsLink }
+
+            let queryParams = buildQueryParameters(op.requestParameters);
+            let headerParams = buildHeaderParameters(op.requestParameters);
+            let pathParams = buildPathParameters(op.requestParameters);
+            if (queryParams.length + headerParams.length + pathParams.length > 0) {
+                pathOpenAPI[op.restMethod].parameters = queryParams.concat(headerParams).concat(pathParams);
+            }
+
+            let { isBodyPresent, requestBody } = buildRequestBody(op.requestParameters, op.requestSchema);
+            if (isBodyPresent) {
+                pathOpenAPI[op.restMethod].requestBody = requestBody;
+            }
+
+            pathOpenAPI[op.restMethod].responses = buildResponses(op.statusCodes, op.response);
+
+            if (op.tag) {
+                if (!pathOpenAPI[op.restMethod].tags) {
+                    pathOpenAPI[op.restMethod].tags = [];
+                }
+                pathOpenAPI[op.restMethod].tags.push(op.tag);
+            }
+        })
+    })
+
+    saveToFile(openApiObj);
+}
+
+const handler = new DomHandler((error, dom) => {
+    if (error) {
+    } else {
+        let detailControlNodes = domutils.findAll((element) => element.name === "section" && element.attribs.class === 'detail-control', dom);
+        let apiDetailControlNodes = domutils.findAll((element) => element.name === "section" && element.attribs.class === 'api-detail collapse', dom);
+
+        console.log('Nodes count:', detailControlNodes.length, apiDetailControlNodes.length);
+        if (detailControlNodes.length !== apiDetailControlNodes.length) {
+            console.error('Count mismatch');
+            throw new Error();
+        }
+        const data = extractData(detailControlNodes, apiDetailControlNodes);
+        createOpenApiFile(data);
+    }
+});
+
+const parser = new Parser(handler);
+parser.write(htmlFile);
+parser.end();

package/test/integration/adapterTestBasicGet.js

@@ -0,0 +1,85 @@
+/* @copyright Itential, LLC 2020 */
+
+/* global describe context before after */
+/* eslint global-require: warn */
+/* eslint no-unused-vars: warn */
+/* eslint import/no-extraneous-dependencies: warn */
+/* eslint import/no-dynamic-require: warn */
+/* eslint import/no-unresolved: warn */
+
+const mocha = require('mocha');
+const path = require('path');
+const assert = require('assert');
+const itParam = require('mocha-param');
+
+const utils = require('../../utils/tbUtils');
+const basicGet = require('../../utils/basicGet');
+const { name } = require('../../package.json');
+const { methods } = require('../../pronghorn.json');
+
+const getPronghornProps = (iapDir) => {
+  const { Discovery } = require('@itential/itential-utils');
+  console.log('Retrieving properties.json file...');
+  const rawProps = require(path.join(iapDir, 'properties.json'));
+  console.log('Decrypting properties...');
+  const discovery = new Discovery();
+  const pronghornProps = utils.decryptProperties(rawProps, path.join(__dirname, '..'), discovery);
+  console.log('Found properties.\n');
+  return pronghornProps;
+};
+
+let a;
+
+describe('[integration] Adapter BasicGET Test', () => {
+  context('Testing GET calls without query parameters', () => {
+    before(async () => {
+      const iapDir = path.join(__dirname, '../../../../../');
+      if (!utils.areWeUnderIAPinstallationDirectory()) {
+        const sampleProperties = require('../../sampleProperties.json');
+        const adapter = { properties: sampleProperties };
+        a = basicGet.getAdapterInstance(adapter);
+      } else {
+        const pronghornProps = getPronghornProps(iapDir);
+        console.log('Connecting to Database...');
+        const database = await basicGet.connect(pronghornProps);
+        console.log('Connection established.');
+        const adapter = await database.collection(utils.SERVICE_CONFIGS_COLLECTION).findOne(
+          { model: name }
+        );
+        a = basicGet.getAdapterInstance(adapter);
+      }
+    });
+
+    after((done) => {
+      done();
+    });
+
+    const basicGets = methods.filter((method) => (method.route.verb === 'GET' && method.input.length === 0));
+    if (basicGets.length === 0) {
+      console.log('No non-parameter GET calls found.');
+      process.exit(0);
+    }
+    const functionNames = basicGets.map((g) => g.name);
+    const request = function request(f, ad) {
+      return new Promise((resolve, reject) => {
+        const getRespCode = (resp) => {
+          if (resp) {
+            if (resp.metrics.code !== 200) {
+              console.log('\x1b[31m', `Testing ${f} \nResponseCode: ${resp.metrics.code}`);
+            }
+            resolve(resp.metrics.code);
+          } else {
+            console.log('\x1b[31m', `call ${f} results in failure`);
+            reject(new Error(`${f} failed`));
+          }
+        };
+        ad[f](getRespCode, console.log);
+      });
+    };
+
+    itParam('GET call should return 200', functionNames, (fname) => {
+      console.log(`\t ${fname}`);
+      return request(fname, a).then((result) => assert.equal(result, 200));
+    });
+  });
+});