helium-npm 1.0.4

package/index.js

@@ -0,0 +1,11 @@
+import {Command} from "commander";
+import {create} from "./lib/createHeliumPkgFile.js";
+
+const command = new Command();
+command
+    .name('create')
+    .description('CLI to create helium json')
+    .action((str, options) => {
+        create();
+    });
+command.parse();

package/lib/createHeliumPkgFile.js

@@ -0,0 +1,57 @@
+import fetch from "node-fetch";
+import Promise from "bluebird";
+import stringify from 'json-stringify-pretty-compact';
+import {getMavenHandler} from "./getMavenArtifactInfo.js";
+import {fetchHelium} from "./fetchHeliumPkgInfo.js";
+import * as fs from "fs";
+import {getBasicMavenHandler} from "./getBasicMavenArtifactInfo.js";
+
+const pushPkgInfoToTmp = async function() {
+    const response = await fetch('https://zeppelin.apache.org/helium.json')
+    const data = await response.json();
+    return data[0];
+}
+
+const delay = (timeToDelay) => new Promise((resolve) => setTimeout(resolve, timeToDelay))
+
+
+export function create() {
+    pushPkgInfoToTmp()
+        .then(async (response) => {
+            const result = await getBasicMavenHandler(() => {});
+            for (const body of result) {
+                for (const key of Object.keys(body)) {
+                    response[key] = body[key];
+                }
+            }
+            return response;
+        })
+        .then(async (response) => {
+            const result = await getMavenHandler(() => {});
+            for (const body of result) {
+                for (const key of Object.keys(body)) {
+                    response[key] = body[key];
+                }
+            }
+            return response;
+        })
+        .then(async (response) => {
+            const result = await fetchHelium();
+            for (const body of result) {
+                for (const key of Object.keys(body)) {
+                    response[key] = body[key];
+                }
+            }
+            return response;
+        })
+        .then(async (response) => {
+            await delay(8000)
+            const result = [response];
+            fs.writeFileSync('./helium.json', stringify(result), (err) => {
+                console.log("error!!!");
+            })
+        })
+    // .then(async (response) => {
+    //     await fetchHelium();
+    // })
+}

package/lib/fetchHeliumPkgInfo.js

@@ -0,0 +1,88 @@
+import Promise from "bluebird";
+import stringify from "json-stringify-pretty-compact";
+import RegistryClient from 'npm-registry-client';
+import dependants from 'npm-dependants'
+
+const depList = [
+    "zeppelin-vis",
+    "zeppelin-spell",
+]
+
+const removeExtraSquareBracket = function(jsonArray) {
+    jsonArray = jsonArray.map(function(e){
+        return stringify(e)
+    })
+    return jsonArray.join(",")
+}
+
+const setEachVersionInfo = function(dependency, data) {
+    const eachVerInfo = {}
+    const versions = data.versions
+    const defaultIcon = '<i class="fa fa-question-circle"></i>'
+    const type = (dependency == "zeppelin-vis") ? "VISUALIZATION" : "SPELL"
+
+    for (let ver in versions) {
+        const key = versions[ver]
+        // to keep "latest" version tag, use artifact's one
+        const verTag = key._id.split('@')[1]
+
+        if (ver == data["dist-tags"].latest) {
+            ver = "latest"
+        }
+        eachVerInfo[ver] = {
+            type: type,
+            name: key.name,
+            version: verTag,
+            published: data.time[verTag],
+            artifact: key._id,
+            author:
+                (key.author?.name == undefined) && (key.maintainers[0].name == undefined)
+                    ?
+                    'unknown'
+                    :
+                    key.author?.name || key.maintainers[0].name,
+            description:
+                (key.description == undefined) ? 'unknown' : key.description,
+            license:
+                (key.license == undefined) ? 'unknown' : key.license,
+            icon:
+                (key.helium)
+                    ?
+                    (key.helium.icon == undefined) ? defaultIcon : key.helium.icon
+                    :
+                    (key.icon == undefined) ? defaultIcon : key.icon,
+            config:
+                (key.helium) ? key.helium.config : undefined,
+            spell:
+                (key.helium) ? key.helium.spell : undefined
+        }
+    }
+    return eachVerInfo
+}
+
+const finalResult = []
+const client = new RegistryClient();
+async function getDependencies(packageName) {
+    await client.get(`https://registry.npmjs.org/${packageName}`, { timeout: 1000 }, (error, data) => {
+        if (error) {
+            console.error(error.message);
+            return reject(error)
+        };
+
+        const pkgInfo = {}
+        pkgInfo[data.name] = setEachVersionInfo(packageName, data)
+        finalResult.push(pkgInfo)
+        // var result = removeExtraSquareBracket(finalResult).replace(/]|[[]/g, '').trim()
+    });
+}
+
+const delay = (timeToDelay) => new Promise((resolve) => setTimeout(resolve, timeToDelay))
+export async function fetchHelium() {
+    depList.map(async function(dependency) {
+        for await (const dependant of dependants(dependency)) {
+            await getDependencies(dependant);
+        }
+    })
+    await delay(10000);
+    return finalResult;
+}

package/lib/getBasicMavenArtifactInfo.js

@@ -0,0 +1,275 @@
+import _ from 'underscore';
+import moment from "moment";
+import Promise from 'bluebird';
+import fetch from "node-fetch";
+import xml2jsImport from 'xml2js';
+
+const xml2js = Promise.promisifyAll(xml2jsImport)
+
+const zeppelinList = []
+const pomUriList = []
+const responseList = []
+const bodyList = []
+const finalArtifactList = []
+
+function filterLessThanOneYear(standard) {
+    const duration = moment.duration({'year': 1})
+    const oneYearAgo = moment().subtract(duration).format()
+
+    // filter only artifacts created within 1 year from now
+    return moment(standard).isAfter(oneYearAgo)
+}
+
+async function searchOnlyZeppelinRelatedArtifact(uri) {
+    const options = {
+        uri: uri,
+        json: true
+    }
+
+    return fetch(uri)
+        .then(async (result) => {
+            const response = (await result.json()).response
+            const body = response.docs;
+
+            for (const artifact in body) {
+                const artifactId = body[artifact].a
+                const groupId = body[artifact].g
+                let published = body[artifact].timestamp
+                published = moment(published).format()
+
+                if (filterLessThanOneYear(published)) {
+                    zeppelinList.push({
+                        groupId,
+                        artifactId
+                    })
+                }
+            }
+            return zeppelinList
+        });
+}
+
+function createSlashedGroupId(groupId) {
+    let slashedGroupId = ""
+
+    for (const id in groupId) {
+        const text = groupId[id] + '/'
+        slashedGroupId = slashedGroupId.concat(text)
+    }
+
+    return slashedGroupId
+}
+
+function createPomUrl(groupId, artifactId, version) {
+    const baseUri = 'http://repo1.maven.org/maven2/'
+    const slashedGroupId = createSlashedGroupId(groupId)
+
+    if (slashedGroupId) {
+        const uri = baseUri + slashedGroupId + artifactId + '/' + version + '/'
+        return uri + artifactId + '-' + version + '.pom';
+    }
+    return undefined;
+}
+
+async function getAllVersionInfo(zeppelinList) {
+    // get all version of an artifact
+    return _.map(zeppelinList, function (resources) {
+        const baseUri =
+            'http://search.maven.org/solrsearch/select?q=g:%22'
+            + resources.groupId + '%22+AND+a:%22' + resources.artifactId + '%22&core=gav&rows=20'
+        const options = {
+            uri: baseUri,
+            json: true
+        }
+
+        return fetch(baseUri)
+            .then(async (result) => {
+                const response = (await result.json()).response
+                const body = response.docs;
+                const pomUriListEachVer = {}
+
+                for (const ver in body) {
+
+                    const artifactId = body[ver].a
+                    const groupId = body[ver].g.split('.')
+                    const version = body[ver].v
+                    let published = body[ver].timestamp
+                    published = moment(published).format()
+
+                    if (filterLessThanOneYear(published) && createSlashedGroupId(groupId)) {
+                        const pomUri = createPomUrl(groupId, artifactId, version)
+
+                        pomUriListEachVer[version] = {
+                            artifactId,
+                            version,
+                            published,
+                            pomUri
+                        }
+                    }
+                }
+                pomUriList.push(pomUriListEachVer)
+                return pomUriList
+            });
+    })
+}
+
+function getEachPomFileContent(pomUriList) {
+    return _.map(pomUriList, function (result) {
+
+        return Promise.all(_.map(result, function (ver) {
+            const options = {
+                uri: ver.pomUri
+            }
+            const artifactId = ver.artifactId
+            const published = ver.published
+            const version = ver.version
+
+            const pomUri = ver.pomUri.replace('http', 'https')
+            return fetch(pomUri)
+                .then(async (result) => {
+                    const response = (await result.text())
+
+                    const eachVersionResponse = {
+                        version: version,
+                        artifactId: artifactId,
+                        published: published,
+                        response: response,
+                    };
+                    return eachVersionResponse;
+                });
+        })).then(function (result) {
+            responseList.push(result);
+        })
+    })
+}
+
+function parseXmlToJson(responseList) {
+    return _.map(responseList, function (response) {
+
+        return Promise.all(_.map(response, function (ver) {
+            // parse each pom.xml to JSON format
+            return xml2js.parseStringAsync(ver.response, {explicitArray: false})
+                .then(function (result) {
+                    const projectDeps = result.project.dependencies
+                    const projectDepManages = result.project.dependencyManagement
+                    const name = result.project.artifactId
+                    const groupId = result.project.groupId ? result.project.groupId : result.project.parent.groupId
+                    const artifactId = result.project.artifactId
+                    const version = result.project.version ? result.project.version : result.project.parent.version
+                    const description = result.project.description ? result.project.description : result.project.name
+                    const published = ver.published
+
+                    let dependencies;
+                    if (projectDeps) {
+                        dependencies = projectDeps.dependency;
+                    } else if (projectDepManages) {
+                        dependencies = projectDepManages.dependencies.dependency;
+                    } else {
+                        dependencies = undefined;
+                    }
+
+                    const eachVerBodyList = {
+                        name: name,
+                        groupId: groupId,
+                        artifactId: artifactId,
+                        version: version,
+                        description: description,
+                        published: published,
+                        dependencies: dependencies,
+                    };
+                    return eachVerBodyList;
+                })
+        }))
+            .then(function (result) {
+                bodyList.push(result);
+            })
+    })
+}
+
+function filterWithGivenArtifact(bodyList) {
+    return _.map(bodyList, function (version) {
+
+        return Promise.all(_.map(version, function (dep, index) {
+            const type = 'INTERPRETER'
+            const license = 'Apache-2.0'
+            const icon = '<i class=\"fa fa-rocket\"></i>'
+            const dependency = dep.dependencies
+            const name = dep.name
+            const groupId = dep.groupId
+            const artifactId = dep.artifactId
+            const version = (index == 0) ? 'latest' : dep.version
+            const artifact = artifactId + '@' + dep.version
+            const description = dep.description ? dep.description : name
+            const published = dep.published
+
+            const interpreters = [
+                'spark-interpreter', 'zeppelin-jupyter-interpreter', 'zeppelin-display', 'zeppelin-flink',
+                'spark-scala-2.11', 'sap', 'spark1-shims', 'spark2-shims', 'spark-scala-parent',
+                'spark-scala-2.10', 'spark-shims', 'zeppelin-beam', 'zeppelin-ignite_2.11', 'zeppelin-kylin',
+                'zeppelin-ignite_2.10', 'zeppelin-cassandra_2.11', 'zeppelin-flink_2.11', 'zeppelin-lens',
+                'zeppelin-cassandra_2.10', 'zeppelin-solr', 'zeppelin-viyadb'
+            ];
+            const result = dependency ?
+                (
+                    _.findWhere(dependency, {artifactId: 'zeppelin-interpreter'})
+                    || interpreters.includes(dep.name)
+                )
+                : undefined
+
+            return result
+                ?
+                {
+                    type: type,
+                    name: name,
+                    version: version,
+                    published: published,
+                    artifact: artifact,
+                    description: description,
+                    license: license,
+                    icon: icon,
+                    groupId: groupId,
+                    artifactId: artifactId
+                }
+                :
+                undefined;
+        }))
+            .then(function (result) {
+                const resultByVersion = _.indexBy(_.omit(result, _.isUndefined), 'version')
+                for (const key in resultByVersion) {
+                    const obj = resultByVersion[key]
+                    if (obj.version == 'latest') {
+                        obj.version = obj.artifact.split('@')[1]
+                    }
+                }
+
+                finalArtifactList.push(resultByVersion)
+            })
+    })
+}
+
+const delay = (timeToDelay) => new Promise((resolve) => setTimeout(resolve, timeToDelay))
+const result = [];
+
+export async function getBasicMavenHandler(callback) {
+    const uri = 'http://search.maven.org/solrsearch/select?q=zeppelin&rows=200';
+    await searchOnlyZeppelinRelatedArtifact(uri);
+    await delay(5000);
+    await getAllVersionInfo(zeppelinList);
+    await delay(3000);
+    await getEachPomFileContent(pomUriList);
+    await delay(3000);
+    await parseXmlToJson(responseList);
+    await delay(3000);
+    await filterWithGivenArtifact(bodyList);
+    await delay(3000);
+    _.map(finalArtifactList, function (artifact) {
+        if (!_.isEmpty(artifact)) {
+            for (const key in artifact) {
+                const artifactId = artifact[key].artifactId
+                const body = {}
+                body[artifactId] = artifact
+                result.push(body);
+            }
+        }
+    });
+    return result;
+}

package/lib/getMavenArtifactInfo.js

@@ -0,0 +1,293 @@
+import _ from 'underscore';
+import moment from "moment";
+import Promise from 'bluebird';
+import fetch from "node-fetch";
+import xml2jsImport from 'xml2js';
+
+const xml2js = Promise.promisifyAll(xml2jsImport)
+
+const zeppelinList = []
+const pomUriList = []
+const responseList = []
+const bodyList = []
+const finalArtifactList = []
+
+function filterLessThanOneYear(standard) {
+    const duration = moment.duration({'year': 1})
+    const oneYearAgo = moment().subtract(duration).format()
+
+    // filter only artifacts created within 1 year from now
+    return moment(standard).isAfter(oneYearAgo)
+}
+
+async function searchOnlyZeppelinRelatedArtifact(uri) {
+    const options = {
+        uri: uri,
+        json: true
+    }
+
+    return fetch(uri)
+        .then(async (result) => {
+            const response = (await result.json()).response
+            const body = response.docs;
+
+            for (const artifact in body) {
+                const artifactId = body[artifact].a
+                const groupId = body[artifact].g
+                let published = body[artifact].timestamp
+                published = moment(published).format()
+
+                if (filterLessThanOneYear(published)) {
+                    zeppelinList.push({
+                        groupId,
+                        artifactId
+                    })
+                }
+            }
+            return zeppelinList
+        });
+}
+
+function createSlashedGroupId(groupId) {
+    let slashedGroupId = ""
+
+    for (const id in groupId) {
+        const text = groupId[id] + '/'
+        slashedGroupId = slashedGroupId.concat(text)
+    }
+
+    return slashedGroupId
+}
+
+function createPomUrl(groupId, artifactId, version) {
+    const baseUri = 'http://repo1.maven.org/maven2/'
+    const slashedGroupId = createSlashedGroupId(groupId)
+
+    if (slashedGroupId) {
+        const uri = baseUri + slashedGroupId + artifactId + '/' + version + '/'
+        return uri + artifactId + '-' + version + '.pom';
+    }
+    return undefined;
+}
+
+async function getAllVersionInfo(zeppelinList) {
+    // get all version of an artifact
+    return _.map(zeppelinList, function (resources) {
+        const baseUri =
+            'http://search.maven.org/solrsearch/select?q=g:%22'
+            + resources.groupId + '%22+AND+a:%22' + resources.artifactId + '%22&core=gav&rows=20'
+        const options = {
+            uri: baseUri,
+            json: true
+        }
+
+        return fetch(baseUri)
+            .then(async (result) => {
+                const response = (await result.json()).response
+                const body = response.docs;
+                const pomUriListEachVer = {}
+
+                for (const ver in body) {
+
+                    const artifactId = body[ver].a
+                    const groupId = body[ver].g.split('.')
+                    const version = body[ver].v
+                    let published = body[ver].timestamp
+                    published = moment(published).format()
+
+                    if (filterLessThanOneYear(published) && createSlashedGroupId(groupId)) {
+                        const pomUri = createPomUrl(groupId, artifactId, version)
+
+                        pomUriListEachVer[version] = {
+                            artifactId,
+                            version,
+                            published,
+                            pomUri
+                        }
+                    }
+                }
+                pomUriList.push(pomUriListEachVer)
+                return pomUriList
+            });
+    })
+}
+
+function getEachPomFileContent(pomUriList) {
+    return _.map(pomUriList, function (result) {
+
+        return Promise.all(_.map(result, function (ver) {
+            const options = {
+                uri: ver.pomUri
+            }
+            const artifactId = ver.artifactId
+            const published = ver.published
+            const version = ver.version
+
+            const pomUri = ver.pomUri.replace('http', 'https')
+            return fetch(pomUri)
+                .then(async (result) => {
+                    const response = (await result.text())
+
+                    const eachVersionResponse = {
+                        version: version,
+                        artifactId: artifactId,
+                        published: published,
+                        response: response,
+                    };
+                    return eachVersionResponse;
+                });
+        })).then(function (result) {
+            responseList.push(result);
+        })
+    })
+}
+
+function parseXmlToJson(responseList) {
+    return _.map(responseList, function (response) {
+
+        return Promise.all(_.map(response, function (ver) {
+            // parse each pom.xml to JSON format
+            return xml2js.parseStringAsync(ver.response, {explicitArray: false})
+                .then(function (result) {
+                    const projectDeps = result.project.dependencies
+                    const projectDepManages = result.project.dependencyManagement
+                    const name = result.project.artifactId
+                    const groupId = result.project.groupId ? result.project.groupId : result.project.parent.groupId
+                    const artifactId = result.project.artifactId
+                    const version = result.project.version ? result.project.version : result.project.parent.version
+                    const description = result.project.description ? result.project.description : result.project.name
+                    const published = ver.published
+
+                    let dependencies;
+                    if (projectDeps) {
+                        dependencies = projectDeps.dependency;
+                    } else if (projectDepManages) {
+                        dependencies = projectDepManages.dependencies.dependency;
+                    } else {
+                        dependencies = undefined;
+                    }
+
+                    const eachVerBodyList = {
+                        name: name,
+                        groupId: groupId,
+                        artifactId: artifactId,
+                        version: version,
+                        description: description,
+                        published: published,
+                        dependencies: dependencies,
+                    };
+                    return eachVerBodyList;
+                })
+        }))
+            .then(function (result) {
+                bodyList.push(result);
+            })
+    })
+}
+
+function filterWithGivenArtifact(bodyList) {
+    return _.map(bodyList, function (version) {
+
+        return Promise.all(_.map(version, function (dep, index) {
+            const type = 'INTERPRETER'
+            const license = 'Apache-2.0'
+            const icon = '<i class=\"fa fa-rocket\"></i>'
+            const dependency = dep.dependencies
+            const name = dep.name
+            const groupId = dep.groupId
+            const artifactId = dep.artifactId
+            const version = (index == 0) ? 'latest' : dep.version
+            const artifact = artifactId + '@' + dep.version
+            const description = dep.description ? dep.description : name
+            const published = dep.published
+
+            // const result = dependency ? _.findWhere(dependency, {artifactId: 'zeppelin-interpreter'}) : undefined
+
+            return result
+                ?
+                {
+                    type: type,
+                    name: name,
+                    version: version,
+                    published: published,
+                    artifact: artifact,
+                    description: description,
+                    license: license,
+                    icon: icon,
+                    groupId: groupId,
+                    artifactId: artifactId
+                }
+                :
+                undefined;
+        }))
+            .then(function (result) {
+                const resultByVersion = _.indexBy(_.omit(result, _.isUndefined), 'version')
+                for (const key in resultByVersion) {
+                    const obj = resultByVersion[key]
+                    if (obj.version == 'latest') {
+                        obj.version = obj.artifact.split('@')[1]
+                    }
+                }
+
+                finalArtifactList.push(resultByVersion)
+            })
+    })
+}
+
+function getMavenUris() {
+    const dependencies = [
+        'zeppelin-alluxio',
+        'zeppelin-angular',
+        'zeppelin-bigquery',
+        'zeppelin-cassandra',
+        'zeppelin-elasticsearch',
+        'zeppelin-file',
+        'zeppelin-flink',
+        'zeppelin-groovy',
+        'zeppelin-hbase',
+        'zeppelin-java',
+        'zeppelin-jdbc',
+        'zeppelin-livy',
+        'zeppelin-markdown',
+        'zeppelin-neo4j',
+        'zeppelin-python',
+        'zeppelin-shell',
+        'zeppelin-sparql',
+    ];
+    const uris = [];
+    for (const dependency of dependencies) {
+        const uri = `http://search.maven.org/solrsearch/select?q=${dependency}&rows=100`;
+        uris.push(uri);
+    }
+    return uris;
+}
+
+const delay = (timeToDelay) => new Promise((resolve) => setTimeout(resolve, timeToDelay))
+const result = [];
+
+export async function getMavenHandler(callback) {
+    let mavenUris = getMavenUris();
+    for (const uri of mavenUris) {
+        await searchOnlyZeppelinRelatedArtifact(uri);
+    }
+    await delay(5000);
+    await getAllVersionInfo(zeppelinList);
+    await delay(3000);
+    await getEachPomFileContent(pomUriList);
+    await delay(3000);
+    await parseXmlToJson(responseList);
+    await delay(3000);
+    await filterWithGivenArtifact(bodyList);
+    await delay(3000);
+    _.map(finalArtifactList, function (artifact) {
+        if (!_.isEmpty(artifact)) {
+            for (const key in artifact) {
+                const artifactId = artifact[key].artifactId
+                const body = {}
+                body[artifactId] = artifact
+                result.push(body);
+            }
+        }
+    });
+    return result;
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "helium-npm",
+  "version": "1.0.4",
+  "description": "create helium json",
+  "main": "index.js",
+  "bin": {
+    "helium-cli": "node ./index.js"
+  },
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "pythonstrup",
+  "license": "ISC",
+  "dependencies": {
+    "bluebird": "^3.7.2",
+    "commander": "^12.1.0",
+    "console-ui": "^3.1.2",
+    "json-stringify-pretty-compact": "^4.0.0",
+    "moment": "^2.30.1",
+    "node-fetch": "^3.3.2",
+    "npm-dependants": "^2.1.2",
+    "npm-registry-client": "^8.6.0",
+    "prttty": "^0.9.1",
+    "silent-npm-registry-client": "^3.0.1",
+    "stream": "^0.0.3",
+    "underscore": "^1.13.6",
+    "xml2js": "^0.6.2"
+  }
+}